├── .gitignore
├── LICENSE
├── Makefile
├── README
├── README.md
├── doc
├── edoc-info
├── erlang.png
├── kvdb.md
├── kvdb_conf.md
├── kvdb_cron.md
├── kvdb_cron_parse.md
├── kvdb_cron_scan.md
├── kvdb_ets_dumper.md
├── kvdb_export.md
├── kvdb_paired.md
├── kvdb_proxy_sup.md
├── kvdb_queue.md
├── kvdb_riak.md
├── kvdb_riak_mapred.md
├── kvdb_riak_proxy.md
├── kvdb_schema.md
├── kvdb_schema_events.md
├── overview.edoc
└── stylesheet.css
├── ebin
└── .gitignore
├── include
├── kvdb.hrl
├── kvdb_conf.hrl
└── riak_kv_wm_raw.hrl
├── priv
├── all_apps.config
├── backends.script
└── excluded_apps.script
├── rebar.config
├── rebar.config.script
├── src
├── kvdb.app.src
├── kvdb.erl
├── kvdb_app.erl
├── kvdb_conf.erl
├── kvdb_cron.erl
├── kvdb_cron_parse.yrl
├── kvdb_cron_scan.xrl
├── kvdb_cron_sup.erl
├── kvdb_db_sup.erl
├── kvdb_diff.erl
├── kvdb_direct.erl
├── kvdb_ets.erl
├── kvdb_ets_dumper.erl
├── kvdb_export.erl
├── kvdb_leveldb.erl
├── kvdb_lib.erl
├── kvdb_log.erl
├── kvdb_meta.erl
├── kvdb_paired.erl
├── kvdb_proxy_sup.erl
├── kvdb_queue.erl
├── kvdb_riak.erl
├── kvdb_riak_mapred.erl
├── kvdb_riak_proxy.erl
├── kvdb_schema.erl
├── kvdb_schema_events.erl
├── kvdb_server.erl
├── kvdb_server_sup.erl
├── kvdb_sqlite3.erl
├── kvdb_sup.erl
├── kvdb_trans.erl
└── log.hrl
├── test.config
├── test
├── feuerlabs_eunit.hrl
├── kvdb_conf_tests.erl
├── kvdb_lib_proper.erl
├── kvdb_tests.erl
├── kvdb_trans_tests.erl
└── kvdb_usb_ets.erl
└── tetrapak
└── config.ini
/.gitignore:
--------------------------------------------------------------------------------
1 | ebin
2 | src/kvdb_cron_parse.erl
3 | src/kvdb_cron_scan.erl
4 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Mozilla Public License Version 2.0
2 | ==================================
3 |
4 | 1. Definitions
5 | --------------
6 |
7 | 1.1. "Contributor"
8 | means each individual or legal entity that creates, contributes to
9 | the creation of, or owns Covered Software.
10 |
11 | 1.2. "Contributor Version"
12 | means the combination of the Contributions of others (if any) used
13 | by a Contributor and that particular Contributor's Contribution.
14 |
15 | 1.3. "Contribution"
16 | means Covered Software of a particular Contributor.
17 |
18 | 1.4. "Covered Software"
19 | means Source Code Form to which the initial Contributor has attached
20 | the notice in Exhibit A, the Executable Form of such Source Code
21 | Form, and Modifications of such Source Code Form, in each case
22 | including portions thereof.
23 |
24 | 1.5. "Incompatible With Secondary Licenses"
25 | means
26 |
27 | (a) that the initial Contributor has attached the notice described
28 | in Exhibit B to the Covered Software; or
29 |
30 | (b) that the Covered Software was made available under the terms of
31 | version 1.1 or earlier of the License, but not also under the
32 | terms of a Secondary License.
33 |
34 | 1.6. "Executable Form"
35 | means any form of the work other than Source Code Form.
36 |
37 | 1.7. "Larger Work"
38 | means a work that combines Covered Software with other material, in
39 | a separate file or files, that is not Covered Software.
40 |
41 | 1.8. "License"
42 | means this document.
43 |
44 | 1.9. "Licensable"
45 | means having the right to grant, to the maximum extent possible,
46 | whether at the time of the initial grant or subsequently, any and
47 | all of the rights conveyed by this License.
48 |
49 | 1.10. "Modifications"
50 | means any of the following:
51 |
52 | (a) any file in Source Code Form that results from an addition to,
53 | deletion from, or modification of the contents of Covered
54 | Software; or
55 |
56 | (b) any new file in Source Code Form that contains any Covered
57 | Software.
58 |
59 | 1.11. "Patent Claims" of a Contributor
60 | means any patent claim(s), including without limitation, method,
61 | process, and apparatus claims, in any patent Licensable by such
62 | Contributor that would be infringed, but for the grant of the
63 | License, by the making, using, selling, offering for sale, having
64 | made, import, or transfer of either its Contributions or its
65 | Contributor Version.
66 |
67 | 1.12. "Secondary License"
68 | means either the GNU General Public License, Version 2.0, the GNU
69 | Lesser General Public License, Version 2.1, the GNU Affero General
70 | Public License, Version 3.0, or any later versions of those
71 | licenses.
72 |
73 | 1.13. "Source Code Form"
74 | means the form of the work preferred for making modifications.
75 |
76 | 1.14. "You" (or "Your")
77 | means an individual or a legal entity exercising rights under this
78 | License. For legal entities, "You" includes any entity that
79 | controls, is controlled by, or is under common control with You. For
80 | purposes of this definition, "control" means (a) the power, direct
81 | or indirect, to cause the direction or management of such entity,
82 | whether by contract or otherwise, or (b) ownership of more than
83 | fifty percent (50%) of the outstanding shares or beneficial
84 | ownership of such entity.
85 |
86 | 2. License Grants and Conditions
87 | --------------------------------
88 |
89 | 2.1. Grants
90 |
91 | Each Contributor hereby grants You a world-wide, royalty-free,
92 | non-exclusive license:
93 |
94 | (a) under intellectual property rights (other than patent or trademark)
95 | Licensable by such Contributor to use, reproduce, make available,
96 | modify, display, perform, distribute, and otherwise exploit its
97 | Contributions, either on an unmodified basis, with Modifications, or
98 | as part of a Larger Work; and
99 |
100 | (b) under Patent Claims of such Contributor to make, use, sell, offer
101 | for sale, have made, import, and otherwise transfer either its
102 | Contributions or its Contributor Version.
103 |
104 | 2.2. Effective Date
105 |
106 | The licenses granted in Section 2.1 with respect to any Contribution
107 | become effective for each Contribution on the date the Contributor first
108 | distributes such Contribution.
109 |
110 | 2.3. Limitations on Grant Scope
111 |
112 | The licenses granted in this Section 2 are the only rights granted under
113 | this License. No additional rights or licenses will be implied from the
114 | distribution or licensing of Covered Software under this License.
115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a
116 | Contributor:
117 |
118 | (a) for any code that a Contributor has removed from Covered Software;
119 | or
120 |
121 | (b) for infringements caused by: (i) Your and any other third party's
122 | modifications of Covered Software, or (ii) the combination of its
123 | Contributions with other software (except as part of its Contributor
124 | Version); or
125 |
126 | (c) under Patent Claims infringed by Covered Software in the absence of
127 | its Contributions.
128 |
129 | This License does not grant any rights in the trademarks, service marks,
130 | or logos of any Contributor (except as may be necessary to comply with
131 | the notice requirements in Section 3.4).
132 |
133 | 2.4. Subsequent Licenses
134 |
135 | No Contributor makes additional grants as a result of Your choice to
136 | distribute the Covered Software under a subsequent version of this
137 | License (see Section 10.2) or under the terms of a Secondary License (if
138 | permitted under the terms of Section 3.3).
139 |
140 | 2.5. Representation
141 |
142 | Each Contributor represents that the Contributor believes its
143 | Contributions are its original creation(s) or it has sufficient rights
144 | to grant the rights to its Contributions conveyed by this License.
145 |
146 | 2.6. Fair Use
147 |
148 | This License is not intended to limit any rights You have under
149 | applicable copyright doctrines of fair use, fair dealing, or other
150 | equivalents.
151 |
152 | 2.7. Conditions
153 |
154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
155 | in Section 2.1.
156 |
157 | 3. Responsibilities
158 | -------------------
159 |
160 | 3.1. Distribution of Source Form
161 |
162 | All distribution of Covered Software in Source Code Form, including any
163 | Modifications that You create or to which You contribute, must be under
164 | the terms of this License. You must inform recipients that the Source
165 | Code Form of the Covered Software is governed by the terms of this
166 | License, and how they can obtain a copy of this License. You may not
167 | attempt to alter or restrict the recipients' rights in the Source Code
168 | Form.
169 |
170 | 3.2. Distribution of Executable Form
171 |
172 | If You distribute Covered Software in Executable Form then:
173 |
174 | (a) such Covered Software must also be made available in Source Code
175 | Form, as described in Section 3.1, and You must inform recipients of
176 | the Executable Form how they can obtain a copy of such Source Code
177 | Form by reasonable means in a timely manner, at a charge no more
178 | than the cost of distribution to the recipient; and
179 |
180 | (b) You may distribute such Executable Form under the terms of this
181 | License, or sublicense it under different terms, provided that the
182 | license for the Executable Form does not attempt to limit or alter
183 | the recipients' rights in the Source Code Form under this License.
184 |
185 | 3.3. Distribution of a Larger Work
186 |
187 | You may create and distribute a Larger Work under terms of Your choice,
188 | provided that You also comply with the requirements of this License for
189 | the Covered Software. If the Larger Work is a combination of Covered
190 | Software with a work governed by one or more Secondary Licenses, and the
191 | Covered Software is not Incompatible With Secondary Licenses, this
192 | License permits You to additionally distribute such Covered Software
193 | under the terms of such Secondary License(s), so that the recipient of
194 | the Larger Work may, at their option, further distribute the Covered
195 | Software under the terms of either this License or such Secondary
196 | License(s).
197 |
198 | 3.4. Notices
199 |
200 | You may not remove or alter the substance of any license notices
201 | (including copyright notices, patent notices, disclaimers of warranty,
202 | or limitations of liability) contained within the Source Code Form of
203 | the Covered Software, except that You may alter any license notices to
204 | the extent required to remedy known factual inaccuracies.
205 |
206 | 3.5. Application of Additional Terms
207 |
208 | You may choose to offer, and to charge a fee for, warranty, support,
209 | indemnity or liability obligations to one or more recipients of Covered
210 | Software. However, You may do so only on Your own behalf, and not on
211 | behalf of any Contributor. You must make it absolutely clear that any
212 | such warranty, support, indemnity, or liability obligation is offered by
213 | You alone, and You hereby agree to indemnify every Contributor for any
214 | liability incurred by such Contributor as a result of warranty, support,
215 | indemnity or liability terms You offer. You may include additional
216 | disclaimers of warranty and limitations of liability specific to any
217 | jurisdiction.
218 |
219 | 4. Inability to Comply Due to Statute or Regulation
220 | ---------------------------------------------------
221 |
222 | If it is impossible for You to comply with any of the terms of this
223 | License with respect to some or all of the Covered Software due to
224 | statute, judicial order, or regulation then You must: (a) comply with
225 | the terms of this License to the maximum extent possible; and (b)
226 | describe the limitations and the code they affect. Such description must
227 | be placed in a text file included with all distributions of the Covered
228 | Software under this License. Except to the extent prohibited by statute
229 | or regulation, such description must be sufficiently detailed for a
230 | recipient of ordinary skill to be able to understand it.
231 |
232 | 5. Termination
233 | --------------
234 |
235 | 5.1. The rights granted under this License will terminate automatically
236 | if You fail to comply with any of its terms. However, if You become
237 | compliant, then the rights granted under this License from a particular
238 | Contributor are reinstated (a) provisionally, unless and until such
239 | Contributor explicitly and finally terminates Your grants, and (b) on an
240 | ongoing basis, if such Contributor fails to notify You of the
241 | non-compliance by some reasonable means prior to 60 days after You have
242 | come back into compliance. Moreover, Your grants from a particular
243 | Contributor are reinstated on an ongoing basis if such Contributor
244 | notifies You of the non-compliance by some reasonable means, this is the
245 | first time You have received notice of non-compliance with this License
246 | from such Contributor, and You become compliant prior to 30 days after
247 | Your receipt of the notice.
248 |
249 | 5.2. If You initiate litigation against any entity by asserting a patent
250 | infringement claim (excluding declaratory judgment actions,
251 | counter-claims, and cross-claims) alleging that a Contributor Version
252 | directly or indirectly infringes any patent, then the rights granted to
253 | You by any and all Contributors for the Covered Software under Section
254 | 2.1 of this License shall terminate.
255 |
256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all
257 | end user license agreements (excluding distributors and resellers) which
258 | have been validly granted by You or Your distributors under this License
259 | prior to termination shall survive termination.
260 |
261 | ************************************************************************
262 | * *
263 | * 6. Disclaimer of Warranty *
264 | * ------------------------- *
265 | * *
266 | * Covered Software is provided under this License on an "as is" *
267 | * basis, without warranty of any kind, either expressed, implied, or *
268 | * statutory, including, without limitation, warranties that the *
269 | * Covered Software is free of defects, merchantable, fit for a *
270 | * particular purpose or non-infringing. The entire risk as to the *
271 | * quality and performance of the Covered Software is with You. *
272 | * Should any Covered Software prove defective in any respect, You *
273 | * (not any Contributor) assume the cost of any necessary servicing, *
274 | * repair, or correction. This disclaimer of warranty constitutes an *
275 | * essential part of this License. No use of any Covered Software is *
276 | * authorized under this License except under this disclaimer. *
277 | * *
278 | ************************************************************************
279 |
280 | ************************************************************************
281 | * *
282 | * 7. Limitation of Liability *
283 | * -------------------------- *
284 | * *
285 | * Under no circumstances and under no legal theory, whether tort *
286 | * (including negligence), contract, or otherwise, shall any *
287 | * Contributor, or anyone who distributes Covered Software as *
288 | * permitted above, be liable to You for any direct, indirect, *
289 | * special, incidental, or consequential damages of any character *
290 | * including, without limitation, damages for lost profits, loss of *
291 | * goodwill, work stoppage, computer failure or malfunction, or any *
292 | * and all other commercial damages or losses, even if such party *
293 | * shall have been informed of the possibility of such damages. This *
294 | * limitation of liability shall not apply to liability for death or *
295 | * personal injury resulting from such party's negligence to the *
296 | * extent applicable law prohibits such limitation. Some *
297 | * jurisdictions do not allow the exclusion or limitation of *
298 | * incidental or consequential damages, so this exclusion and *
299 | * limitation may not apply to You. *
300 | * *
301 | ************************************************************************
302 |
303 | 8. Litigation
304 | -------------
305 |
306 | Any litigation relating to this License may be brought only in the
307 | courts of a jurisdiction where the defendant maintains its principal
308 | place of business and such litigation shall be governed by laws of that
309 | jurisdiction, without reference to its conflict-of-law provisions.
310 | Nothing in this Section shall prevent a party's ability to bring
311 | cross-claims or counter-claims.
312 |
313 | 9. Miscellaneous
314 | ----------------
315 |
316 | This License represents the complete agreement concerning the subject
317 | matter hereof. If any provision of this License is held to be
318 | unenforceable, such provision shall be reformed only to the extent
319 | necessary to make it enforceable. Any law or regulation which provides
320 | that the language of a contract shall be construed against the drafter
321 | shall not be used to construe this License against a Contributor.
322 |
323 | 10. Versions of the License
324 | ---------------------------
325 |
326 | 10.1. New Versions
327 |
328 | Mozilla Foundation is the license steward. Except as provided in Section
329 | 10.3, no one other than the license steward has the right to modify or
330 | publish new versions of this License. Each version will be given a
331 | distinguishing version number.
332 |
333 | 10.2. Effect of New Versions
334 |
335 | You may distribute the Covered Software under the terms of the version
336 | of the License under which You originally received the Covered Software,
337 | or under the terms of any subsequent version published by the license
338 | steward.
339 |
340 | 10.3. Modified Versions
341 |
342 | If you create software not governed by this License, and you want to
343 | create a new license for such software, you may create and use a
344 | modified version of this License if you rename the license and remove
345 | any references to the name of the license steward (except to note that
346 | such modified license differs from this License).
347 |
348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary
349 | Licenses
350 |
351 | If You choose to distribute Source Code Form that is Incompatible With
352 | Secondary Licenses under the terms of this version of the License, the
353 | notice described in Exhibit B of this License must be attached.
354 |
355 | Exhibit A - Source Code Form License Notice
356 | -------------------------------------------
357 |
358 | This Source Code Form is subject to the terms of the Mozilla Public
359 | License, v. 2.0. If a copy of the MPL was not distributed with this
360 | file, You can obtain one at http://mozilla.org/MPL/2.0/.
361 |
362 | If it is not possible or desirable to put the notice in a particular
363 | file, then You may include the notice in a location (such as a LICENSE
364 | file in a relevant directory) where a recipient would be likely to look
365 | for such a notice.
366 |
367 | You may add additional accurate notices of copyright ownership.
368 |
369 | Exhibit B - "Incompatible With Secondary Licenses" Notice
370 | ---------------------------------------------------------
371 |
372 | This Source Code Form is "Incompatible With Secondary Licenses", as
373 | defined by the Mozilla Public License, v. 2.0.
374 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: test doc compile trans_test basic_test
2 |
3 | all: compile
4 |
5 | compile:
6 | rebar compile
7 |
8 | doc:
9 | rebar doc
10 |
11 | test: compile
12 | rebar eunit skip_deps=true
13 |
14 | trans_test: compile
15 | rebar eunit skip_deps=true suite=kvdb_trans
16 |
17 | conf_test: compile
18 | rebar eunit skip_deps=true suite=kvdb_conf
19 |
20 | basic_test: compile
21 | rebar eunit skip_deps=true suite=kvdb
--------------------------------------------------------------------------------
/README:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Feuerlabs/kvdb/c8c71bf5422a8fa2e58c608c65629be7a58e27f3/README
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | # The kvdb application #
4 |
5 | __Authors:__ Ulf Wiger ([`ulf@feuerlabs.com`](mailto:ulf@feuerlabs.com)), Tony Rogvall ([`tony@rogvall.se`](mailto:tony@rogvall.se)).
6 |
7 | KVDB - Database Management System for Connected Device Management
8 |
9 | KVDB was initially designed to support the requirements of the Exosense
10 | system for managing Connected Devices, but is essentially a general-purpose
11 | DBMS. A requirement was that it should be useable both in an embedded device
12 | and on the device management server. To this end, KVDB supports a number of
13 | different storage backends, each with different characteristics.
14 |
15 | Features:
16 |
17 | * Ordered-set semantics
18 |
19 | * Transaction semantics
20 |
21 | * Persistent queues
22 |
23 | * CRON-like persistent timers
24 |
25 | * Extensible indexing
26 |
27 | * Storage backend plugins
28 |
29 | * Schema callback behavior
30 |
31 | See also the [Wiki](http://github.com/Feuerlabs/kvdb/wiki) for further description and examples.
32 |
33 |
34 | ## Modules ##
35 |
36 |
37 |
53 |
54 |
--------------------------------------------------------------------------------
/doc/edoc-info:
--------------------------------------------------------------------------------
1 | {application,kvdb}.
2 | {packages,[]}.
3 | {modules,[kvdb,kvdb_conf,kvdb_cron,kvdb_cron_parse,kvdb_cron_scan,
4 | kvdb_ets_dumper,kvdb_export,kvdb_paired,kvdb_proxy_sup,kvdb_queue,
5 | kvdb_riak,kvdb_riak_mapred,kvdb_riak_proxy,kvdb_schema,
6 | kvdb_schema_events]}.
7 |
--------------------------------------------------------------------------------
/doc/erlang.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Feuerlabs/kvdb/c8c71bf5422a8fa2e58c608c65629be7a58e27f3/doc/erlang.png
--------------------------------------------------------------------------------
/doc/kvdb_cron.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | # Module kvdb_cron #
4 | * [Description](#description)
5 | * [Function Index](#index)
6 | * [Function Details](#functions)
7 |
8 |
9 | Persistent timers for KVDB.
10 | __Behaviours:__ [`gen_server`](gen_server.md).
11 |
12 | __Authors:__ Ulf Wiger ([`ulf@feuerlabs.com`](mailto:ulf@feuerlabs.com)).
13 |
24 | ```
25 | ::= "{" "}"
26 | | "{" ";" "}"
27 | | "{" ";" ";" "}"
28 | ::= "in" | "at"
29 | ::= |
30 | | ","
31 | ::=
32 | |
33 | ::=
34 | | ","
35 | ::= |
36 | ::= "/" "/"
37 | | "-" "-"
38 | ::= ":" ":"
39 | | ":" ":"
40 | ::= "repeat"
41 | | "once"
42 | | "times"
43 | | "each"
44 | | "daily" | "weekly" | "monthly" | "yearly" | "annually"
45 | |
46 | ::=
47 | | "until"
48 | ::= "forever"
49 | |
50 | ::=
51 | | ","
52 | ::=
53 | ::= "last_day_of_month" | "ldom"
54 | ::= "ms"
55 | | "sec" | "secs" | "seconds"
56 | | "min" | "minutes"
57 | | "hr" | "hrs" | "hours"
58 | | "mo" | "months"
59 | | "y" | "yr" | "years"
60 | ```
61 |
62 |
80 | add(Db, Tab, When::timespec() , Key::any(), M::atom(), F::atom(), As::[any()]) -> ok | error()
81 | yecc_ret() ###
18 |
19 |
20 |
21 |
22 | yecc_ret() = {error, term()} | {ok, term()}
23 |
44 | format_error(Message::any()) -> [char() | list()]
45 |
57 | parse(Tokens::list()) -> yecc_ret()
58 |
70 | parse_and_scan(X1::{function() | {atom(), atom()}, [term()]} | {atom(), atom(), [term()]}) -> yecc_ret()
71 | tab() ###
18 |
19 |
20 |
21 |
22 | tab() = atom() | tid()
23 | tid() ###
32 |
33 |
34 | __abstract datatype__: `tid()`
35 |
36 |
54 | file2tab(Filename) -> {ok, Tab} | {error, Reason}
55 |
66 | file2tab(Filename, Options) -> {ok, Tab} | {error, Reason}
67 | Filename = file:name() Tab = tab() Options = [Option]Option = {verify, boolean()}Reason = term()
78 | tab2file(Tab, Filename) -> ok | {error, Reason}
79 |
90 | tab2file(Tab, Filename, Options) -> ok | {error, Reason}
91 | Tab = tab() Filename = file:name() Options = [Option]Option = {extended_info, [ExtInfo]}ExtInfo = md5sum | object_countReason = term()
2 | @author Tony Rogvall
3 |
4 | @doc KVDB - Database Management System for Connected Device Management
5 |
6 | KVDB was initially designed to support the requirements of the Exosense
7 | system for managing Connected Devices, but is essentially a general-purpose
8 | DBMS. A requirement was that it should be useable both in an embedded device
9 | and on the device management server. To this end, KVDB supports a number of
10 | different storage backends, each with different characteristics.
11 |
12 | Features:
13 |
14 |
15 | Ordered-set semantics
16 | Transaction semantics
17 | Persistent queues
18 | CRON-like persistent timers
19 | Extensible indexing
20 | Storage backend plugins
21 | Schema callback behavior
22 |
23 |
24 | See also the Wiki for further description and examples.
25 |
26 | @end
--------------------------------------------------------------------------------
/doc/stylesheet.css:
--------------------------------------------------------------------------------
1 | /* standard EDoc style sheet */
2 | body {
3 | font-family: Verdana, Arial, Helvetica, sans-serif;
4 | margin-left: .25in;
5 | margin-right: .2in;
6 | margin-top: 0.2in;
7 | margin-bottom: 0.2in;
8 | color: #000000;
9 | background-color: #ffffff;
10 | }
11 | h1,h2 {
12 | margin-left: -0.2in;
13 | }
14 | div.navbar {
15 | background-color: #add8e6;
16 | padding: 0.2em;
17 | }
18 | h2.indextitle {
19 | padding: 0.4em;
20 | background-color: #add8e6;
21 | }
22 | h3.function,h3.typedecl {
23 | background-color: #add8e6;
24 | padding-left: 1em;
25 | }
26 | div.spec {
27 | margin-left: 2em;
28 | background-color: #eeeeee;
29 | }
30 | a.module,a.package {
31 | text-decoration:none
32 | }
33 | a.module:hover,a.package:hover {
34 | background-color: #eeeeee;
35 | }
36 | ul.definitions {
37 | list-style-type: none;
38 | }
39 | ul.index {
40 | list-style-type: none;
41 | background-color: #eeeeee;
42 | }
43 |
44 | /*
45 | * Minor style tweaks
46 | */
47 | ul {
48 | list-style-type: square;
49 | }
50 | table {
51 | border-collapse: collapse;
52 | }
53 | td {
54 | padding: 3
55 | }
56 |
--------------------------------------------------------------------------------
/ebin/.gitignore:
--------------------------------------------------------------------------------
1 | *.beam
2 | *.app
3 |
--------------------------------------------------------------------------------
/include/kvdb.hrl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc kvdb definitions.
12 | %%% @end
13 |
14 | -type table() :: atom() | binary().
15 | -type int_table_name() :: binary().
16 | -type queue_name() :: any().
17 |
18 | -record(kvdb_ref, {name, tref, db, mod, schema = db}).
19 | -type db_ref() :: #kvdb_ref{}.
20 |
21 | -record(db, {ref, encoding = sext, metadata, log = false, st}).
22 | -record(table, {name,
23 | type = set,
24 | encoding = sext,
25 | columns,
26 | schema,
27 | nodes = [node()],
28 | index = []}).
29 |
30 | -record(event, {event, tab, info}).
31 |
32 | -record(dbst, {encoding,
33 | type}).
34 |
35 | -type db() :: #db{}.
36 |
37 | -define(META_TABLE, <<"kvdb__META">>).
38 |
39 | -type db_name() :: any().
40 | -type key() :: any().
41 | -type value() :: any().
42 | -type attr_name() :: atom().
43 | -type attr_value() :: any().
44 | -type attrs() :: [{atom(), any()}].
45 | -type options() :: [{atom(), any()}].
46 | -type basic_encoding() :: sext | raw | term.
47 | -type encoding() :: basic_encoding() |
48 | {basic_encoding(), basic_encoding()} |
49 | {basic_encoding(), basic_encoding(), basic_encoding()}.
50 |
51 | -type object() :: {key(), value()} | {key(), attrs(), value()}.
52 | -type increment() :: integer().
53 | -type status() :: active | inactive | blocking.
54 | -type cont() :: fun(() -> {[object()], cont()} | done).
55 | -type schema_category() :: property | tabrec | global.
56 |
57 | -record(commit, {add_tables = [],
58 | del_tables = [],
59 | write = [],
60 | delete = [],
61 | events = []}).
62 |
63 | -record(thr, {writes,
64 | bytes,
65 | time}).
66 |
67 | -record(q_key, {queue,
68 | ts,
69 | key}).
70 |
71 | -define(Q_HEAD_FLOOR, -1).
72 | -define(Q_HEAD_CEIL, []).
73 | -define(Q_HEAD_KEY, <<"_KVDB_Q_HEAD_">>).
74 | -define(IS_Q_HEAD_TS(TS), TS==?Q_HEAD_FLOOR; TS==?Q_HEAD_CEIL).
75 |
76 | -define(KVDB_CATCH(Expr, Args),
77 | try Expr
78 | catch
79 | throw:{kvdb_return, __R} ->
80 | __R;
81 | throw:{kvdb_throw, __E} ->
82 | %% error(__E, Args)
83 | erlang:error(__E, erlang:get_stacktrace())
84 | end).
85 |
86 | -define(KVDB_THROW(E), throw({kvdb_throw, E})).
87 | -define(KVDB_RETURN(R), throw({kvdb_return, R})).
88 |
89 | -define(KVDB_LOG_INSERT(Tab, Obj), {insert, Tab, Obj}).
90 | -define(KVDB_LOG_DELETE(Tab, Key), {delete, Tab, Key}).
91 | -define(KVDB_LOG_Q_INSERT(Tab, QKey, St, Obj),
92 | {q_insert, Tab, QKey, St, Obj}).
93 | -define(KVDB_LOG_Q_DELETE(Tab, QKey), {q_delete, Tab, QKey}).
94 | -define(KVDB_LOG_ADD_TABLE(Tab, TabR), {add_table, Tab, TabR}).
95 | -define(KVDB_LOG_DELETE_TABLE(Tab), {delete_table, Tab}).
96 | -define(KVDB_LOG_WRITE_META(Op), {write_meta, Op}).
97 | -define(KVDB_LOG_COMMIT(CommitRec), {commit, CommitRec}).
98 |
--------------------------------------------------------------------------------
/include/kvdb_conf.hrl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc kvdb definitions.
12 | %%% @end
13 |
14 | -type key_part() :: binary() | {binary(), integer()}.
15 | -type key() :: binary().
16 | -type attrs() :: [{atom(), any()}].
17 | -type value() :: any().
18 | -type conf_data() :: [conf_node() | conf_obj()].
19 |
20 | -record(conf_tree, {root = <<>> :: key(),
21 | tree = [] :: conf_data()
22 | }).
23 | -type conf_tree() :: #conf_tree{}.
24 |
25 | -type node_key() :: key() | integer().
26 | -type conf_obj() :: {node_key(), attrs(), value()}.
27 | -type conf_node() :: {node_key(), attrs(), value(), conf_tree()}
28 | | {node_key(), conf_tree()}.
29 | -type shift_op() :: up | down | top | bottom.
30 |
31 |
32 |
--------------------------------------------------------------------------------
/include/riak_kv_wm_raw.hrl:
--------------------------------------------------------------------------------
1 | %% This file is provided to you under the Apache License,
2 | %% Version 2.0 (the "License"); you may not use this file
3 | %% except in compliance with the License. You may obtain
4 | %% a copy of the License at
5 |
6 | %% http://www.apache.org/licenses/LICENSE-2.0
7 |
8 | %% Unless required by applicable law or agreed to in writing,
9 | %% software distributed under the License is distributed on an
10 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
11 | %% KIND, either express or implied. See the License for the
12 | %% specific language governing permissions and limitations
13 | %% under the License.
14 |
15 | %% Constants used by the raw_http resources
16 |
17 | %% Names of riak_object metadata fields
18 | -define(MD_CTYPE, <<"content-type">>).
19 | -define(MD_CHARSET, <<"charset">>).
20 | -define(MD_ENCODING, <<"content-encoding">>).
21 | -define(MD_VTAG, <<"X-Riak-VTag">>).
22 | -define(MD_LINKS, <<"Links">>).
23 | -define(MD_LASTMOD, <<"X-Riak-Last-Modified">>).
24 | -define(MD_USERMETA, <<"X-Riak-Meta">>).
25 | -define(MD_INDEX, <<"index">>).
26 | -define(MD_DELETED, <<"X-Riak-Deleted">>).
27 |
28 | %% Names of HTTP header fields
29 | -define(HEAD_CTYPE, "Content-Type").
30 | -define(HEAD_VCLOCK, "X-Riak-Vclock").
31 | -define(HEAD_LINK, "Link").
32 | -define(HEAD_ENCODING, "Content-Encoding").
33 | -define(HEAD_CLIENT, "X-Riak-ClientId").
34 | -define(HEAD_USERMETA_PREFIX, "x-riak-meta-").
35 | -define(HEAD_INDEX_PREFIX, "x-riak-index-").
36 | -define(HEAD_DELETED, "X-Riak-Deleted").
37 | -define(HEAD_TIMEOUT, "X-Riak-Timeout").
38 |
39 | %% Names of JSON fields in bucket properties
40 | -define(JSON_PROPS, <<"props">>).
41 | -define(JSON_BUCKETS, <<"buckets">>).
42 | -define(JSON_KEYS, <<"keys">>).
43 | -define(JSON_LINKFUN, <<"linkfun">>).
44 | -define(JSON_MOD, <<"mod">>).
45 | -define(JSON_FUN, <<"fun">>).
46 | -define(JSON_ARG, <<"arg">>).
47 | -define(JSON_CHASH, <<"chash_keyfun">>).
48 | -define(JSON_JSFUN, <<"jsfun">>).
49 | -define(JSON_JSANON, <<"jsanon">>).
50 | -define(JSON_JSBUCKET, <<"bucket">>).
51 | -define(JSON_JSKEY, <<"key">>).
52 | -define(JSON_ALLOW_MULT, <<"allow_mult">>).
53 | -define(JSON_EXTRACT, <<"search_extractor">>).
54 | -define(JSON_EXTRACT_LEGACY, <<"rs_extractfun">>).
55 |
56 | %% Names of HTTP query parameters
57 | -define(Q_PROPS, "props").
58 | -define(Q_BUCKETS, "buckets").
59 | -define(Q_KEYS, "keys").
60 | -define(Q_FALSE, "false").
61 | -define(Q_TRUE, "true").
62 | -define(Q_STREAM, "stream").
63 | -define(Q_VTAG, "vtag").
64 | -define(Q_RETURNBODY, "returnbody").
65 | -define(Q_2I_RETURNTERMS, "return_terms").
66 | -define(Q_2I_MAX_RESULTS, "max_results").
67 | -define(Q_2I_CONTINUATION, "continuation").
68 | -define(Q_RESULTS, "results").
69 | -define(Q_RETURNVALUE, "returnvalue").
70 |
--------------------------------------------------------------------------------
/priv/all_apps.config:
--------------------------------------------------------------------------------
1 | {"leveldb",eleveldb}.
2 | {"sqlite3",sqlite3}.
3 | {"sqlite3",resource}.
4 |
--------------------------------------------------------------------------------
/priv/backends.script:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | Backends = [{"leveldb",eleveldb},
3 | {"sqlite3",sqlite3}].
4 | case os:getenv("KVDB_BACKENDS") of
5 | Str when is_list(Str) ->
6 | Split = re:split(Str, "[, ]+", [{return,list}]),
7 | case [B || B <- Split, not lists:member(
8 | B, ["none", "ets",
9 | "leveldb", "sqlite3"])] of
10 | [] -> [];
11 | [_|_] = Unknown ->
12 | io:fwrite("Error - unknown backends: ~p~n", [Unknown]),
13 | error({unknown_backends, Unknown})
14 | end,
15 | io:fwrite("Backends selected: ~p~n", [Split]),
16 | [App || {K, App} = Pair <- Backends,
17 | lists:member(K, Split)];
18 | false ->
19 | [App || {_, App} <- Backends]
20 | end.
21 |
--------------------------------------------------------------------------------
/priv/excluded_apps.script:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | case os:getenv("KVDB_BACKENDS") of
3 | Str when is_list(Str) ->
4 | Split = re:split(Str, "[, ]+", [{return,list}]),
5 | {ok, Backends} =
6 | file:consult(filename:join(CWD, "all_apps.config")),
7 | case [B || B <- Split, not lists:member(
8 | B, ["none", "ets",
9 | "leveldb", "sqlite3"])] of
10 | [] -> [];
11 | [_|_] = Unknown ->
12 | io:fwrite("Error - unknown backends: ~p~n", [Unknown]),
13 | error({unknown_backends, Unknown})
14 | end,
15 | io:fwrite("Backends selected: ~p~n", [Split]),
16 | [App || {K, App} = Pair <- Backends,
17 | not lists:member(K, Split)];
18 | false ->
19 | []
20 | end.
21 |
--------------------------------------------------------------------------------
/rebar.config:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | {erl_first_files, ["src/kvdb.erl","src/kvdb_schema.erl"]}.
3 | {erl_opts, [debug_info, {parse_transform, lager_transform}]}.
4 | {xref_checks, [undefined_function_calls, undefined_functions,
5 | locals_not_used, deprecated_function_calls,
6 | deprecated_functions]}.
7 | {deps,
8 | [
9 | {sqlite3, ".*", {git, "git://github.com/Feuerlabs/erlang-sqlite3.git", "1.0.1.fl.3"}},
10 | {gproc, ".*", {git, "git://github.com/uwiger/gproc.git", "0.2.13.2"}},
11 | {sext, ".*", {git, "git://github.com/uwiger/sext.git", "0.6"}},
12 | {resource, ".*", {git, "git://github.com/Feuerlabs/resource.git", "0.2"}},
13 | {eleveldb, ".*", {git, "git://github.com/Feuerlabs/eleveldb.git", "2.0.0.fl.1"}},
14 | {edown, ".*", {git, "git://github.com/uwiger/edown.git", "HEAD"}},
15 | {lager, ".*", {git, "git://github.com/basho/lager.git", "3.0.1"}},
16 | {parse_trans, ".*", {git, "git://github.com/uwiger/parse_trans.git", "HEAD"}}
17 | ]}.
18 | {edoc_opts, [{doclet, edown_doclet},
19 | {top_level_readme,
20 | {"./README.md",
21 | "http://github.com/Feuerlabs/kvdb"}}]}.
22 |
--------------------------------------------------------------------------------
/rebar.config.script:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | %%
3 | %% If the environment variable X_COMP is set, it is an indication that self
4 | %% is being compiled as a dependency under exodev. Since exodev lists all
5 | %% dependency in its root rebar.config file in order to have a flat view of
6 | %% the dependency tree, we should nil out our local
7 | %% deps list provided by our local rebar.config
8 | %%
9 | CFG1 = case os:getenv("KVDB_BACKENDS") of
10 | Str when is_list(Str) ->
11 | Priv = filename:join(filename:dirname(SCRIPT), "priv"),
12 | {ok,Exclude} = file:script(
13 | filename:join(Priv, "excluded_apps.script"),
14 | [{'CWD', Priv}]),
15 | Deps0 = proplists:get_value(deps, CONFIG, []),
16 | Deps1 = [D || D <- Deps0,
17 | not lists:member(element(1,D), Exclude)],
18 | lists:keystore(deps, 1, CONFIG, {deps, Deps1});
19 | false ->
20 | io:fwrite("Using all backends~n", []),
21 | CONFIG
22 | end.
23 |
24 | %% The following is a temporary fix for Feuerlabs' device SW build system
25 | case os:getenv("EXODEV_COMP") of
26 | E when E==false; E==[] ->
27 | case os:getenv("REBAR_DEPS") of
28 | D when D==false; D==[] -> CFG1;
29 | Dir -> lists:keystore(deps_dir, 1, CFG1, {deps_dir, Dir})
30 | end;
31 | _ ->
32 | case lists:keytake(deps, 1, CFG1) of
33 | { value, _, Remainder } -> Remainder;
34 | _ -> CFG1
35 | end
36 | end.
37 |
--------------------------------------------------------------------------------
/src/kvdb.app.src:
--------------------------------------------------------------------------------
1 | %%% -*- erlang -*-
2 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
3 | %%%
4 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
5 | %%%
6 | %%% This Source Code Form is subject to the terms of the Mozilla Public
7 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
8 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 | %%%
10 | %%%---- END COPYRIGHT ---------------------------------------------------------
11 | %%% @author Ulf Wiger
12 | {application, kvdb,
13 | [
14 | {description, "key value database API"},
15 | {vsn, git},
16 | {registered, []},
17 | {applications, [
18 | kernel,
19 | stdlib,
20 | gproc
21 | ]},
22 | {mod, {kvdb_app, []}},
23 | {start_phases, [{open_dbs, []}]},
24 | {env, [
25 | {backend, ets} % default
26 | %% {dir, DbDirectory}
27 | ]}
28 | ]}.
29 |
--------------------------------------------------------------------------------
/src/kvdb_app.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | -module(kvdb_app).
13 |
14 | -behaviour(application).
15 |
16 | %% Application callbacks
17 | -export([start/2, stop/1,
18 | start_phase/3]).
19 |
20 | %% ===================================================================
21 | %% Application callbacks
22 | %% ===================================================================
23 |
24 | start(_StartType, _StartArgs) ->
25 | io:fwrite("starting kvdb~n", []),
26 | %% dbg:tracer(),
27 | %% dbg:tpl(kvdb,x),
28 | %% dbg:tpl(kvdb_sup,x),
29 | %% dbg:tp(kvdb_sqlite,x),
30 | %% dbg:tp(kvdb_leveldb,x),
31 | %% dbg:p(all,[c]),
32 | kvdb_sup:start_link().
33 |
34 | start_phase(open_dbs, _, []) ->
35 | Dbs = get_databases(),
36 | io:fwrite("KVDB Dbs = ~p~n", [Dbs]),
37 | [{ok,_} = kvdb:open(Name, Opts) || {Name, Opts} <- Dbs],
38 | ok.
39 |
40 | stop(_State) ->
41 | ok.
42 |
43 |
44 |
45 | get_databases() ->
46 | %% If 'setup' is available, query for other databases
47 | OtherDBs =
48 | case lists:keymember(setup, 1, application:loaded_applications()) of
49 | true ->
50 | [DB || {_, DB} <- setup:find_env_vars(kvdb_databases)];
51 | false ->
52 | []
53 | end,
54 | lists:flatten(
55 | case application:get_env(databases) of
56 | {ok, DBs} when is_list(DBs) ->
57 | [DBs | OtherDBs];
58 | _ ->
59 | OtherDBs
60 | end).
61 |
--------------------------------------------------------------------------------
/src/kvdb_cron_parse.yrl:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | %%---- BEGIN COPYRIGHT -------------------------------------------------------
3 | %%
4 | %% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
5 | %%
6 | %% This Source Code Form is subject to the terms of the Mozilla Public
7 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
8 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 | %%
10 | %%---- END COPYRIGHT ---------------------------------------------------------
11 |
12 | Nonterminals
13 | expr time_spec in_spec at_spec in_expr at_expr each_spec each_expr
14 | repeat_spec until_spec time_unit.
15 |
16 | Terminals
17 | integer 'in' 'at' 'each' fixnum '{' '}' ';' ':' ',' '/' '-'
18 | ms seconds hours minutes days months years times repeat until once forever
19 | daily weekly monthly yearly last_day_of_month.
20 |
21 | Rootsymbol expr.
22 |
23 | expr -> integer : {{in, ?line('$1'), [{'$1',{ms,?line('$1')}}]},
24 | {nil,?line('$1')}, {nil, ?line('$1')}}.
25 | expr -> 'in' in_spec : {{in, ?line('$1'), '$2'},
26 | {nil,?line('$2')}, {nil, ?line('$2')}}.
27 | expr -> 'at' at_spec : {{at, ?line('$1'), '$2'},
28 | {nil,?line('$2')}, {nil, ?line('$2')}}.
29 | expr -> '{' time_spec '}' : {'$2', {nil,?line('$2')}, {nil, ?line('$2')}}.
30 | expr -> '{' time_spec ';' repeat_spec '}' : {'$2', '$4', {nil, ?line('$4')}}.
31 | expr -> '{' time_spec ';' repeat_spec ';' until_spec '}' : {'$2', '$4', '$6'}.
32 |
33 | time_spec -> 'in' in_spec : {in, ?line('$1'), '$2'}.
34 | time_spec -> 'at' at_spec : {at, ?line('$1'), '$2'}.
35 |
36 | repeat_spec -> repeat : {repeat, ?line('$1')}.
37 | repeat_spec -> once : {times, ?line('$1'), {integer, ?line('$1'), 1}}.
38 | repeat_spec -> integer 'times' : {times, ?line('$1'), '$1'}.
39 | repeat_spec -> 'each' each_spec : {each, ?line('$1'), '$2'}.
40 | repeat_spec -> daily : '$1'.
41 | repeat_spec -> weekly : '$1'.
42 | repeat_spec -> monthly : '$1'.
43 | repeat_spec -> yearly : '$1'.
44 | repeat_spec -> last_day_of_month : '$1'.
45 |
46 | until_spec -> forever : {nil, ?line('$1')}.
47 | until_spec -> until forever : {nil, ?line('$1')}.
48 | until_spec -> until at_spec : {at, ?line('$1'), '$2'}.
49 | until_spec -> at_spec : {at, ?line('$1'), '$1'}.
50 |
51 | in_spec -> integer : [{'$1', {ms, ?line('$1')}}].
52 | in_spec -> in_expr : ['$1'].
53 | in_spec -> in_expr ',' in_spec : ['$1' | '$3'].
54 |
55 | in_expr -> integer time_unit : {'$1', '$2'}.
56 | in_expr -> last_day_of_month : '$1'.
57 |
58 | at_spec -> at_expr : ['$1'].
59 | at_spec -> at_expr ',' at_spec : ['$1' | '$3'].
60 |
61 | at_expr -> integer '/' integer '/' integer :
62 | {date, ?line('$1'), {y('$5'),m('$3'),d('$1')}}.
63 | at_expr -> integer '-' integer '-' integer :
64 | {date, ?line('$1'), {y('$1'),m('$3'),d('$5')}}.
65 | at_expr -> integer ':' integer ':' integer :
66 | {time, ?line('$1'), {{h('$1'),mi('$3'),s('$5')}, 0}}.
67 | at_expr -> integer ':' integer ':' fixnum :
68 | {time, ?line('$1'),
69 | {{h('$1'),mi('$3'),s(int('$5'))}, frac('$5')}}.
70 |
71 | each_spec -> each_expr : ['$1'].
72 | each_spec -> each_expr ',' each_spec : ['$1' | '$3'].
73 |
74 | each_expr -> integer time_unit : {'$1', '$2'}.
75 |
76 |
77 | time_unit -> ms : '$1'.
78 | time_unit -> seconds : '$1'.
79 | time_unit -> hours : '$1'.
80 | time_unit -> minutes : '$1'.
81 | time_unit -> days : '$1'.
82 | time_unit -> months : '$1'.
83 | time_unit -> years : '$1'.
84 |
85 |
86 |
87 |
88 |
89 | Erlang code.
90 |
91 | %% @hidden
92 |
93 | -define(line(T), line_of(T)).
94 |
95 | line_of([T|_]) ->
96 | line_of(T);
97 | line_of(T) when is_tuple(T) ->
98 | element(2, T).
99 |
100 |
101 | int({fixnum, L, {I,_}}) ->
102 | {integer, L, I}.
103 |
104 | frac({fixnum, L, {_,F}}) ->
105 | {integer, L, F}.
106 |
107 | y({integer,_,I} = T) when I >= 1000, I =< 9999 ->
108 | T;
109 | y({integer,L,I}) when I >= 0, I =< 99 ->
110 | {Y,_,_} = date(),
111 | {integer, L, (Y div 100) * 100 + I};
112 | y(T) ->
113 | {error, ?line(T), {invalid_year, T}}.
114 |
115 |
116 |
117 | m({integer,_,I} = T) when I >= 0, I =< 12 ->
118 | T;
119 | m(T) ->
120 | {error, ?line(T), {invalid_month, T}}.
121 |
122 | d({integer, _, I} = T) when I >= 0, I =< 31 ->
123 | T;
124 | d(T) ->
125 | {error, ?line(T), {invalid_day, T}}.
126 |
127 | h({integer, _, I} = T) when I >= 0, I =< 23 ->
128 | T;
129 | h(T) ->
130 | {error, ?line(T), {invalid_hour, T}}.
131 |
132 | mi({integer, _, I} = T) when I >= 0, I =< 59 ->
133 | T;
134 | mi(T) ->
135 | {error, ?line(T), {invalid_minute, T}}.
136 |
137 | s({integer, _, I} = T) when I >= 0, I =< 59 ->
138 | T;
139 | s(T) ->
140 | {error, ?line(T), {invalid_second, T}}.
141 |
--------------------------------------------------------------------------------
/src/kvdb_cron_scan.xrl:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | %%---- BEGIN COPYRIGHT -------------------------------------------------------
3 | %%
4 | %% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
5 | %%
6 | %% This Source Code Form is subject to the terms of the Mozilla Public
7 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
8 | %% file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 | %%
10 | %%---- END COPYRIGHT ---------------------------------------------------------
11 | Definitions.
12 |
13 | D = [0-9]
14 | WS = ([\000-\s]|%.*)
15 |
16 | Rules.
17 |
18 | { : {token, {'{', TokenLine}}.
19 | } : {token, {'}', TokenLine}}.
20 | / : {token, {'/', TokenLine}}.
21 | , : {token, {',', TokenLine}}.
22 | : : {token, {':', TokenLine}}.
23 | ; : {token, {';', TokenLine}}.
24 | - : {token, {'-', TokenLine}}.
25 |
26 | {D}+ :
27 | {token, {integer, TokenLine, list_to_integer(TokenChars)}}.
28 |
29 | {D}+\.{D}+ :
30 | {token, {fixnum, TokenLine, mk_fixnum(TokenChars)}}.
31 |
32 | in : {token, {in, TokenLine}}.
33 | at : {token, {at, TokenLine}}.
34 | each : {token, {each, TokenLine}}.
35 | repeat : {token, {repeat, TokenLine}}.
36 | once : {token, {once, TokenLine}}.
37 | forever : {token, {forever, TokenLine}}.
38 | until : {token, {until, TokenLine}}.
39 | times : {token, {times, TokenLine}}.
40 | ms : {token, {ms, TokenLine}}.
41 | sec|secs|seconds : {token, {seconds, TokenLine}}.
42 | min|minutes : {token, {minutes, TokenLine}}.
43 | hr|hrs|hours : {token, {hours, TokenLine}}.
44 | mo|months : {token, {months, TokenLine}}.
45 | y|yr|years : {token, {years, TokenLine}}.
46 | daily : {token, {daily, TokenLine}}.
47 | monthly : {token, {monthly, TokenLine}}.
48 | weekly : {token, {weekly, TokenLine}}.
49 | annually : {token, {yearly, TokenLine}}.
50 | yearly : {token, {yearly, TokenLine}}.
51 | last_day_of_month : {token, {last_day_of_month, TokenLine}}.
52 | ldom : {token, {last_day_of_month, TokenLine}}.
53 |
54 | {WS}+ : skip_token.
55 |
56 | Erlang code.
57 |
58 | %%% @hidden
59 |
60 | mk_fixnum(Cs) ->
61 | [I,F] = re:split(Cs, <<"\\.">>, [{return, list}]),
62 | {list_to_integer(I), to_ms(F)}.
63 |
64 | to_ms([A,B,C|_]) -> list_to_integer("1" ++ [A,B,C]) - 1000;
65 | to_ms([_|_] = S) -> to_ms(S, 100).
66 |
67 | to_ms("0" ++ S, M) -> to_ms(S, M div 10);
68 | to_ms([], _) -> 0;
69 | to_ms(S, M) -> list_to_integer(S) * M.
70 |
--------------------------------------------------------------------------------
/src/kvdb_cron_sup.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%%
13 | -module(kvdb_cron_sup).
14 |
15 | -behaviour(supervisor).
16 |
17 | %% API
18 | -export([start_link/0]).
19 | -export([start_child/2]).
20 | %% childspec/1]).
21 |
22 | %% Supervisor callbacks
23 | -export([init/1]).
24 |
25 | %% Helper macro for declaring children of supervisor
26 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
27 |
28 | %% ===================================================================
29 | %% API functions
30 | %% ===================================================================
31 |
32 | start_link() ->
33 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
34 |
35 | %% ===================================================================
36 | %% Supervisor callbacks
37 | %% ===================================================================
38 |
39 | init([]) ->
40 | %% Children = childspecs(DBs = get_databases()),
41 | %% io:fwrite("DBs = ~p~n", [DBs]),
42 | {ok, { {simple_one_for_one, 5, 10},
43 | [{id, {kvdb_cron, start_link, []},
44 | transient, 5000, worker, [kvdb_cron]}] }}.
45 |
46 |
47 | start_child(Name, Options) ->
48 | supervisor:start_child(?MODULE, [Name, Options]).
49 |
50 |
--------------------------------------------------------------------------------
/src/kvdb_db_sup.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%%
13 | -module(kvdb_db_sup).
14 |
15 | -behaviour(supervisor).
16 |
17 | %% API
18 | -export([start_link/2,
19 | stop_child/1]).
20 |
21 | %% Supervisor callbacks
22 | -export([init/1]).
23 |
24 | %% Helper macro for declaring children of supervisor
25 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
26 |
27 | %% ===================================================================
28 | %% API functions
29 | %% ===================================================================
30 |
31 | start_link(Name, Options) ->
32 | supervisor:start_link(?MODULE, {Name, Options}).
33 |
34 | %% ===================================================================
35 | %% Supervisor callbacks
36 | %% ===================================================================
37 |
38 | init({Name, Options}) ->
39 | gproc:reg({n,l,{?MODULE, Name}}),
40 | {ok, { {rest_for_one, 5, 10},
41 | [{proxy, {kvdb_proxy_sup, start_link, [Name, Options]},
42 | permanent, infinity, supervisor, [kvdb_proxy_sup]},
43 | {db, {kvdb_server, start_link, [Name, Options]},
44 | permanent, 5000, worker, [kvdb_server]},
45 | {cron, {kvdb_cron, start_link, [Name, Options]},
46 | permanent, 5000, worker, [kvdb_cron]}
47 | ]}}.
48 |
49 | stop_child(Name) ->
50 | case gproc:where({n,l,{?MODULE,Name}}) of
51 | Pid when is_pid(Pid) ->
52 | supervisor:terminate_child(kvdb_sup, Pid);
53 | _ ->
54 | {error, unknown_db}
55 | end.
56 |
--------------------------------------------------------------------------------
/src/kvdb_diff.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @author Tony Rogvall
12 | %%% @hidden
13 | %%% @doc
14 | %%% Simple diff algorithm
15 | %%% @end
16 |
17 | -module(kvdb_diff).
18 |
19 | -compile(export_all).
20 |
21 | %%
22 | %% diff to parts of a config tree like:
23 | %% A = <<"/devices/123/config/candidate">>
24 | %% B = <<"/devices/123/config/running">>
25 | %% paths(A,B)
26 | %% return a list of items:
27 | %% [ {add,{Key,As,Value,Tree}} | {delete,Key}, {update,Key,Value} ]
28 | %% that when applied to A make that subtree equal to B's subtree
29 | %%
30 | tree(A, B) ->
31 | tree_obj(A, prefix_read(A,A), B, prefix_read(B,B), []).
32 |
33 | tree_obj(Ax, {A,AObj={Ak,_As,Av}}, Bx, {B,BObj={Bk,Bs,Bv}}, Acc) ->
34 | if A < B ->
35 | tree_obj(Ax, prefix_next(Ax,Ak), Bx, {B,BObj},
36 | [{delete,A}|Acc]);
37 | A > B ->
38 | tree_obj(Ax, {A,AObj}, Bx, prefix_next(Bx,Bk),
39 | [{add,{B,Bs,Bv}}|Acc]);
40 | A =:= B ->
41 | Acc1 = if Av =:= Bv -> Acc;
42 | true -> [{update,A,Bv}|Acc]
43 | end,
44 | tree_obj(Ax, prefix_next(Ax,Ak), Bx, prefix_next(Bx,Bk),Acc1)
45 | end;
46 | tree_obj(_Ax, done, _Bx, done, Acc) ->
47 | Acc;
48 | tree_obj(Ax, done, Bx, {B,{Bk,Bs,Bv}}, Acc) ->
49 | tree_obj(Ax, done, Bx, prefix_next(Bx,Bk),[{add,{B,Bs,Bv}}|Acc]);
50 | tree_obj(Ax, {A,{Ak,_,_}}, Bx, done, Acc) ->
51 | tree_obj(Ax, prefix_next(Ax,Ak), Bx, done, [{delete,A}|Acc]).
52 |
53 | prefix_read(Prefix, Key) ->
54 | case kvdb_conf:read(Key) of
55 | {ok, Obj} ->
56 | {remove_prefix(Prefix,Key),Obj};
57 | {error,_} ->
58 | prefix_next(Prefix, Key)
59 | end.
60 |
61 | prefix_next(Prefix, Key) ->
62 | case kvdb_conf:next(Key) of
63 | {ok, Obj={Key1,_As,_Data}} ->
64 | case prefix(Prefix, Key1) of
65 | true ->
66 | {remove_prefix(Prefix,Key1), Obj};
67 | false ->
68 | done
69 | end;
70 | done ->
71 | done
72 | end.
73 |
74 | %% Prefix = <<"">> | << "abc" >> | << "abc*edfg" >>
75 | remove_prefix(Prefix, Key) ->
76 | PrefixSz = byte_size(Prefix),
77 | case Key of
78 | <> ->
79 | Key1;
80 | <> ->
81 | Key1
82 | end.
83 |
84 |
85 | prefix(Key1, Key2) ->
86 | Key1Sz = byte_size(Key1),
87 | case Key2 of
88 | <> ->
89 | true;
90 | _ ->
91 | false
92 | end.
93 |
94 | trim_prefix(A, At) ->
95 | Parts = binary:split(A, <<"*">>, [global]),
96 | trim_parts(Parts, At).
97 |
98 | trim_parts([K|Ks], [{K,[],<<>>,T}]) ->
99 | trim_parts(Ks, T);
100 | trim_parts([], T) ->
101 | T.
102 |
--------------------------------------------------------------------------------
/src/kvdb_ets_dumper.erl:
--------------------------------------------------------------------------------
1 | %%
2 | %% %CopyrightBegin%
3 | %%
4 | %% Copyright Ericsson AB 1996-2013. All Rights Reserved.
5 | %%
6 | %% The contents of this file are subject to the Erlang Public License,
7 | %% Version 1.1, (the "License"); you may not use this file except in
8 | %% compliance with the License. You should have received a copy of the
9 | %% Erlang Public License along with this software. If not, it can be
10 | %% retrieved online at http://www.erlang.org/.
11 | %%
12 | %% Software distributed under the License is distributed on an "AS IS"
13 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
14 | %% the License for the specific language governing rights and limitations
15 | %% under the License.
16 | %%
17 | %% %CopyrightEnd%
18 | %%
19 | %% ===================================================================
20 | %% This code is a modified version of ets:tab2file/[2,3],
21 | %% adding a 'sync' option. This allows the dumped file to be flushed
22 | %% directly to disk. /Ulf Wiger
23 | %% ===================================================================
24 | -module(kvdb_ets_dumper).
25 |
26 | -export([tab2file/2,
27 | tab2file/3]).
28 | -export([file2tab/1,
29 | file2tab/2]).
30 |
31 | -type tab() :: atom() | tid().
32 | %% a similar definition is also in erl_types
33 | -opaque tid() :: integer().
34 |
35 | -define(MAJOR_F2T_VERSION,1).
36 | -define(MINOR_F2T_VERSION,0).
37 |
38 | -record(filetab_options,
39 | {
40 | object_count = false :: boolean(),
41 | md5sum = false :: boolean(),
42 | sync = false :: boolean() % UW addition
43 | }).
44 |
45 |
46 | %% file2tab/[1,2] are just here for symmetry; they pass through to ets. /UW
47 | -spec file2tab(Filename) -> {'ok', Tab} | {'error', Reason} when
48 | Filename :: file:name(),
49 | Tab :: tab(),
50 | Reason :: term().
51 |
52 | file2tab(File) ->
53 | ets:file2tab(File).
54 |
55 | -spec file2tab(Filename, Options) -> {'ok', Tab} | {'error', Reason} when
56 | Filename :: file:name(),
57 | Tab :: tab(),
58 | Options :: [Option],
59 | Option :: {'verify', boolean()},
60 | Reason :: term().
61 |
62 | file2tab(File, Options) ->
63 | ets:file2tab(File, Options).
64 |
65 |
66 | -spec tab2file(Tab, Filename) -> 'ok' | {'error', Reason} when
67 | Tab :: tab(),
68 | Filename :: file:name(),
69 | Reason :: term().
70 |
71 | tab2file(Tab, File) ->
72 | tab2file(Tab, File, []).
73 |
74 |
75 |
76 | -spec tab2file(Tab, Filename, Options) -> 'ok' | {'error', Reason} when
77 | Tab :: tab(),
78 | Filename :: file:name(),
79 | Options :: [Option],
80 | Option :: {'extended_info', [ExtInfo]},
81 | ExtInfo :: 'md5sum' | 'object_count',
82 | Reason :: term().
83 |
84 | tab2file(Tab, File, Options) ->
85 | try
86 | {ok, FtOptions} = parse_ft_options(Options),
87 | file:delete(File),
88 | case file:read_file_info(File) of
89 | {error, enoent} -> ok;
90 | _ -> throw(eaccess)
91 | end,
92 | Name = make_ref(),
93 | case disk_log:open([{name, Name}, {file, File}, {repair, truncate }]) of
94 | {ok, Name} -> ok;
95 | {repaired, Name} -> ok;
96 | {error, Reason} ->
97 | throw(Reason)
98 | end,
99 | try
100 | Info0 = case ets:info(Tab) of
101 | undefined ->
102 | %% erlang:error(badarg, [Tab, File, Options]);
103 | throw(badtab);
104 | I ->
105 | I
106 | end,
107 | Info = [list_to_tuple(Info0 ++
108 | [{major_version,?MAJOR_F2T_VERSION},
109 | {minor_version,?MINOR_F2T_VERSION},
110 | {extended_info,
111 | ft_options_to_list(FtOptions)}])],
112 | {LogFun, InitState} =
113 | case FtOptions#filetab_options.md5sum of
114 | true ->
115 | {fun(Oldstate,Termlist) ->
116 | {NewState,BinList} =
117 | md5terms(Oldstate,Termlist),
118 | disk_log:blog_terms(Name,BinList),
119 | NewState
120 | end,
121 | erlang:md5_init()};
122 | false ->
123 | {fun(_,Termlist) ->
124 | disk_log:log_terms(Name,Termlist),
125 | true
126 | end,
127 | true}
128 | end,
129 | ets:safe_fixtable(Tab,true),
130 | {NewState1,Num} = try
131 | NewState = LogFun(InitState,Info),
132 | dump_file(
133 | ets:select(Tab,[{'_',[],['$_']}],100),
134 | LogFun, NewState, 0)
135 | after
136 | (catch ets:safe_fixtable(Tab,false))
137 | end,
138 | EndInfo =
139 | case FtOptions#filetab_options.object_count of
140 | true ->
141 | [{count,Num}];
142 | false ->
143 | []
144 | end ++
145 | case FtOptions#filetab_options.md5sum of
146 | true ->
147 | [{md5,erlang:md5_final(NewState1)}];
148 | false ->
149 | []
150 | end,
151 | case EndInfo of
152 | [] ->
153 | ok;
154 | List ->
155 | LogFun(NewState1,[['$end_of_table',List]])
156 | end,
157 | %% UW addition:
158 | case FtOptions#filetab_options.sync of
159 | true ->
160 | disk_log:sync(Name);
161 | false ->
162 | ok
163 | end,
164 | disk_log:close(Name)
165 | catch
166 | throw:TReason ->
167 | disk_log:close(Name),
168 | file:delete(File),
169 | throw(TReason);
170 | exit:ExReason ->
171 | disk_log:close(Name),
172 | file:delete(File),
173 | exit(ExReason);
174 | error:ErReason ->
175 | disk_log:close(Name),
176 | file:delete(File),
177 | erlang:raise(error,ErReason,erlang:get_stacktrace())
178 | end
179 | catch
180 | throw:TReason2 ->
181 | {error,TReason2};
182 | exit:ExReason2 ->
183 | {error,ExReason2}
184 | end.
185 |
186 | dump_file('$end_of_table', _LogFun, State, Num) ->
187 | {State,Num};
188 | dump_file({Terms, Context}, LogFun, State, Num) ->
189 | Count = length(Terms),
190 | NewState = LogFun(State, Terms),
191 | dump_file(ets:select(Context), LogFun, NewState, Num + Count).
192 |
193 |
194 | ft_options_to_list(#filetab_options{md5sum = MD5, object_count = PS}) ->
195 | case PS of
196 | true ->
197 | [object_count];
198 | _ ->
199 | []
200 | end ++
201 | case MD5 of
202 | true ->
203 | [md5sum];
204 | _ ->
205 | []
206 | end.
207 |
208 | md5terms(State, []) ->
209 | {State, []};
210 | md5terms(State, [H|T]) ->
211 | B = term_to_binary(H),
212 | NewState = erlang:md5_update(State, B),
213 | {FinState, TL} = md5terms(NewState, T),
214 | {FinState, [B|TL]}.
215 |
216 | parse_ft_options(Options) when is_list(Options) ->
217 | {Opt,Rest} = case (catch lists:keytake(extended_info,1,Options)) of
218 | false ->
219 | {[],Options};
220 | {value,{extended_info,L},R} when is_list(L) ->
221 | {L,R}
222 | end,
223 | parse_ft_info_options(#filetab_options{}, Opt ++ Rest);
224 | parse_ft_options(Malformed) ->
225 | throw({malformed_option, Malformed}).
226 |
227 | parse_ft_info_options(FtOpt,[]) ->
228 | {ok,FtOpt};
229 | parse_ft_info_options(FtOpt,[object_count | T]) ->
230 | parse_ft_info_options(FtOpt#filetab_options{object_count = true}, T);
231 | parse_ft_info_options(FtOpt,[md5sum | T]) ->
232 | parse_ft_info_options(FtOpt#filetab_options{md5sum = true}, T);
233 | parse_ft_info_options(FtOpt, [sync | T]) ->
234 | parse_ft_info_options(FtOpt#filetab_options{sync = true}, T);
235 | parse_ft_info_options(_,[Unexpected | _]) ->
236 | throw({unknown_option,[{extended_info,[Unexpected]}]});
237 | parse_ft_info_options(_,Malformed) ->
238 | throw({malformed_option,Malformed}).
239 |
--------------------------------------------------------------------------------
/src/kvdb_export.erl:
--------------------------------------------------------------------------------
1 | -module(kvdb_export).
2 |
3 | -compile(export_all).
4 | -include("kvdb.hrl").
5 |
6 | export(#kvdb_ref{mod = M, db = Db} = Ref, File) ->
7 | Format = format(File),
8 | Tabs = M:list_tables(Db),
9 | {Out, Close} = open_log(File, Format),
10 | try dump(Tabs, Ref, Out)
11 | after
12 | Close()
13 | end.
14 |
15 | import(#kvdb_ref{mod = M, db = Db}, File) ->
16 | import(M, Db, File, format(File)).
17 |
18 | import(M, Db, File) ->
19 | import(M, Db, File, format(File)).
20 |
21 | import(M, Db, File, binary) ->
22 | case disk_log:open([{name, ?MODULE},
23 | {file, File},
24 | {format, internal}]) of
25 | {ok, Log} ->
26 | try import_bin_log(Log, M, Db)
27 | after
28 | disk_log:close(Log)
29 | end;
30 | Error ->
31 | Error
32 | end.
33 |
34 | format(F) ->
35 | case filename:extension(F) of
36 | ".KBUPB" -> binary;
37 | _ -> error(unknown_extension)
38 | end.
39 |
40 | dump([T|Tabs], Ref, Out) ->
41 | #table{type = Type} = TabR = kvdb:info(Ref, {T, tabrec}),
42 | Out({table, T, lists:keydelete(name, 1, kvdb_lib:tabrec_to_list(TabR))}),
43 | with_tab(T, Type, Ref, Out),
44 | dump(Tabs, Ref, Out);
45 | dump([], _, _) ->
46 | ok.
47 |
48 | open_log(File, binary) ->
49 | case disk_log:open([{name, ?MODULE},
50 | {file, File},
51 | {format, internal}]) of
52 | {ok, Log} ->
53 | {fun(Term) ->
54 | disk_log:blog(Log, term_to_binary(Term, [compressed]))
55 | end,
56 | fun() ->
57 | disk_log:close(Log)
58 | end};
59 | {error, Reason} ->
60 | error({Reason, {open_log, File}})
61 | end.
62 |
63 | import_bin_log(Log, M, Db) ->
64 | import_bin_log(disk_log:bchunk(Log, start), Log, undefined, M, Db).
65 |
66 | import_bin_log({Cont, Bins}, Log, Tab, M, Db) ->
67 | Tab1 = import_bin_log_(Bins, Tab, M, Db),
68 | import_bin_log(disk_log:bchunk(Log, Cont), Log, Tab1, M, Db);
69 | import_bin_log({Cont, Bins, BadBytes}, Log, Tab, M, Db) ->
70 | io:fwrite("Warning: BadBytes = ~p~n", [BadBytes]),
71 | Tab1 = import_bin_log_(Bins, Tab, M, Db),
72 | import_bin_log(disk_log:bchunk(Log, Cont), Log, Tab1, M, Db);
73 | import_bin_log(eof, _, _, _, _) ->
74 | ok.
75 |
76 | import_bin_log_([Bin|Bins], Tab, M, Db) ->
77 | case binary_to_term(Bin) of
78 | {table, T, Opts} ->
79 | #table{type = Type} = TR = kvdb_lib:make_tabrec(T, Opts),
80 | M:add_table(Db, T, TR),
81 | import_bin_log_(Bins, {T,Type}, M, Db);
82 | {obj, Obj} ->
83 | add_to_tab(Tab, Obj, M, Db),
84 | import_bin_log_(Bins, Tab, M, Db)
85 | end;
86 | import_bin_log_([], Tab, _, _) ->
87 | Tab.
88 |
89 | add_to_tab(undefined, Obj, _, _) ->
90 | io:fwrite("Unknown tab, skipping ~p~n", [Obj]);
91 | add_to_tab({T, Type}, Obj, M, Db) ->
92 | IsQueue = kvdb_lib:valid_queue(Type),
93 | if Type == set ->
94 | M:put(Db, T, Obj);
95 | IsQueue == true ->
96 | {QK, St, O} = Obj,
97 | M:queue_insert(Db, T, QK, St, O)
98 | end.
99 |
100 | with_tab(Tab, Type, Ref, Out) ->
101 | IsQueue = kvdb_lib:valid_queue(Type),
102 | if Type == set ->
103 | Pat = make_pattern(Ref, Tab),
104 | chunk_load(kvdb:select(Ref, Tab, Pat, 1000), Out);
105 | IsQueue == true ->
106 | Filter = fun(St, QKey, Obj) ->
107 | {keep, {obj,{QKey, St, Obj}}}
108 | end,
109 | #kvdb_ref{mod = M, db = Db} = Ref,
110 | all_queues(
111 | fun(Q) ->
112 | load_queue(
113 | kvdb:list_queue(Ref, Tab, Q, Filter, false, 100), Out)
114 | end, M, Db, Tab)
115 | end.
116 |
117 | make_pattern(Db, Tab) ->
118 | case kvdb:info(Db, {Tab, encoding}) of
119 | {_,_,_} ->
120 | [{{'_','_','_'}, [], ['$_']}];
121 | _ ->
122 | [{{'_','_'}, [], ['$_']}]
123 | end.
124 |
125 |
126 | chunk_load({Objs, Cont}, Out) ->
127 | [Out({obj, Obj}) || Obj <- Objs],
128 | chunk_load(Cont(), Out);
129 | chunk_load(done, _) ->
130 | ok.
131 |
132 | all_queues(F, M, Db, T) ->
133 | all_queues(M:first_queue(Db, T), F, M, Db, T).
134 |
135 | all_queues(done, _, _, _, _) ->
136 | done;
137 | all_queues({ok,Q}, F, M, Db, T) ->
138 | F(Q),
139 | all_queues(M:next_queue(Db, T, Q), F, M, Db, T).
140 |
141 | load_queue({Objs, Cont}, F) ->
142 | lists:foreach(F, Objs),
143 | load_queue(Cont(), F);
144 | load_queue(done, _) ->
145 | done.
146 |
--------------------------------------------------------------------------------
/src/kvdb_log.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%% @doc
13 | %%% Experimental logging API
14 | %%% @end
15 | -module(kvdb_log).
16 |
17 | -export([create_table/2, create_table/3,
18 | add_log/3, add_log/4]).
19 | -export([log/3, adjust/3, list/3]).
20 |
21 | -include("kvdb.hrl").
22 | -record(i, {cur = 0, max = infinity}).
23 |
24 | create_table(Db, Table) ->
25 | create_table(Db, Table, {sext, term, term}).
26 |
27 | create_table(Db, Table, Enc) ->
28 | kvdb:add_table(Db, Table, [{type, fifo}, {encoding,Enc}]).
29 |
30 | add_log(Db, Table, Log) ->
31 | add_log(Db, Table, Log, infinity).
32 |
33 | add_log(Db, Table, Log, Max) when Max == infinity;
34 | is_integer(Max), Max > 0 ->
35 | case kvdb:is_queue_empty(Db, Table, Log) of
36 | true ->
37 | kvdb:queue_insert(Db, Table, meta_key(Log),
38 | inactive, meta_obj(Db, Table, Max)),
39 | ok;
40 | false ->
41 | {error, log_exists}
42 | end.
43 |
44 | log(_Db, _Table, _Obj) ->
45 | erlang:error(nyi).
46 |
47 | adjust(_Db, _Table, _Info) ->
48 | erlang:error(nyi).
49 |
50 | list(_Db, _Table, _Filter) ->
51 | erlang:error(nyi).
52 |
53 | meta_key(Log) ->
54 | #q_key{queue = Log, ts = 0, key = <<>>}.
55 |
56 | meta_obj(Db, Table, Max) ->
57 | case kvdb:info(Db, {Table, encoding}) of
58 | T when tuple_size(T) == 3 ->
59 | {<<>>, [], #i{max = Max}};
60 | _ ->
61 | {<<>>, #i{max = Max}}
62 | end.
63 |
--------------------------------------------------------------------------------
/src/kvdb_meta.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%% @doc
13 | %%% KVDB meta-data management API
14 | %%% @end
15 | %%%
16 | -module(kvdb_meta).
17 |
18 | -export([write/3,
19 | write_new/3,
20 | read/3,
21 | delete/2,
22 | update_counter/3]).
23 |
24 | -include("kvdb.hrl").
25 |
26 | write(#db{metadata = Ets}, Key, Value) ->
27 | ets:insert(Ets, {key(Key), Value}).
28 |
29 | write_new(#db{metadata = Ets}, Key, Value) ->
30 | ets:insert_new(Ets, {key(Key), Value}).
31 |
32 | read(#db{metadata = Ets}, Key, Default) ->
33 | case ets:lookup(Ets, key(Key)) of
34 | [] ->
35 | Default;
36 | [{_, Value}] ->
37 | Value
38 | end.
39 |
40 | delete(#db{metadata = Ets}, Key) ->
41 | %% io:fwrite("delete meta (~p) ~p~n", [Ets, Key]),
42 | ets:delete(Ets, key(Key)).
43 |
44 | update_counter(#db{metadata = Ets}, K, Incr) ->
45 | Key = key(K),
46 | try ets:update_counter(Ets, Key, Incr)
47 | catch
48 | error:_ ->
49 | ets:insert_new(Ets, {Key, 0}),
50 | ets:update_counter(Ets, Key, Incr)
51 | end.
52 |
53 | key(K) ->
54 | {'$kvdb_meta', K}.
55 |
--------------------------------------------------------------------------------
/src/kvdb_paired.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Paired backend for kvdb
13 | %%%
14 | %%% NOTE: This is work in progress. Several things still do not work
15 | %%%
16 | %%% The idea with this backend is to combine two backends as a write-through
17 | %%% pair. That is: all reads are done on 'backend 1', but writes are served
18 | %%% in both 'backend 1' and 'backend 2'. An example of how to use this would
19 | %%% be e.g. a kvdb_ets backend in front of a kvdb_riak backend:
20 | %%%
21 | %%%
22 | %%% kvdb:open(p, [{backend, kvdb_paired},
23 | %%% {module1, kvdb_ets},
24 | %%% {module2, kvdb_riak},
25 | %%% {options2, [{update_index, false}]}]).
26 | %%%
27 | %%%
28 | %%% The 'options2' list applies only to the 'backend 2' (riak in this case).
29 | %%% With `{update_index, false}' on 'backend 2', we will not maintain indexes
30 | %%% on the riak side, but rebuild them when the ets backend is populated.
31 | %%% @end
32 | -module(kvdb_paired).
33 |
34 | -behaviour(kvdb).
35 |
36 | -export([open/2, close/1]).
37 | -export([add_table/3, delete_table/2, list_tables/1]).
38 | -export([put/3, push/4, get/3, get_attrs/4, index_get/4, index_keys/4,
39 | update_counter/4, pop/3, prel_pop/3, extract/3, delete/3,
40 | list_queue/3, list_queue/6, list_queue/7, is_queue_empty/3,
41 | queue_read/3, queue_insert/5, queue_delete/3, mark_queue_object/4,
42 | queue_head_write/4, queue_head_read/3, queue_head_delete/3]).
43 | -export([first_queue/2, next_queue/3]).
44 | -export([first/2, last/2, next/3, prev/3,
45 | prefix_match/3, prefix_match/4, prefix_match_rel/5]).
46 | -export([get_schema_mod/2,
47 | schema_write/4,
48 | schema_read/3,
49 | schema_delete/3,
50 | schema_fold/3]).
51 | -export([info/2, is_table/2]).
52 | -export([dump_tables/1]).
53 |
54 | -export([proxy_childspecs/2]).
55 |
56 | -include("kvdb.hrl").
57 |
58 | -define(if_table(Db, Tab, Expr), if_table(Db, Tab, fun() -> Expr end)).
59 |
60 | info(#db{ref = {{M,Db},_} = Ref}, What) ->
61 | case What of
62 | ref -> Ref;
63 | _ ->
64 | M:info(Db, What)
65 | end.
66 |
67 | proxy_childspecs(Name, Options) ->
68 | {M1, Opts1, M2, Opts2} = db_pair(Options),
69 | proxy_childspecs_(M1, Name, Opts1) ++
70 | proxy_childspecs_(M2, Name, Opts2).
71 |
72 | proxy_childspecs_(M, Name, Opts) ->
73 | try M:proxy_childspecs(Name, Opts)
74 | catch
75 | error:undef ->
76 | []
77 | end.
78 |
79 | is_table(#db{ref = {{M,Db},_}}, Tab) ->
80 | M:is_table(Db, Tab).
81 |
82 | get_schema_mod(#db{ref = {{M,Db},_}}, Default) ->
83 | M:get_schema_mod(Db, Default).
84 |
85 | schema_write(#db{ref = {{M1,Db1},{M2,Db2}}}, Cat, K, V) ->
86 | ok = M1:schema_write(Db1, Cat, K, V),
87 | ok = M2:schema_write(Db2, Cat, K, V).
88 |
89 | schema_read(#db{ref = {{M1,Db1},_}}, Cat, K) ->
90 | M1:schema_read(Db1, Cat, K).
91 |
92 | schema_delete(#db{ref = {{M1,Db1},{M2,Db2}}}, Cat, K) ->
93 | ok = M1:schema_delete(Db1, Cat, K),
94 | ok = M2:schema_delete(Db2, Cat, K).
95 |
96 | schema_fold(#db{ref = {{M1,Db1},_}}, F, A) ->
97 | M1:schema_fold(Db1, F, A).
98 |
99 | dump_tables(#db{ref = {{M,Db},_}}) ->
100 | M:dump_tables(Db).
101 |
102 |
103 | open(DbName, Options) ->
104 | {M1, O1, M2, O2} = db_pair(Options),
105 | case M1:open(DbName, O1) of
106 | {ok, Db1} ->
107 | case M2:open(DbName, O2) of
108 | {ok, Db2} ->
109 | Db = Db1#db{ref = {{M1, Db1}, {M2, Db2}}},
110 | ok = load(Db),
111 | {ok, Db};
112 | Error2 ->
113 | M1:close(Db1),
114 | Error2
115 | end;
116 | Error1 ->
117 | Error1
118 | end.
119 |
120 | db_pair(Options) ->
121 | {_, M1} = lists:keyfind(module1, 1, Options),
122 | {_, M2} = lists:keyfind(module2, 1, Options),
123 | {O1, O2} = split_options(Options),
124 | {M1, O1, M2, O2}.
125 |
126 | split_options(Options) ->
127 | {O1, R1} = take(options1, Options),
128 | {O2, R2} = take(options2, R1),
129 | {O1 ++ R2, O2 ++ R2}.
130 |
131 | take(K, Options) ->
132 | case lists:keytake(K, 1, Options) of
133 | {value, {_, Found}, Rest} ->
134 | {Found, Rest};
135 | false ->
136 | {[], Options}
137 | end.
138 |
139 | load(#db{ref = {{M1,Db1}, {M2, Db2}}} = Db) ->
140 | Tabs = M2:list_tables(Db2),
141 | lists:foreach(
142 | fun(T) ->
143 | case M2:schema_read(Db2, property, {T,autoload}) of
144 | false ->
145 | M1:schema_write(Db1, property, {T, ram}, false),
146 | ok;
147 | _ ->
148 | M1:schema_write(Db1, property, {T, ram}, true),
149 | M1:delete_table(Db1, T),
150 | TabR = M2:info(Db2, {T, tabrec}),
151 | M1:add_table(Db1, T, TabR),
152 | case has_disk(M2, Db2, T) of
153 | false ->
154 | io:fwrite("Table ~s defined as {disk, false}~n", [T]),
155 | ok;
156 | _ ->
157 | case TabR#table.type of
158 | set ->
159 | chunk_load(M2:prefix_match(Db2, T, <<>>, 1000),
160 | M1, Db1, T);
161 | Type when Type==fifo; Type==lifo;
162 | Type=={keyed,fifo}; Type=={keyed,lifo} ->
163 | Filter = fun(St, QKey, Obj) ->
164 | {keep, {QKey, St, Obj}}
165 | end,
166 | all_queues(
167 | fun(Q) ->
168 | load_queue(
169 | M2:list_queue(Db2, T, Q,
170 | Filter, false, 100),
171 | M1, Db1, T)
172 | end, M2, Db2, T)
173 | end
174 | end
175 | end
176 | end, Tabs),
177 | load_schema(Db).
178 |
179 | load_schema(#db{ref = {{M1,Db1}, {M2, Db2}}}) ->
180 | M2:schema_fold(
181 | Db2, fun(C, {K,V}, _) ->
182 | case M1:schema_read(Db1, C, K) of
183 | undefined ->
184 | M1:schema_write(Db1, C, K, V);
185 | _ ->
186 | ok
187 | end
188 | end, ok).
189 |
190 | all_queues(F, M, Db, T) ->
191 | all_queues(M:first_queue(Db, T), F, M, Db, T).
192 |
193 | all_queues(done, _, _, _, _) ->
194 | done;
195 | all_queues({ok,Q}, F, M, Db, T) ->
196 | F(Q),
197 | all_queues(M:next_queue(Db, T, Q), F, M, Db, T).
198 |
199 | load_queue({Objs, Cont}, M, Db, T) ->
200 | lists:foreach(
201 | fun({QKey, St, Obj}) ->
202 | M:queue_insert(Db, T, QKey, St, Obj)
203 | end, Objs),
204 | load_queue(Cont(), M, Db, T);
205 | load_queue(done, _, _, _) ->
206 | done.
207 |
208 |
209 |
210 | chunk_load({Objs, Cont}, M, Db, T) ->
211 | [M:put(Db, T, Obj) || Obj <- Objs],
212 | chunk_load(Cont(), M, Db, T);
213 | chunk_load(done, _, _, _) ->
214 | ok.
215 |
216 |
217 | close(#db{ref = {{M1,Db1},{M2,Db2}}}) ->
218 | M1:close(Db1),
219 | M2:close(Db2).
220 |
221 | add_table(#db{ref = {{M1,Db1},{M2,Db2}}}, Table, Opts) when is_list(Opts) ->
222 | case M1:info(Db1, {Table, type}) of
223 | undefined ->
224 | _TabR = kvdb_lib:make_tabrec(Table, check_encoding(Opts, Db1)),
225 | M2:add_table(Db2, Table, Opts),
226 | M1:add_table(Db1, Table, Opts);
227 | _ -> ok
228 | end;
229 | add_table(#db{ref = {{M1,Db1},{M2,Db2}}}, Table, #table{} = TabR) ->
230 | case M1:is_table(Db1, Table) of
231 | true -> ok;
232 | false ->
233 | M2:add_table(Db2, Table, TabR),
234 | M1:add_table(Db1, Table, TabR)
235 | end.
236 |
237 |
238 | list_tables(#db{ref = {{M,Db},_}}) ->
239 | M:list_tables(Db).
240 |
241 | delete_table(#db{ref = {{M1,Db1},{M2,Db2}}}, Table) ->
242 | case M2:delete_table(Db2, Table) of
243 | ok ->
244 | M1:delete_table(Db1, Table);
245 | Other ->
246 | Other
247 | end.
248 |
249 | put(#db{ref = {{M1,Db1},{M2,Db2}}}, Table, Obj) ->
250 | case has_disk(M1, Db1, Table) of
251 | true ->
252 | case M2:put(Db2, Table, Obj) of
253 | ok ->
254 | M1:put(Db1, Table, Obj);
255 | Other ->
256 | Other
257 | end;
258 | false ->
259 | M1:put(Db1, Table, Obj)
260 | end.
261 |
262 | update_counter(#db{ref = {{M1,Db1},{M2,Db2}}}, Table, Key, Incr) ->
263 | NewValue = M1:update_counter(Db1, Table, Key, Incr),
264 | {ok, Obj} = M1:get(Db1, Table, Key),
265 | case has_disk(M2, Db2, Table) of
266 | true -> M2:put(Db2, Table, Obj);
267 | false -> ok
268 | end,
269 | NewValue.
270 | %% M2:update_counter(Db2, Table, Key, Incr),
271 | %% M1:update_counter(Db1, Table, Key, Incr).
272 |
273 | push(#db{ref = {{M1,Db1},{M2,Db2}}}, Table, Q, Obj) ->
274 | case has_disk(M1, Db1, Table) of
275 | false -> ok;
276 | _ ->
277 | M2:push(Db2, Table, Q, Obj)
278 | end,
279 | M1:push(Db1, Table, Q, Obj).
280 |
281 | get(#db{ref = {{M,Db},_}}, Tab, K) ->
282 | M:get(Db, Tab, K).
283 |
284 | get_attrs(#db{ref = {{M,Db},_}}, Tab, K, As) ->
285 | M:get_attrs(Db, Tab, K, As).
286 |
287 | index_get(#db{ref = {{M,Db},_}}, Tab, K, V) ->
288 | M:index_get(Db, Tab, K, V).
289 |
290 | index_keys(#db{ref = {{M,Db},_}}, T, K, V) ->
291 | M:index_keys(Db, T, K, V).
292 |
293 | pop(#db{ref = {{M1,Db1},{M2,Db2}}}, T, Q) ->
294 | M2:pop(Db2, T, Q),
295 | M1:pop(Db1, T, Q).
296 |
297 | prel_pop(#db{ref = {{M1,Db1},{M2,Db2}}}, T, Q) ->
298 | M2:prel_pop(Db2, T, Q),
299 | M1:prel_pop(Db1, T, Q).
300 |
301 | extract(#db{ref = {{M1,Db1},{M2,Db2}}}, T, K) ->
302 | M2:extract(Db2, T, K),
303 | M1:extract(Db1, T, K).
304 |
305 | delete(#db{ref = {{M1,Db1},{M2,Db2}}}, T, K) ->
306 | case has_disk(M1, Db1, T) of
307 | true -> M2:delete(Db2, T, K);
308 | false -> ok
309 | end,
310 | M1:delete(Db1, T, K).
311 |
312 | list_queue(#db{ref = {{M,Db},_}}, T, Q) ->
313 | M:list_queue(Db, T, Q).
314 |
315 | list_queue(#db{ref = {{M,Db},_}}, T, Q, Fltr, HeedBlock, Limit) ->
316 | M:list_queue(Db, T, Q, Fltr, HeedBlock, Limit).
317 |
318 | list_queue(#db{ref = {{M,Db},_}}, T, Q, Fltr, HeedBlock, Limit, Reverse) ->
319 | M:list_queue(Db, T, Q, Fltr, HeedBlock, Limit, Reverse).
320 |
321 | is_queue_empty(#db{ref = {{M,Db},_}}, T, Q) ->
322 | M:is_queue_empty(Db, T, Q).
323 |
324 | queue_read(#db{ref = {{M,Db},_}}, T, K) ->
325 | M:queue_read(Db, T, K).
326 |
327 | queue_insert(#db{ref = {{M1,Db1},{M2,Db2}}}, T, K, St, Obj) ->
328 | case has_disk(M1, Db1, T) of
329 | true -> M2:queue_insert(Db2, T, K, St, Obj);
330 | false -> ok
331 | end,
332 | M1:queue_insert(Db1, T, K, St, Obj).
333 |
334 | queue_delete(#db{ref = {{M1,Db1},{M2,Db2}}}, T, K) ->
335 | case has_disk(M1, Db1, T) of
336 | true -> M2:queue_delete(Db2, T, K);
337 | false -> ok
338 | end,
339 | M1:queue_delete(Db1, T, K).
340 |
341 | mark_queue_object(#db{ref = {{M1,Db1},{M2,Db2}}}, Tab, K, St) ->
342 | case has_disk(M1, Db1, Tab) of
343 | true -> M2:mark_queue_object(Db2, Tab, K, St);
344 | false -> ok
345 | end,
346 | M1:mark_queue_object(Db1, Tab, K, St).
347 |
348 | queue_head_write(#db{ref = {{M1,Db1},{M2,Db2}}}, Tab, Q, Obj) ->
349 | case has_disk(M1, Db1, Tab) of
350 | true -> M2:queue_head_write(Db2, Tab, Q, Obj);
351 | false -> ok
352 | end,
353 | M1:queue_head_write(Db1, Tab, Q, Obj).
354 |
355 | queue_head_read(#db{ref = {{M1,Db1},_}}, Tab, Q) ->
356 | M1:queue_head_read(Db1, Tab, Q).
357 |
358 | queue_head_delete(#db{ref = {{M1,Db1},{M2,Db2}}}, Tab, Q) ->
359 | case has_disk(M1, Db1, Tab) of
360 | true -> M2:queue_head_delete(Db2, Tab, Q);
361 | false -> ok
362 | end,
363 | M1:queue_head_delete(Db1, Tab, Q).
364 |
365 | first_queue(#db{ref = {{M,Db},_}}, Tab) ->
366 | M:first_queue(Db, Tab).
367 |
368 | next_queue(#db{ref = {{M,Db},_}}, Tab, Q) ->
369 | M:next_queue(Db, Tab, Q).
370 |
371 | first(#db{ref = {{M,Db},_}}, Tab) ->
372 | M:first(Db, Tab).
373 |
374 | last(#db{ref = {{M,Db},_}}, Tab) ->
375 | M:last(Db, Tab).
376 |
377 | next(#db{ref = {{M,Db},_}}, Tab, K) ->
378 | M:next(Db, Tab, K).
379 |
380 | prev(#db{ref = {{M,Db},_}}, Tab, K) ->
381 | M:prev(Db, Tab, K).
382 |
383 | prefix_match(#db{ref = {{M,Db},_}}, Tab, Pfx) ->
384 | M:prefix_match(Db, Tab, Pfx).
385 |
386 | prefix_match(#db{ref = {{M,Db},_}}, Tab, Pfx, Limit) ->
387 | M:prefix_match(Db, Tab, Pfx, Limit).
388 |
389 | prefix_match_rel(#db{ref = {{M,Db},_}}, Tab, Prefix, Start, Limit) ->
390 | M:prefix_match_rel(Db, Tab, Prefix, Start, Limit).
391 |
392 |
393 | check_encoding(Opts, #db{encoding = Enc0}) ->
394 | case lists:keymember(encoding, 1, Opts) of
395 | true ->
396 | Opts;
397 | false ->
398 | [{encoding, Enc0}|Opts]
399 | end.
400 |
401 | has_disk(M1, Db1, T) ->
402 | case M1:schema_read(Db1, property, {T, disk}) of
403 | false -> false;
404 | _ -> true
405 | end.
406 |
--------------------------------------------------------------------------------
/src/kvdb_proxy_sup.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Database proxy supervisor.
13 | %%%
14 | %%% NOTE: This is work in progress, highly experimental.
15 | %%% The general idea is that a backend can specify 'proxy processes'
16 | %%% as a list of childspecs. An example of such a proxy is kvdb_riak_proxy.
17 | %%%
18 | %%% @end
19 | -module(kvdb_proxy_sup).
20 | -behaviour(supervisor).
21 |
22 | -export([start_link/2]).
23 |
24 | -export([init/1]).
25 |
26 | %% ===================================================================
27 | %% API functions
28 | %% ===================================================================
29 |
30 | start_link(Name, Options) ->
31 | supervisor:start_link(?MODULE, {Name, Options}).
32 |
33 | %% ===================================================================
34 | %% Supervisor callbacks
35 | %% ===================================================================
36 |
37 | init({Name, Options}) ->
38 | gproc:reg({n,l,{?MODULE,Name}}),
39 | Backend = proplists:get_value(backend, Options, ets),
40 | Mod = kvdb_lib:backend_mod(Backend),
41 | Children = try Mod:proxy_childspecs(Name, Options)
42 | catch
43 | error:undef ->
44 | []
45 | end,
46 | {ok, { {one_for_one, 5, 10}, Children } }.
47 |
--------------------------------------------------------------------------------
/src/kvdb_queue.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Useful queue inspection and management functions
13 | %%% @end
14 | %%%
15 | -module(kvdb_queue).
16 |
17 | -export([push/3, push/4,
18 | pop/2, pop/3,
19 | prel_pop/2, prel_pop/3,
20 | delete/3, extract/3,
21 | is_empty/3,
22 | size/3,
23 | list_queues/2, list_queues/3,
24 | list/3, list/6, list_full/3,
25 | first/2,
26 | next/3,
27 | clear_queue/3,
28 | clear_queues/2]).
29 |
30 | -export([info/4]).
31 |
32 | push(Db, Table, Obj) -> push(Db, Table, Obj).
33 | push(Db, Table, Q, Obj) -> kvdb:push(Db, Table, Q, Obj).
34 | pop(Db, Table) -> pop(Db, Table, <<>>).
35 | pop(Db, Table, Q) -> kvdb:pop(Db, Table, Q).
36 | prel_pop(Db, Table) -> prel_pop(Db, Table, <<>>).
37 | prel_pop(Db, Table, Q) -> kvdb:prel_pop(Db, Table, Q).
38 | delete(Db, Table, Key) -> kvdb:delete(Db, Table, Key).
39 | extract(Db, Table, Key) -> kvdb:extract(Db, Table, Key).
40 | is_empty(Db, Table, Q) -> kvdb:is_queue_empty(Db, Table, Q).
41 | list(Db, Table, Q) -> kvdb:list_queue(Db, Table, Q).
42 | list(Db,Tab,Q,Fltr,HeedBlock,Limit) ->
43 | kvdb:list_queue(Db, Tab, Q, Fltr, HeedBlock, Limit).
44 | list_full(Db,Tab,Q) ->
45 | list(Db, Tab, Q, fun(S,K,O) -> {keep,{S,K,O}} end, false, infinity).
46 | first(Db, Table) -> kvdb:first_queue(Db, Table).
47 | next(Db, Table, PrevQ) -> kvdb:next_queue(Db, Table, PrevQ).
48 |
49 | size(Db, Table, Q) ->
50 | try kvdb:queue_head_read(Db, Table, Q) of
51 | {error, not_found} ->
52 | 0;
53 | {ok, {_,<>}} -> Sz;
54 | {ok, {_, _, <>}} -> Sz
55 | catch
56 | error:badarg -> undefined
57 | end.
58 |
59 | list_queues(Db, Table) ->
60 | list_queues(Db, Table, 30).
61 |
62 | list_queues(Db, Table, Limit) when
63 | Limit == infinity; is_integer(Limit), Limit >= 0 ->
64 | list_queues_(first(Db, Table), Db, Table, Limit, Limit, []).
65 |
66 | clear_queue(Db, Table, Q) ->
67 | clear_queue_(kvdb:list_queue(Db, Table, Q,
68 | fun(_,K,_) -> {keep,K} end,
69 | false, infinity), Db, Table, Q).
70 |
71 | clear_queues(Db, Table) ->
72 | clear_queues_(first(Db, Table), Db, Table).
73 |
74 | clear_queues_({ok, Q}, Db, Table) ->
75 | clear_queue(Db, Table, Q),
76 | clear_queues_(next(Db, Table, Q), Db, Table);
77 | clear_queues_(done, _, _) ->
78 | done.
79 |
80 |
81 | clear_queue_(done, _, _, _) ->
82 | done;
83 | clear_queue_({Keys, Cont}, Db, Table, Q) ->
84 | lists:foreach(
85 | fun(Key) ->
86 | kvdb:delete(Db, Table, Key)
87 | end, Keys),
88 | clear_queue_(Cont(), Db, Table, Q).
89 |
90 | list_queues_({ok, Q}, Db, Table, Limit0, Limit, Acc) ->
91 | case decr(Limit) of
92 | 0 ->
93 | {lists:reverse([Q|Acc]),
94 | fun() ->
95 | Next = next(Db, Table, Q),
96 | list_queues_(Next, Db, Table, Limit0, Limit0, [])
97 | end};
98 | Limit1 ->
99 | list_queues_(next(Db, Table, Q), Db, Table, Limit0, Limit1, [Q|Acc])
100 | end;
101 | list_queues_(done, _Db, _Table, _Limit0, _Limit, Acc) ->
102 | if Acc == [] -> done;
103 | true -> {lists:reverse(Acc), fun() -> done end}
104 | end.
105 |
106 | decr(infinity) ->
107 | infinity;
108 | decr(I) when is_integer(I) ->
109 | I-1.
110 |
111 |
112 | info(Db, Table, Q, size) ->
113 | case kvdb:queue_head_read(Db, Table, Q) of
114 | {ok, {_, <>}} -> Sz;
115 | {ok, {_, _, <>}} -> Sz;
116 | _ -> 0
117 | end;
118 | info(Db, Table, _Q, type) ->
119 | case kvdb:info(Db, {Table, type}) of
120 | T when T==fifo; T==lifo -> T;
121 | T when element(1,T) == keyed -> T;
122 | {fifo,_} = T -> T;
123 | {lifo,_} = T -> T;
124 | undefined -> undefined;
125 | Other ->
126 | error({not_a_queue, Other})
127 | end.
128 |
--------------------------------------------------------------------------------
/src/kvdb_riak.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Highly experimental riak backend for kvdb
13 | %%%
14 | %%% NOTE: This is work in progress. Several things still do not work
15 | %%%
16 | %%% The best way to use this for now is via the kvdb_paired backend.
17 | %%% @end
18 | -module(kvdb_riak).
19 |
20 | -behaviour(kvdb).
21 |
22 | -export([open/2, close/1]).
23 | -export([add_table/3, delete_table/2, list_tables/1]).
24 | -export([put/3, push/4, get/3, get_attrs/4, index_get/4, index_keys/4,
25 | update_counter/4, pop/3, prel_pop/3, extract/3, delete/3,
26 | list_queue/3, list_queue/6, is_queue_empty/3,
27 | queue_read/3, queue_insert/5, queue_delete/3, mark_queue_object/4,
28 | queue_head_write/4, queue_head_read/3, queue_head_delete/3]).
29 | -export([first_queue/2, next_queue/3]).
30 | -export([first/2, last/2, next/3, prev/3,
31 | prefix_match/3, prefix_match/4, prefix_match_rel/5]).
32 | -export([get_schema_mod/2]).
33 | -export([info/2, is_table/2]).
34 | -export([dump_tables/1]).
35 |
36 | -export([proxy_childspecs/2]).
37 |
38 | -include("kvdb.hrl").
39 | %% This is obviously a crude hack (riak_kv_wm_raw.hrl copied into include/)
40 | -include("riak_kv_wm_raw.hrl").
41 |
42 | -define(if_table(Db, Tab, Expr), if_table(Db, Tab, fun() -> Expr end)).
43 |
44 | -record(ref, {bucket,
45 | update_index = true}).
46 |
47 | info(#db{} = Db, What) ->
48 | case What of
49 | tables -> list_tables(Db);
50 | encoding -> Db#db.encoding;
51 | ref -> Db#db.ref;
52 | {Tab,encoding} -> ?if_table(Db, Tab, encoding(Db, Tab));
53 | {Tab,index } -> ?if_table(Db, Tab, index(Db, Tab));
54 | {Tab,type } -> ?if_table(Db, Tab, type(Db, Tab));
55 | {Tab,schema } -> ?if_table(Db, Tab, schema(Db, Tab));
56 | {Tab,tabrec } -> schema_lookup(Db, {table, Tab}, undefined);
57 | _ -> undefined
58 | end.
59 |
60 | encoding(_, _) -> error(nyi).
61 | index(#db{ref = #ref{update_index = B}} = Db, Tab) ->
62 | if B -> schema_lookup(Db, {Tab, index}, []);
63 | true -> []
64 | end.
65 | type(_, _) -> error(nyi).
66 | schema(_, _) -> error(nyi).
67 |
68 | schema_lookup(#db{metadata = Ets}, Key, Default) ->
69 | case ets:lookup(Ets, Key) of
70 | [] ->
71 | Default;
72 | [{_, V}] ->
73 | V
74 | end.
75 |
76 | schema_write(#db{metadata = Ets}, {_, _} = Obj) ->
77 | ets:insert(Ets, Obj).
78 |
79 | is_table(#db{metadata = ETS}, Tab) ->
80 | ets:member(ETS, {table, Tab}).
81 |
82 | get_schema_mod(Db, Default) ->
83 | case schema_lookup(Db, schema_mod, undefined) of
84 | undefined ->
85 | schema_write(Db, {schema_mod, Default}),
86 | Default;
87 | M ->
88 | M
89 | end.
90 |
91 | if_table(Db, Tab, F) ->
92 | case is_table(Db, Tab) of
93 | true -> F();
94 | false -> undefined
95 | end.
96 |
97 | dump_tables(_) ->
98 | [].
99 |
100 |
101 | proxy_childspecs(Name, Options) ->
102 | [{riak_proxy, {kvdb_riak_proxy, start_link, [Name, Options]},
103 | permanent, 5000, worker, [kvdb_riak_proxy]}].
104 |
105 | open(DbName, Options) ->
106 | case kvdb_riak_proxy:get_session(DbName) of
107 | undefined ->
108 | error(no_session); % might want to improve this later
109 | Session ->
110 | Ets = ets:new(kvdb_schema, [ordered_set, public]),
111 | ets:insert(Ets, {riak_session, Session}),
112 | Ref = #ref{bucket = <<"kvdb*", (to_bin(DbName))/binary>>,
113 | update_index = proplists:get_value(update_index,
114 | Options, true)},
115 | Db = load_schema(#db{ref = Ref, metadata = Ets}, Session),
116 | {ok, Db}
117 | end.
118 |
119 | close(Db) ->
120 | case session_pid(Db) of
121 | undefined -> ok;
122 | {client, _} -> ok;
123 | {pb, Session} ->
124 | riakc_pb_socket:stop(Session),
125 | ok
126 | end.
127 |
128 | add_table(#db{ref = Ref, metadata = Ets} = Db, Table, Opts) ->
129 | case is_table(Db, Table) of
130 | false ->
131 | TabR = case Opts of
132 | _ when is_list(Opts) ->
133 | kvdb_lib:make_tabrec(Table, Opts);
134 | #table{} ->
135 | Opts
136 | end,
137 | case session_pid(Db) of
138 | undefined -> error(no_session);
139 | Session ->
140 | riak_put(Session, Ref#ref.bucket, Table,
141 | {Table, TabR}, []),
142 | write_tabrec_ets(Ets, TabR)
143 | end;
144 | true ->
145 | {error, exists}
146 | end.
147 |
148 |
149 | list_tables(#db{metadata = Ets}) ->
150 | ets:select(Ets, [{ {{table,'$1'},'_'}, [], ['$1']}]).
151 |
152 | delete_table(#db{}, _Table) ->
153 | error(nyi).
154 |
155 | put(#db{ref = Ref} = Db, Table, {K,_} = Obj) ->
156 | case session_pid(Db) of
157 | undefined -> error(no_session);
158 | Session ->
159 | Bucket = tab_key(Ref, Table),
160 | riak_put(Session, Bucket, K, Obj, []),
161 | ok
162 | end;
163 | put(#db{ref = Ref} = Db, Table, {K, Attrs, V} = Obj) ->
164 | case session_pid(Db) of
165 | undefined -> error(no_session);
166 | Session ->
167 | Ix = index(Db, Table),
168 | IxVals = kvdb_lib:index_vals(Ix, K, Attrs, fun() -> V end),
169 | riak_put(Session, tab_key(Ref, Table), K, Obj, IxVals),
170 | ok
171 | end.
172 |
173 | riak_put({pb, Session}, Bucket, K, Val, IxVals) ->
174 | O = riakc_obj:new(Bucket, K, Val),
175 | O1 = if IxVals =/= [] ->
176 | riakc_obj:set_secondary_index(
177 | O, [{{binary_index, to_bin(N)}, [I]}
178 | || {N,I} <- IxVals]);
179 | true ->
180 | O
181 | end,
182 | riakc_pb_socket:put(Session, O1);
183 | riak_put({client, Session}, Bucket, K, Val0, IxVals) ->
184 | Val = term_to_binary(Val0),
185 | Obj = if IxVals =/= [] ->
186 | Ix = [{ix_field(Ni), Vi} || {Ni, Vi} <- IxVals],
187 | riak_object:new(Bucket, K, Val,
188 | dict:from_list(
189 | [{?MD_INDEX, Ix}]));
190 | true ->
191 | riak_object:new(Bucket, K, Val)
192 | end,
193 | riak_client:put(Obj, Session).
194 |
195 | ix_field(Name) ->
196 | <<(to_bin(Name))/binary, "_bin">>.
197 |
198 | to_bin(B) when is_binary(B) -> B;
199 | to_bin(L) when is_list(L) -> list_to_binary(L);
200 | to_bin(A) when is_atom(A) -> atom_to_binary(A, latin1).
201 |
202 | update_counter(_Db, _Table, _Key, _Incr) ->
203 | error(nyi).
204 |
205 | push(_, _, _, _) ->
206 | error(nyi).
207 |
208 | get(#db{ref = Ref} = Db, Tab, K) ->
209 | case session_pid(Db) of
210 | undefined -> error(no_session);
211 | Session ->
212 | riak_get(tab_key(Ref, Tab), K, Session)
213 | end.
214 |
215 | get_attrs(_, _, _, _) ->
216 | error(nyi).
217 |
218 | index_get(_, _, _, _) ->
219 | error(nyi).
220 |
221 | index_keys(_, _, _, _) ->
222 | error(nyi).
223 |
224 | pop(_, _, _) ->
225 | error(nyi).
226 |
227 | prel_pop(_, _, _) ->
228 | error(nyi).
229 |
230 | extract(_, _, _) ->
231 | error(nyi).
232 |
233 | delete(#db{ref = Ref} = Db, Tab, K) ->
234 | case session_pid(Db) of
235 | undefined -> error(no_session);
236 | Session ->
237 | riak_delete(tab_key(Ref, Tab), K, Session)
238 | end.
239 |
240 | list_queue(_, _, _) ->
241 | error(nyi).
242 |
243 | list_queue(_, _, _, _, _, _) ->
244 | error(nyi).
245 |
246 | is_queue_empty(_, _, _) ->
247 | error(nyi).
248 |
249 | queue_read(_, _, _) ->
250 | error(nyi).
251 |
252 | queue_insert(_, _, _, _, _) ->
253 | error(nyi).
254 |
255 | queue_delete(_, _, _) ->
256 | error(nyi).
257 |
258 | mark_queue_object(_, _, _, _) ->
259 | error(nyi).
260 |
261 | queue_head_write(_, _, _, _) ->
262 | error(nyi).
263 |
264 | queue_head_read(_, _, _) ->
265 | error(nyi).
266 |
267 | queue_head_delete(_, _, _) ->
268 | error(nyi).
269 |
270 | first_queue(_, _) ->
271 | error(nyi).
272 |
273 | next_queue(_, _, _) ->
274 | error(nyi).
275 |
276 | first(_, _) ->
277 | error(nyi).
278 |
279 | last(_, _) ->
280 | error(nyi).
281 |
282 | next(_, _, _) ->
283 | error(nyi).
284 |
285 | prev(_, _, _) ->
286 | error(nyi).
287 |
288 | prefix_match(Db, Table, Pfx) ->
289 | ?if_table(Db, Table, prefix_match_(Db, Table, Pfx)).
290 |
291 | prefix_match_(#db{ref = Ref} = Db, Table, Pfx) ->
292 | case session_pid(Db) of
293 | undefined -> error(no_session);
294 | {pb, Session} ->
295 | Bucket = tab_key(Ref, Table),
296 | case riakc_pb_socket:mapred_bucket(
297 | Session, Bucket,
298 | [kvdb_riak_mapred:map_prefix_match(Pfx),
299 | riak_kv_mapreduce:reduce_sort(true)]) of
300 | {ok, []} ->
301 | [];
302 | {ok, MapRedRes} ->
303 | lists:last(MapRedRes);
304 | Other ->
305 | error(Other)
306 | end;
307 | _ ->
308 | error(nyi)
309 | end.
310 |
311 | prefix_match(Db, Table, Pfx, infinity) ->
312 | {prefix_match(Db, Table, Pfx), fun() -> done end};
313 | prefix_match(Db, Table, Pfx, Limit) when is_integer(Limit), Limit > 0 ->
314 | %% Fake chunking for now
315 | chunk(prefix_match(Db, Table, Pfx), Limit).
316 |
317 | chunk(L, Limit) ->
318 | if length(L) >= Limit ->
319 | {A, B} = lists:split(Limit, L),
320 | {A, fun() ->
321 | chunk(B, Limit)
322 | end};
323 | true ->
324 | {L, fun() -> done end}
325 | end.
326 |
327 | prefix_match_rel(_, _, _, _, _) ->
328 | error(nyi).
329 |
330 |
331 | tab_key(#ref{bucket = B}, Tab) ->
332 | <>.
333 |
334 | session_pid(#db{metadata = Ets}) ->
335 | case ets:lookup(Ets, riak_session) of
336 | [] ->
337 | undefined;
338 | [{_, Pid}] ->
339 | Pid
340 | end.
341 |
342 | riak_get(Bucket, K, {pb, Session}) ->
343 | case riakc_pb_socket:get(Session, Bucket, K) of
344 | {ok, Obj} ->
345 | {ok, binary_to_term(riakc_obj:get_value(Obj))};
346 | {error,notfound} ->
347 | {error, not_found}
348 | end;
349 | riak_get(Bucket, K, {client, C}) ->
350 | case riak_client:get(Bucket, K, C) of
351 | {ok, Obj} ->
352 | {ok, binary_to_term(riak_object:get_value(Obj))};
353 | {error, notfound} -> {error, not_found}
354 | end.
355 |
356 | riak_delete(Bucket, K, {pb, Session}) ->
357 | case riakc_pb_socket:delete(Session, Bucket, K) of
358 | ok ->
359 | ok;
360 | {error,_} = Error ->
361 | Error
362 | end;
363 | riak_delete(Bucket, K, {client, C}) ->
364 | case riak_client:delete(Bucket, K, C) of
365 | ok ->
366 | ok;
367 | {error,_} = Error ->
368 | Error
369 | end.
370 |
371 |
372 | load_schema(#db{ref = Ref, metadata = Ets} = Db, Session) ->
373 | case riak_list_keys(Bucket = Ref#ref.bucket, Session) of
374 | {ok, Keys} ->
375 | lists:foreach(
376 | fun(K) ->
377 | case riak_get(Bucket, K, Session) of
378 | {ok, {_Tab, TabR}} ->
379 | write_tabrec_ets(Ets, TabR);
380 | {error, not_found} ->
381 | %% ?
382 | ok
383 | end
384 | end, Keys);
385 | _ ->
386 | ok
387 | end,
388 | Db.
389 |
390 | riak_list_keys(Bucket, {client, Session}) ->
391 | riak_client:list_keys(Bucket, Session);
392 | riak_list_keys(Bucket, {pb, Session}) ->
393 | riakc_pb_socket:list_keys(Session, Bucket).
394 |
395 |
396 | write_tabrec_ets(Ets, #table{name = Name,
397 | type = Type,
398 | index = Ix,
399 | encoding = Enc} = R) ->
400 | ets:insert(Ets, [{{table, Name}, R},
401 | {{Name, type}, Type},
402 | {{Name, index}, Ix},
403 | {{Name, encoding, Enc}}]).
404 |
--------------------------------------------------------------------------------
/src/kvdb_riak_mapred.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Riak map-reduce hooks for kvdb_riak backend
13 | %%%
14 | %%% NOTE: This is work in progress, highly experimental.
15 | %%%
16 | %%% @end
17 | -module(kvdb_riak_mapred).
18 |
19 | -compile(export_all).
20 |
21 |
22 |
23 | map_prefix_match(Pfx) ->
24 | {map, {modfun, ?MODULE, map_prefix_match}, Pfx, false}.
25 |
26 | map_prefix_match(Obj, _, Pfx) ->
27 | Sz = byte_size(Pfx),
28 | Key = riak_object:key(Obj),
29 | case Key of
30 | <> ->
31 | [binary_to_term(riak_object:get_value(Obj))];
32 | _ ->
33 | []
34 | end.
35 |
36 |
--------------------------------------------------------------------------------
/src/kvdb_riak_proxy.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% Riak connectivity manager (proxy) for kvdb_riak backend
13 | %%%
14 | %%% NOTE: This is work in progress, highly experimental.
15 | %%%
16 | %%% @end
17 | -module(kvdb_riak_proxy).
18 | -behaviour(gen_server).
19 |
20 | -export([start_link/2]).
21 |
22 | -export([get_session/1]).
23 |
24 | -export([init/1,
25 | handle_call/3,
26 | handle_cast/2,
27 | handle_info/2,
28 | terminate/2,
29 | code_change/3]).
30 |
31 | -record(st, {session}).
32 |
33 | start_link(Name, Options) ->
34 | gen_server:start_link(?MODULE, {Name, Options}, []).
35 |
36 | get_session(Name) ->
37 | case gproc:where({n,l,{?MODULE,Name}}) of
38 | undefined ->
39 | undefined;
40 | Pid ->
41 | gen_server:call(Pid, get_session)
42 | end.
43 |
44 | init({Name, Options}) ->
45 | gproc:reg({n,l,{?MODULE, Name}}),
46 | Session = open_session(Options),
47 | {ok, #st{session = Session}}.
48 |
49 | handle_call(get_session, _, #st{session = Session} = S) ->
50 | {reply, Session, S}.
51 |
52 | handle_cast(_, S) ->
53 | {noreply, S}.
54 |
55 | handle_info(_, S) ->
56 | {noreply, S}.
57 |
58 | terminate(_, _) ->
59 | ok.
60 |
61 | code_change(_, S, _) ->
62 | {ok, S}.
63 |
64 |
65 | open_session(Options) ->
66 | case proplists:get_value(riak, Options, {client, node()}) of
67 | {client, Node} ->
68 | case riak:client_connect(Node) of
69 | {ok, Pid} -> {client, Pid};
70 | _ -> undefined
71 | end;
72 | {pb, Conf} ->
73 | {Host, Port} =
74 | case Conf of
75 | {_,_} -> Conf;
76 | local ->
77 | case riak_api_pb_listener:get_listeners() of
78 | [{_, P}|_] ->
79 | {"127.0.0.1", P};
80 | _ -> error(cannot_get_pb_config)
81 | end
82 | end,
83 | case riakc_pb_socket:start_link(Host, Port) of
84 | {ok, Pid} ->
85 | {pb, Pid};
86 | _ ->
87 | undefined
88 | end
89 | end.
90 |
--------------------------------------------------------------------------------
/src/kvdb_schema.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% KVDB schema callback module behavior
13 | %%% @end
14 | -module(kvdb_schema).
15 |
16 | -export([
17 | validate/3, on_update/4
18 | ]).
19 | -export([write/2, read/1, read/2]).
20 | -export([pre_commit/2, post_commit/2]).
21 |
22 | -export([fold_schema/3, all_ok/3]).
23 |
24 | -export([behaviour_info/1]).
25 |
26 | -include("kvdb.hrl").
27 |
28 | behaviour_info(callbacks) ->
29 | [{validate, 3},
30 | {on_update, 4},
31 | {pre_commit, 2},
32 | {post_commit, 2}];
33 | behaviour_info(_) ->
34 | undefined.
35 |
36 | fold_schema(Schema, F, Obj) when is_function(F, 2) ->
37 | if is_atom(Schema) ->
38 | F(Schema, Obj);
39 | is_list(Schema) ->
40 | lists:foldl(
41 | fun(S, Acc) ->
42 | fold_schema(S, F, Acc)
43 | end, Schema, Obj);
44 | is_tuple(Schema), tuple_size(Schema) == 1 ->
45 | {M} = Schema,
46 | Expanded = M:expand_schema(),
47 | fold_schema(Expanded, F, Obj)
48 | end.
49 |
50 | all_ok(Schema, F, Obj) ->
51 | R = if is_atom(Schema) -> F(Schema, Obj);
52 | is_list(Schema) -> catch all_ok_(Schema, F, Obj);
53 | is_tuple(Schema), tuple_size(Schema) == 1 ->
54 | {M} = Schema,
55 | catch all_ok_([_|_] = M:expand_schema(), F, Obj)
56 | end,
57 | case R of
58 | {'EXIT', Reason} -> {error, Reason};
59 | Other -> Other
60 | end.
61 |
62 | all_ok_([H|T], F, Obj) ->
63 | case F(H, Obj) of
64 | ok -> all_ok_(T, F, Obj);
65 | Other -> Other
66 | end;
67 | all_ok_([], _, _) ->
68 | ok.
69 |
70 |
71 | validate(_Db, _Type, Obj) ->
72 | Obj.
73 |
74 | on_update(_Op, _Db, _Table, _Obj) ->
75 | ok.
76 |
77 | write(Db, Schema) ->
78 | [kvdb:put(Db, ?META_TABLE, X) || X <- Schema],
79 | ok.
80 |
81 | read(Db) ->
82 | match_(kvdb:select(Db, ?META_TABLE, [{'_',[],['$_']}], 100), []).
83 |
84 | read(Db, Item) ->
85 | case kvdb:get(Db, ?META_TABLE, Item) of
86 | {ok, {_,_,V}} -> {ok, V};
87 | {ok, {_, V}} -> {ok, V};
88 | Error ->
89 | Error
90 | end.
91 |
92 | match_({Objs, Cont}, Acc) ->
93 | match_(Cont(), acc_(Objs, Acc));
94 | match_(done, Acc) ->
95 | lists:reverse(Acc).
96 |
97 | acc_([{K,_,V}|T], Acc) ->
98 | acc_(T, [{K,V}|Acc]);
99 | acc_([{_,_}|_] = L, Acc) ->
100 | lists:reverse(L) ++ Acc;
101 | acc_([], Acc) ->
102 | Acc.
103 |
104 | pre_commit(C, _) ->
105 | C.
106 |
107 | post_commit(_, _) ->
108 | ok.
109 |
--------------------------------------------------------------------------------
/src/kvdb_schema_events.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @doc
12 | %%% KVDB schema callback module for queue-related events
13 | %%% @end
14 | %%%
15 | -module(kvdb_schema_events).
16 | -behaviour(kvdb_schema).
17 |
18 | -export([notify_when_not_empty/3,
19 | notify_all_queues/2,
20 | cancel_notify_all_queues/2]).
21 |
22 | -include("kvdb.hrl").
23 |
24 | %% kvdb_schema callbacks
25 | -export([validate/3,
26 | on_update/4,
27 | pre_commit/2,
28 | post_commit/2]).
29 |
30 | -include("log.hrl").
31 |
32 | notify_when_not_empty(#kvdb_ref{name = DBN} = Db, Table0, Q) ->
33 | Table = kvdb_lib:table_name(Table0),
34 | Evt = {kvdb, DBN, Table, Q, queue_status},
35 | gproc_ps:notify_single_if_true(
36 | l, Evt, fun() -> not kvdb:is_queue_empty(Db,Table,Q) end, not_empty).
37 |
38 | notify_all_queues(#kvdb_ref{name = DBN}, Table0) ->
39 | Table = kvdb_lib:table_name(Table0),
40 | Evt = {kvdb, DBN, Table, queue_status},
41 | gproc_ps:subscribe(l, Evt).
42 |
43 | cancel_notify_all_queues(#kvdb_ref{name = DBN}, Table0) ->
44 | gproc_ps:unsubscribe(l, {kvdb, DBN, kvdb_lib:table_name(Table0),
45 | queue_status}).
46 |
47 | validate(_, _, Obj) ->
48 | Obj.
49 |
50 | on_update({q_op,_,Q,true}, DB, Table, _) ->
51 | notify_queue_status(DB, Table, Q, empty),
52 | ok;
53 | on_update({q_op,_,Q,false}, DB, Table, _) ->
54 | notify_queue_status(DB, Table, Q, not_empty),
55 | ok;
56 | on_update(_, _, _, _) ->
57 | ok.
58 |
59 | pre_commit(C, _) ->
60 | C.
61 |
62 | post_commit(_, _) ->
63 | ok.
64 |
65 | notify_queue_status(#kvdb_ref{name = DBN, db = #db{metadata = Ets}} = Ref,
66 | Table, Q, Status) ->
67 | ?debug("notify_queue_statusf(Ref = ~p, ~p, ~p, ~p)~n",
68 | [Ref,Table,Q,Status]),
69 | case set_status(Ets, Table, Q, Status) of
70 | changed ->
71 | _ = gproc_ps:publish(l, {kvdb, DBN, Table, queue_status},
72 | {Q, Status}),
73 | _ = gproc_ps:tell_singles(l, {kvdb, DBN, Table, Q, queue_status},
74 | Status),
75 | ok;
76 | same ->
77 | ok
78 | end.
79 |
80 | set_status(Ets, Tab, Q, Status) ->
81 | Key = {q_status,Tab,Q},
82 | Change = case Status of
83 | empty -> [{2,0},{2,-1,0,0}];
84 | not_empty -> [{2,0},{2,1,1,1}]
85 | end,
86 | try ets:update_counter(Ets, Key, Change) of
87 | [X,X] ->
88 | same;
89 | [_,_] ->
90 | changed
91 | catch
92 | error:_ ->
93 | ets:insert(Ets, {Key, case Status of
94 | empty -> 0;
95 | not_empty -> 1
96 | end}),
97 | changed
98 | end.
99 |
--------------------------------------------------------------------------------
/src/kvdb_server.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%% @doc
13 | %%% KVDB Database instance owner process
14 | %%% @end
15 | %%%
16 | -module(kvdb_server).
17 | -behaviour(gen_server).
18 |
19 | -export([start_link/2,
20 | start_session/2,
21 | db/1, db/2,
22 | await/1, await/2,
23 | close/1,
24 | call/2,
25 | cast/2]).
26 |
27 | -export([begin_trans/3,
28 | start_commit/2,
29 | end_trans/2]).
30 |
31 | -export([init/1,
32 | handle_call/3,
33 | handle_cast/2,
34 | handle_info/2,
35 | terminate/2,
36 | code_change/3]).
37 |
38 | -include("kvdb.hrl").
39 | -include("log.hrl").
40 | -import(kvdb_lib, [table_name/1]).
41 |
42 | -record(st, {name, db, is_owner = false,
43 | switch_pending = false,
44 | transactions = [],
45 | commits = [],
46 | pending_commits = []}).
47 |
48 | -record(trans, {pid, mref, tref, role = master, db}).
49 |
50 | start_link(Name, Backend) ->
51 | gen_server:start_link(?MODULE, {owner, Name, Backend}, []).
52 |
53 | start_session(Name, Id) ->
54 | gen_server:start_link(?MODULE, session(Name, Id), []).
55 |
56 | session(Name, Id) ->
57 | {Name, session, Id}.
58 |
59 |
60 | db(Name) ->
61 | call(Name, db).
62 |
63 | db(Name, TRef) ->
64 | call(Name, {db, TRef}).
65 |
66 | begin_trans(Name, TRef, Db) ->
67 | call(Name, {begin_trans, TRef, Db}).
68 |
69 | start_commit(Name, TRef) ->
70 | try call(Name, {start_commit, TRef}) of
71 | #kvdb_ref{} = Res ->
72 | Res;
73 | Other ->
74 | erlang:error(Other)
75 | catch
76 | exit:Err ->
77 | io:fwrite(user, "kvdb_server status:~n~s~n",
78 | [print_state(Name)]),
79 | exit(Err)
80 | end.
81 |
82 | print_state(Name) ->
83 | {status,_,_,[_,_,_,_,[_,_,{data,[{"State",S}]}]]} =
84 | sys:get_status(get_pid(Name)),
85 | Sz = tuple_size(S) -1,
86 | RF = fun(st, Size) when Size == Sz ->
87 | record_info(fields, st);
88 | (_, _) ->
89 | no
90 | end,
91 | io_lib_pretty:print(S, RF).
92 |
93 | end_trans(Name, Ref) ->
94 | cast(Name, {end_trans, self(), Ref}).
95 |
96 | close(Name) ->
97 | call(Name, close).
98 |
99 | await(Name) ->
100 | await(Name, timer:minutes(1)).
101 |
102 | await(Name, Timeout) ->
103 | gproc:await({n,l,{kvdb,Name}}, Timeout),
104 | ok.
105 |
106 | cast(Name, Msg) ->
107 | gen_server:cast(gproc:where({n,l,{kvdb,Name}}), Msg).
108 |
109 | call(Name, Req) ->
110 | call(Name, Req, 5000).
111 |
112 | call(Name, Req, Timeout) ->
113 | Pid = get_pid(Name),
114 | case gen_server:call(Pid, Req, Timeout) of
115 | badarg ->
116 | ?KVDB_THROW(badarg);
117 | {badarg,_} = Err ->
118 | ?KVDB_THROW(Err);
119 | Res ->
120 | Res
121 | end.
122 |
123 | get_pid(Name) ->
124 | case Name of
125 | #kvdb_ref{name = N} ->
126 | gproc:where({n, l, {kvdb, N}});
127 | P when is_pid(P) ->
128 | P;
129 | _ ->
130 | gproc:where({n,l,{kvdb,Name}})
131 | end.
132 |
133 |
134 | %% @private
135 | init(Alias) ->
136 | process_flag(trap_exit, true),
137 | try init_(Alias)
138 | catch
139 | error:Reason ->
140 | Trace = erlang:get_stacktrace(),
141 | error_logger:error_report([{error_opening_kvdb_db, Alias},
142 | {error, Reason},
143 | {stacktrace, Trace}]),
144 | erlang:error({Reason, Trace}, [Alias])
145 | end.
146 |
147 | init_({Name, session, _Id} = Alias) ->
148 | Db = db(Name),
149 | gproc:reg({p, l, {kvdb, session}}, Alias),
150 | gproc:reg({n, l, {kvdb, Alias}}),
151 | {ok, #st{db = Db}};
152 | init_({owner, Name, Opts}) ->
153 | Backend = proplists:get_value(backend, Opts, ets),
154 | gproc:reg({n, l, {kvdb,Name}}, Backend),
155 | DbMod = mod(Backend),
156 | NewOpts = check_log_options(
157 | Name,
158 | lists:keystore(backend, 1,
159 | %% lists:keystore(file, 1, Opts, {file, File}),
160 | Opts,
161 | {backend, DbMod})),
162 | case do_open(Name, NewOpts) of
163 | {ok, Db} ->
164 | kvdb_cron:init_meta(Db),
165 | create_tables_(Db, Opts),
166 | case common_open(Db, NewOpts) of
167 | {ok, Db1} ->
168 | {ok, #st{name = Name, db = Db1, is_owner = true}};
169 | {error,_} = CommonOpenErr ->
170 | CommonOpenErr
171 | end;
172 | {error,_} = Error ->
173 | io:fwrite("error opening kvdb database ~w:~n"
174 | "Error: ~p~n"
175 | "Opts = ~p~n", [Name, Error, NewOpts]),
176 | Error
177 | end.
178 |
179 | check_log_options(Name, Opts) ->
180 | case {lists:keymember(log_dir, 1, Opts),
181 | lists:keymember(log_threshold, 1, Opts)} of
182 | {false, true} ->
183 | {File, Opts1} = ensure_file(Name, Opts),
184 | [{log_dir, File ++ ".log"} | Opts1];
185 | _ ->
186 | Opts
187 | end.
188 |
189 | ensure_file(Name, Opts) ->
190 | case lists:keyfind(file, 1, Opts) of
191 | {_, F} ->
192 | {F, Opts};
193 | false ->
194 | F = kvdb_lib:db_file(Name),
195 | {F, [{file, F}|Opts]}
196 | end.
197 |
198 | %% @private
199 | handle_call(Req, From, St) ->
200 | try handle_call_(Req, From, St)
201 | catch
202 | error:badarg ->
203 | {reply, {badarg, erlang:get_stacktrace()}, St};
204 | error:E ->
205 | {reply, {badarg,[E, erlang:get_stacktrace()]}, St}
206 | end.
207 |
208 | handle_call_({put, Tab, Obj}, _From, #st{db = Db} = St) ->
209 | {reply, kvdb_direct:put(Db, Tab, Obj), St};
210 | handle_call_({update_counter, Table, Key, Incr}, _From, #st{db = Db} = St) ->
211 | {reply, kvdb_direct:update_counter(Db, Table, Key, Incr), St};
212 | handle_call_({push, Tab, Q, Obj}, _From, #st{db = Db} = St) ->
213 | {reply, kvdb_direct:push(Db, Tab, Q, Obj), St};
214 | handle_call_({pop, Tab, Q}, _From, #st{db = Db} = St) ->
215 | {reply, kvdb_direct:pop(Db, Tab, Q), St};
216 | handle_call_({prel_pop, Tab, Q}, _From, #st{db = Db} = St) ->
217 | {reply, kvdb_direct:prel_pop(Db, Tab, Q), St};
218 | handle_call_({extract, Tab, Key}, _From, #st{db = Db} = St) ->
219 | {reply, kvdb_direct:extract(Db, Tab, Key), St};
220 | handle_call_({mark_queue_object, Table, Key, OSt}, _From, #st{db = Db} = St) ->
221 | {reply, kvdb_direct:mark_queue_object(Db, Table, Key, OSt), St};
222 | handle_call_({delete, Tab, Key}, _From, #st{db = Db} = St) ->
223 | {reply, kvdb_direct:delete(Db, Tab, Key), St};
224 | handle_call_({add_table, Table, Opts}, _From, #st{db = Db} = St) ->
225 | {reply, kvdb_direct:add_table(Db, Table, Opts), St};
226 | handle_call_({delete_table, Table}, _From, #st{db = Db} = St) ->
227 | {reply, kvdb_direct:delete_table(Db, Table), St};
228 | handle_call_(close, _From, #st{is_owner = true} = St) ->
229 | {stop, normal, ok, St};
230 | handle_call_({begin_trans, Ref, DbT}, {Pid,_}, #st{transactions = Ts} = St) ->
231 | lager:debug("begin_trans, Ref = ~p, Pid = ~p", [Ref, Pid]),
232 | MRef = erlang:monitor(process, Pid),
233 | Ts1 = [#trans{pid = Pid, tref = Ref, mref = MRef, db = DbT}|Ts],
234 | {reply, ok, St#st{transactions = Ts1}};
235 | handle_call_({start_commit, Ref}, {Pid,_} = From, #st{switch_pending = SwPend,
236 | transactions = Ts,
237 | db = Db} = St) ->
238 | case is_transaction_master(Pid, Ref, Ts) of
239 | yes ->
240 | case SwPend of false ->
241 | {reply, Db, St#st{commits = [Ref|St#st.commits]}};
242 | {true,_,_} ->
243 | Pend = St#st.pending_commits,
244 | {noreply, St#st{pending_commits = [From|Pend]}}
245 | end;
246 | {no, Reason} ->
247 | {reply, {error, Reason}, St}
248 | end;
249 | handle_call_({new_log, Log}, From, #st{db = Db, commits = Commits} = St) ->
250 | %% Need to switch logs. We try to do this atomically. For individual
251 | %% updates, it's no problem, since they will call on us for the db ref.
252 | %% If there are ongoing transactions, we must wait for them to end,
253 | %% queueing new transaction requests in the meantime.
254 | case Commits of
255 | [] ->
256 | NewDb = switch_logs(Db, Log),
257 | {reply, {ok, NewDb}, logs_switched(NewDb, St)};
258 | [_|_] ->
259 | %% io:fwrite("won't switch now; Ts = ~p~n", [Ts]),
260 | {noreply, St#st{switch_pending = {true, From, Log}}}
261 | end;
262 | handle_call_(db, _From, #st{db = Db} = St) ->
263 | {reply, Db, St}.
264 |
265 | handle_info({'DOWN', Ref, _,_,_}, #st{transactions = Ts,
266 | commits = Commits,
267 | switch_pending = Pend} = St) ->
268 | case lists:keytake(Ref, #trans.mref, Ts) of
269 | {value, #trans{tref = TRef, role = master}, Ts1} ->
270 | case Commits -- [TRef] of
271 | [] ->
272 | case Pend of
273 | {true, _From, Log} ->
274 | NewDb = switch_logs(St#st.db, Log),
275 | {noreply, logs_switched(
276 | NewDb, St#st{transactions = Ts1,
277 | commits = []})};
278 | false ->
279 | {noreply, St#st{transactions = Ts1,
280 | commits = []}}
281 | end;
282 | Commits1 ->
283 | {noreply, St#st{transactions = Ts1,
284 | commits = Commits1}}
285 | end;
286 | {value, _, Ts1} ->
287 | {noreply, St#st{transactions = Ts1}};
288 | false ->
289 | {noreply, St}
290 | end;
291 | handle_info(_, St) ->
292 | {noreply, St}.
293 |
294 | %% @private
295 | handle_cast({end_trans, _Pid, Ref} = _M, #st{db = Db,
296 | commits = Commits,
297 | transactions = Ts} = St) ->
298 | lager:debug("~p", [_M]),
299 | Ts1 = lists:foldr(
300 | fun(#trans{tref = R, mref = MRef}, Acc) when R =:= Ref ->
301 | erlang:demonitor(MRef),
302 | Acc;
303 | (T, Acc) ->
304 | [T|Acc]
305 | end, [], Ts),
306 | case Commits1 = Commits -- [Ref] of
307 | [] ->
308 | case St#st.switch_pending of
309 | false ->
310 | {noreply, St#st{transactions = Ts1, commits = []}};
311 | {true, From, Log} ->
312 | NewDb = switch_logs(Db, Log),
313 | gen_server:reply(From, {ok, NewDb}),
314 | {noreply, logs_switched(NewDb, St#st{transactions = Ts1,
315 | commits = []})}
316 | end;
317 | [_|_] = Commits1 ->
318 | {noreply, St#st{transactions = Ts1,
319 | commits = Commits1}}
320 | end;
321 | handle_cast(log_threshold, #st{db = Db} = St) ->
322 | %% io:fwrite("threshold reached, ~p~n", [Db]),
323 | Me = self(),
324 | spawn_monitor(fun() ->
325 | %% io:fwrite("~p spawned to switch logs~n", [self()]),
326 | {ok, Log} = open_new_log(Db, Me),
327 | %% io:fwrite("new log opened: ~p~n", [Log]),
328 | TS = os:timestamp(),
329 | {ok, NewDb} = call(Me, {new_log, Log}, 15000),
330 | %% io:fwrite("NewDb = ~p~n", [NewDb]),
331 | #kvdb_ref{db = D} = NewDb,
332 | case kvdb_meta:read(D, last_dump, undefined) of
333 | DumpTS when DumpTS > TS ->
334 | kvdb_lib:purge_logs(D, DumpTS);
335 | %% io:fwrite("logs purged~n", []);
336 | _ ->
337 | %% io:fwrite("nothing to purge~n", []),
338 | ignore
339 | end
340 | end),
341 | {noreply, St};
342 | handle_cast(_, St) ->
343 | {noreply, St}.
344 |
345 | %% @private
346 | terminate(_Reason, #st{db = #kvdb_ref{mod = M, db = Db}} = S) ->
347 | M:close(Db),
348 | close_log(S),
349 | ok.
350 |
351 | %% @private
352 | code_change(_FromVsn, St, _Extra) ->
353 | {ok, St}.
354 |
355 | is_transaction_master(Pid, Ref, Ts) ->
356 | case [T || #trans{pid = P, tref = R} = T <- Ts,
357 | P =:= Pid, R =:= Ref] of
358 | [#trans{role = master}] ->
359 | yes;
360 | [_] ->
361 | {no, wrong_pid};
362 | [] ->
363 | {no, not_found}
364 | end.
365 |
366 | switch_logs(#kvdb_ref{mod = M, db = #db{log = {OldLog,_}} = Db} = Ref, Log) ->
367 | #db{} = NewDb = M:switch_logs(Db, Log),
368 | disk_log:close(OldLog),
369 | Ref#kvdb_ref{db = NewDb}.
370 |
371 | close_log(#st{db = #db{log = {Log, _}}}) ->
372 | disk_log:close(Log);
373 | close_log(_) ->
374 | ok.
375 |
376 | logs_switched(NewDb, St) ->
377 | St1 = case St#st.pending_commits of
378 | [] ->
379 | St#st{db = NewDb,
380 | switch_pending = false};
381 | Pend ->
382 | lists:foreach(fun(From) ->
383 | gen_server:reply(From, NewDb)
384 | end, Pend),
385 | St#st{
386 | db = NewDb,
387 | pending_commits = [],
388 | switch_pending = false}
389 | end,
390 | %% io:fwrite("logs switched; new st= ~p~n", [St1]),
391 | St1.
392 |
393 |
394 | open_new_log(#kvdb_ref{db = Db}, Pid) ->
395 | LogDir = kvdb_meta:read(Db, log_dir, undefined),
396 | kvdb_lib:open_log(kvdb_lib:log_filename(LogDir), Pid).
397 |
398 |
399 | do_open(Name, Options) when is_list(Options) ->
400 | DbMod = proplists:get_value(backend, Options, kvdb_sqlite3),
401 | Fallback = check_for_fallback(Name, Options),
402 | case DbMod:open(Name,Options) of
403 | {ok, Db} ->
404 | %% io:fwrite("opened ~p database: ~p~n", [DbMod, Options]),
405 | Default = DbMod:get_schema_mod(Db, kvdb_schema),
406 | Schema = proplists:get_value(schema, Options, Default),
407 | kvdb_meta:write(Db, name, Name),
408 | maybe_import(Fallback, DbMod, Db),
409 | {ok, #kvdb_ref{name = Name, mod = DbMod, db = Db, schema = Schema}};
410 | Error ->
411 | io:fwrite("ERROR opening ~p database: ~p. Opts = ~p~n",
412 | [DbMod, Error, Options]),
413 | Error
414 | end.
415 |
416 | maybe_import(false, _, _) ->
417 | ok;
418 | maybe_import(Fallback, Mod, Db) ->
419 | io:fwrite("Importing from ~p~n", [Fallback]),
420 | kvdb_export:import(Mod, Db, Fallback).
421 |
422 |
423 | check_for_fallback(Name, Options) ->
424 | File = case lists:keyfind(file, 1, Options) of
425 | {_, F} -> F;
426 | false -> kvdb_lib:db_file(Name)
427 | end,
428 | io:fwrite("File = ~p~n", [File]),
429 | Dir = filename:dirname(File),
430 | lager:debug("check_for_fallback: Dir = ~p~n", [Dir]),
431 | case filelib:wildcard(to_list(Name) ++ ".KBUP?", Dir) of
432 | [] ->
433 | false;
434 | [_|_] = Files ->
435 | Sorted = lists:sort(fun fcomp/2, Files),
436 | lager:debug("Found backups: ~p~n", [Sorted]),
437 | Pick = hd(Sorted),
438 | move_file(Options),
439 | filename:join(Dir, Pick)
440 | end.
441 |
442 | to_list(A) when is_atom(A) -> atom_to_list(A);
443 | to_list(L) when is_list(L) -> L;
444 | to_list(B) when is_binary(B) -> binary_to_list(B).
445 |
446 |
447 | move_file(Options) ->
448 | case lists:keyfind(file, 1, Options) of
449 | {_, F} ->
450 | BakName = bak_name(F),
451 | file:rename(F, BakName);
452 | false ->
453 | ok
454 | end.
455 |
456 | bak_name(F) ->
457 | bak_name(F, 0).
458 |
459 | bak_name(F, N) ->
460 | case file:read_link_info(Bak = F ++ bak_ext(N)) of
461 | {ok,_} -> bak_name(F, N+1);
462 | {error,enoent} -> Bak
463 | end.
464 |
465 | bak_ext(0) -> ".bak";
466 | bak_ext(N) -> ".bak." ++ integer_to_list(N).
467 |
468 |
469 | fcomp(A, B) ->
470 | case {filename:extension(A), filename:extension(B)} of
471 | {".KBUPB", _} -> true;
472 | {_, ".KBUPB"} -> false;
473 | {_, _} -> true % just pick one
474 | end.
475 |
476 | common_open(#kvdb_ref{mod = DbMod, db = Db} = DbRef, Options) ->
477 | case kvdb_lib:common_open(DbMod, Db, Options) of
478 | {ok, Db1} ->
479 | {ok, DbRef#kvdb_ref{db = Db1}};
480 | Other ->
481 | Other
482 | end.
483 |
484 |
485 | mod(mnesia ) -> kvdb_mnesia;
486 | mod(leveldb) -> kvdb_leveldb;
487 | mod(sqlite3) -> kvdb_sqlite3;
488 | mod(sqlite ) -> kvdb_sqlite3;
489 | mod(ets ) -> kvdb_ets;
490 | mod(M) ->
491 | case is_behaviour(M) of
492 | true ->
493 | M % TODO: implement actual check
494 | %% false ->
495 | %% error(illegal_backend_type)
496 | end.
497 |
498 | %% name2file(X) ->
499 | %% kvdb_lib:good_string(X).
500 |
501 |
502 |
503 | %% to_atom(A) when is_atom(A) ->
504 | %% A;
505 | %% to_atom(S) when is_list(S) ->
506 | %% list_to_atom(S).
507 |
508 |
509 | is_behaviour(_M) ->
510 | %% TODO: check that exported functions match those listed in
511 | %% behaviour_info(callbacks).
512 | true.
513 |
514 | create_tables_(Db, Opts) ->
515 | case proplists:get_value(tables, Opts, []) of
516 | [] ->
517 | ok;
518 | Ts ->
519 | Tabs0 = lists:map(fun({T,Os}) ->
520 | {table_name(T), Os};
521 | (T) -> {table_name(T),[]}
522 | end, Ts),
523 | %% We don't warn if there are more tables than we've specified,
524 | %% and we certainly don't remove them. Ok to do nothing?
525 | Tables = internal_tables() ++ Tabs0,
526 | Existing = kvdb:list_tables(Db),
527 | New = lists:filter(fun({T,_}) ->
528 | not lists:member(T, Existing) end,
529 | Tables),
530 | [kvdb_direct:add_table(Db, T, Os) || {T, Os} <- New]
531 | end.
532 |
533 | internal_tables() ->
534 | [].
535 |
--------------------------------------------------------------------------------
/src/kvdb_server_sup.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%% @doc
13 | %%% KVDB database instance supervisor
14 | %%% @end
15 | %%%
16 | -module(kvdb_server_sup).
17 |
18 | -behaviour(supervisor).
19 |
20 | %% API
21 | -export([start_link/0]).
22 | -export([start_child/2]).
23 | %% childspec/1]).
24 |
25 | %% Supervisor callbacks
26 | -export([init/1]).
27 |
28 | %% Helper macro for declaring children of supervisor
29 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
30 |
31 | %% ===================================================================
32 | %% API functions
33 | %% ===================================================================
34 |
35 | start_link() ->
36 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
37 |
38 | %% ===================================================================
39 | %% Supervisor callbacks
40 | %% ===================================================================
41 |
42 | init([]) ->
43 | %% Children = childspecs(DBs = get_databases()),
44 | %% io:fwrite("DBs = ~p~n", [DBs]),
45 | {ok, { {simple_one_for_one, 5, 10},
46 | [{id, {kvdb_server, start_link, []},
47 | transient, 5000, worker, [kvdb_server]}] }}.
48 |
49 |
50 | start_child(Name, Opts) ->
51 | supervisor:start_child(?MODULE, [Name, Opts]).
52 |
53 |
--------------------------------------------------------------------------------
/src/kvdb_sup.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%% @hidden
12 | %%% @doc
13 | %%% KVDB main supervisor
14 | %%% @end
15 | %%%
16 | -module(kvdb_sup).
17 |
18 | -behaviour(supervisor).
19 |
20 | %% API
21 | -export([start_link/0]).
22 | -export([start_child/2]).
23 | %% childspec/1]).
24 |
25 | %% Supervisor callbacks
26 | -export([init/1]).
27 |
28 | %% Helper macro for declaring children of supervisor
29 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
30 |
31 | %% ===================================================================
32 | %% API functions
33 | %% ===================================================================
34 |
35 | start_link() ->
36 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
37 |
38 | %% ===================================================================
39 | %% Supervisor callbacks
40 | %% ===================================================================
41 |
42 | init([]) ->
43 | %% Children = childspecs(DBs = get_databases()),
44 | %% io:fwrite("DBs = ~p~n", [DBs]),
45 | {ok, { {simple_one_for_one, 0, 10},
46 | [{id, {kvdb_db_sup, start_link, []},
47 | permanent, infinity, supervisor, [kvdb_db_sup]}] }}.
48 | %% [{id, {kvdb_server, start_link, []},
49 | %% transient, 5000, worker, [kvdb_server]}] }}.
50 |
51 |
52 | start_child(Name, Opts) ->
53 | supervisor:start_child(?MODULE, [Name, Opts]).
54 |
55 |
--------------------------------------------------------------------------------
/src/log.hrl:
--------------------------------------------------------------------------------
1 | %%
2 | %% Log macros
3 | %%
4 | -ifndef(__LOG_HRL__).
5 | -define(__LOG_HRL__, true).
6 |
7 | -compile({parse_transform, lager_transform}).
8 |
9 | %% Lager logging levels
10 | %% debug, info, notice, warning, error, critical, alert, emergency, none.
11 |
12 | -define(debug(Fmt), lager:debug(Fmt)).
13 | -define(debug(Fmt, Args), lager:debug(Fmt, Args)).
14 | -define(debug(Attrs, Fmt, Args), lager:debug(Attrs, Fmt, Args)).
15 |
16 | -define(info(Fmt), lager:info(Fmt)).
17 | -define(info(Fmt, Args), lager:info(Fmt, Args)).
18 | -define(info(Attrs, Fmt, Args), lager:info(Attrs, Fmt, Args)).
19 |
20 | -define(notice(Fmt), lager:notice(Fmt)).
21 | -define(notice(Fmt, Args), lager:notice(Fmt, Args)).
22 | -define(notice(Attrs, Fmt, Args), lager:notice(Attrs, Fmt, Args)).
23 |
24 | -define(warning(Fmt), lager:warning(Fmt)).
25 | -define(warning(Fmt, Args), lager:warning(Fmt, Args)).
26 | -define(warning(Attrs, Fmt, Args), lager:warning(Attrs, Fmt, Args)).
27 |
28 | -define(error(Fmt), lager:error(Fmt)).
29 | -define(error(Fmt, Args), lager:error(Fmt, Args)).
30 | -define(error(Attrs, Fmt, Args), lager:error(Attrs, Fmt, Args)).
31 |
32 | -define(critical(Fmt), lager:critical(Fmt)).
33 | -define(critical(Fmt, Args), lager:critical(Fmt, Args)).
34 | -define(critical(Attrs, Fmt, Args), lager:critical(Attrs, Fmt, Args)).
35 |
36 | -define(alert(Fmt), lager:alert(Fmt)).
37 | -define(alert(Fmt, Args), lager:alert(Fmt, Args)).
38 | -define(alert(Attrs, Fmt, Args), lager:alert(Attrs, Fmt, Args)).
39 |
40 | -define(emergency(Fmt), lager:emergency(Fmt)).
41 | -define(emergency(Fmt, Args), lager:emergency(Fmt, Args)).
42 | -define(emergency(Attrs, Fmt, Args), lager:emergency(Attrs, Fmt, Args)).
43 |
44 | -endif.
45 |
--------------------------------------------------------------------------------
/test.config:
--------------------------------------------------------------------------------
1 | %% -*- erlang -*-
2 | [{kvdb, [
3 | {databases, [
4 | {s, [{backend, sqlite},
5 | {tables, [t]}
6 | ]},
7 | {l, [{backend, leveldb},
8 | {tables, [t]}
9 | ]}
10 | ]}
11 | ]}
12 | ].
13 |
--------------------------------------------------------------------------------
/test/feuerlabs_eunit.hrl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | %%%
12 | %%% Helpful macros for EUnit tests
13 |
14 | %% ?_t(E): Like ?test(E), but with a 1-minute timeout, and generates a stack
15 | %% trace after exceptions.
16 | -define(_t(E), {timeout,60000,
17 | [?_test(try E catch error:_R_ ->
18 | error({_R_, erlang:get_stacktrace()})
19 | end)]}).
20 |
21 |
22 | %% ?dbg(E): executes test case, printing the result on success, and a
23 | %% thorough error report on failure.
24 | -define(dbg(E),
25 | (fun() ->
26 | try (E) of
27 | __V ->
28 | ?debugFmt(<<"~s = ~P">>, [(??E), __V, 15]),
29 | __V
30 | catch
31 | error:__Err ->
32 | io:fwrite(user,
33 | "FAIL: test = ~s~n"
34 | "Error = ~p~n"
35 | "Trace = ~p~n", [(??E), __Err,
36 | erlang:get_stacktrace()]),
37 | error(__Err)
38 | end
39 | end)()).
40 |
41 | %% ?trace(Mods, E) Enables a call trace on the modules (or {Mod,Fun} patterns)
42 | %% in Mods, evaluates expression E and then stops the trace.
43 | -define(trace(Mods, Expr), begin dbg:tracer(),
44 | lists:foreach(
45 | fun(_M_) when is_atom(_M_) ->
46 | dbg:tpl(_M_,x);
47 | ({_M_,_F_}) ->
48 | dbg:tpl(_M_,_F_,x)
49 | end, Mods),
50 | dbg:p(all,[c]),
51 | try Expr
52 | after
53 | lists:foreach(
54 | fun(_M_) when is_atom(_M_) ->
55 | dbg:ctpl(_M_);
56 | ({_M_,_F_}) ->
57 | dbg:ctpl(_M_,_F_)
58 | end, Mods),
59 | dbg:stop()
60 | end
61 | end).
62 |
--------------------------------------------------------------------------------
/test/kvdb_conf_tests.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | -module(kvdb_conf_tests).
12 |
13 | -ifdef(TEST).
14 |
15 | -include_lib("eunit/include/eunit.hrl").
16 | -include("feuerlabs_eunit.hrl").
17 |
18 | -define(tab, iolist_to_binary([<<"t_">>, integer_to_list(?LINE)])).
19 | -define(mktab(T, Os), T = ?tab, ok = kvdb_conf:add_table(T, Os)).
20 | -define(my_t(E), ?_t(?dbg(E))).
21 |
22 |
23 | conf_test_() ->
24 | [
25 | ?my_t(join_split_key()),
26 | {setup,
27 | fun() ->
28 | application:start(gproc),
29 | application:start(kvdb),
30 | kvdb_conf:open(undefined, [{backend, ets}]),
31 | ok
32 | end,
33 | fun(_) ->
34 | application:stop(kvdb),
35 | application:stop(gproc)
36 | end,
37 | [?my_t(read_write())
38 | , ?my_t(prefix_match())
39 | , ?my_t(next_at_level())
40 | , ?my_t(conf_tree())
41 | , ?my_t(write_tree())
42 | , ?my_t(first_next_child())
43 | , ?my_t(fold_list())
44 | , ?my_t(fold_children())
45 | ]}].
46 |
47 | join_split_key() ->
48 | <<"=a*=b">> = kvdb_conf:join_key(<<"a">>, <<"b">>),
49 | <<"=a*">> = kvdb_conf:join_key(<<"a">>, <<>>),
50 | <<"=a*=b">> = kvdb_conf:join_key([<<"a">>, <<"b">>]),
51 | <<"=a*">> = kvdb_conf:join_key([<<"a">>, <<>>]),
52 | ok.
53 |
54 |
55 | read_write() ->
56 | ?mktab(T, []),
57 | ok = kvdb_conf:write(T, {<<"a">>, [{a,1}], <<"1">>}),
58 | {ok, {<<"a">>, [{a,1}], <<"1">>}} = kvdb_conf:read(T, <<"a">>),
59 | ok = kvdb_conf:delete(T, <<"a">>),
60 | {error, not_found} = kvdb_conf:read(T, <<"a">>),
61 | ok = kvdb_conf:delete_table(T).
62 |
63 | prefix_match() ->
64 | ?mktab(T, []),
65 | ok = kvdb_conf:write(T, {<<"aabbcc">>, [], <<>>}),
66 | ok = kvdb_conf:write(T, {<<"aabbdd">>, [], <<>>}),
67 | ok = kvdb_conf:write(T, {<<"aaccdd">>, [], <<>>}),
68 | {[{<<"aabbcc">>,[],<<>>},{<<"aabbdd">>,[],<<>>}],_} =
69 | kvdb_conf:prefix_match(T, <<"aabb">>),
70 | ok = kvdb_conf:delete_table(T).
71 |
72 | next_at_level() ->
73 | ?mktab(T, []),
74 | ok = kvdb_conf:write(T, {<<"aa">>, [], <<>>}),
75 | ok = kvdb_conf:write(T, {<<"aa*bb">>, [], <<>>}),
76 | ok = kvdb_conf:write(T, {<<"aa*bb*cc">>, [], <<>>}),
77 | ok = kvdb_conf:write(T, {<<"aa*bb*dd">>, [], <<>>}),
78 | ok = kvdb_conf:write(T, {<<"aa*cc">>, [], <<>>}),
79 | ok = kvdb_conf:write(T, {<<"aa*cc*cc">>, [], <<>>}),
80 | ok = kvdb_conf:write(T, {<<"aa*cc*dd">>, [], <<>>}),
81 | ok = kvdb_conf:write(T, {<<"aa*dd*dd">>, [], <<>>}),
82 | ok = kvdb_conf:write(T, {<<"bb">>, [], <<>>}),
83 | {ok, {<<"aa">>,_,_}} = kvdb_conf:first(T),
84 | {ok, <<"bb">>} = kvdb_conf:next_at_level(T, <<"aa">>),
85 | {ok, <<"aa*cc">>} = kvdb_conf:next_at_level(T, <<"aa*bb">>),
86 | {ok, <<"aa*bb*cc">>} = kvdb_conf:next_at_level(T, <<"aa*bb*">>),
87 | %% Next - find an implicit node (its children are present, so therefore, it
88 | %% exists conceptually, even if not physically present).
89 | {ok, <<"aa*dd">>} = kvdb_conf:next_at_level(T, <<"aa*cc">>),
90 | done = kvdb_conf:next_at_level(T, <<"aa*cc*dd">>),
91 | {ok, <<"aa*bb*dd">>} = kvdb_conf:next_at_level(T, <<"aa*bb*cc">>),
92 | done = kvdb_conf:next_at_level(T, <<"aa*bb*dd">>),
93 | ok = kvdb_conf:delete_table(T).
94 |
95 | first_next_tree() ->
96 | ?mktab(T, []),
97 | ok = kvdb_conf:write(T, {<<"a*b*a">>, [], <<>>}),
98 | ok = kvdb_conf:write(T, {<<"a*b*c">>, [], <<>>}),
99 | ok = kvdb_conf:write(T, {<<"a*c*a">>, [], <<>>}),
100 | ok = kvdb_conf:write(T, {<<"b*b*a">>, [], <<>>}),
101 | ok = kvdb_conf:write(T, {<<"b*b*c">>, [], <<>>}),
102 | {conf_tree, <<"a*b">>, [{<<"a">>, [], <<>>}]} = First =
103 | kvdb_conf:first_tree(T),
104 | {conf_tree, <<"b*b">>, [{<<"a">>, [], <<>>}]} =
105 | kvdb_conf:first_tree(T, First),
106 | {conf_tree, <<"b*b">>, [{<<"c">>, [], <<>>}]} =
107 | kvdb_conf:last_tree(T),
108 | ok = kvdb_conf:delete_table(T).
109 |
110 |
111 | conf_tree() ->
112 | ?mktab(T, []),
113 | ok = kvdb_conf:write(
114 | T,
115 | {kvdb_conf:join_key(<<"a">>, <<"b">>), [{b,1}],<<"1">>}),
116 | ok = kvdb_conf:write(
117 | T,
118 | {kvdb_conf:join_key(<<"a">>, <<"c">>), [{c,1}],<<"2">>}),
119 | ok = kvdb_conf:write(
120 | T,
121 | {kvdb_conf:join_key(<<"aa">>, <<"d">>), [{d,1}],<<"3">>}),
122 | [{<<"a*b">>,[{b,1}],<<"1">>},
123 | {<<"a*c">>,[{c,1}],<<"2">>},
124 | {<<"aa*d">>,[{d,1}],<<"3">>}] = kvdb_conf:all(T),
125 | {conf_tree, <<"a">>, [{<<"b">>, [{b,1}], <<"1">>},
126 | {<<"c">>, [{c,1}], <<"2">>}]} =
127 | kvdb_conf:read_tree(T, <<"a">>),
128 | {conf_tree, <<"aa">>, [{<<"d">>, [{d,1}], <<"3">>}]} =
129 | kvdb_conf:read_tree(T, <<"aa">>),
130 | ok = kvdb_conf:write(T, {<<"a">>, [], <<>>}),
131 | %% <<"a">> is a separate object, so must be part of the tree. It cannot
132 | %% be lifted into the root key.
133 | {conf_tree, <<>>, [{<<"a">>, [], <<>>,
134 | [{<<"b">>, [{b,1}], <<"1">>},
135 | {<<"c">>, [{c,1}], <<"2">>}]}]} =
136 | kvdb_conf:first_tree(T),
137 | ok = kvdb_conf:delete_table(T).
138 |
139 | write_tree() ->
140 | ?mktab(T, []),
141 | L = [{<<"a*b">>, [], <<"1">>},
142 | {<<"a*c">>, [], <<"2">>},
143 | {<<"a*c*a[00000001]">>,[],<<"3">>},
144 | {<<"a*c*a[00000002]">>,[],<<"4">>}],
145 | {conf_tree, _, _} = Tree = kvdb_conf:make_tree(L),
146 | %% To write the whole tree, we must first shift the root into the tree.
147 | ok = kvdb_conf:write_tree(T, <<>>, kvdb_conf:shift_root(top, Tree)),
148 | {L,_} = kvdb_conf:prefix_match(T, <<>>),
149 | ok = kvdb_conf:delete_table(T).
150 |
151 | first_next_child() ->
152 | ?mktab(T, []),
153 | ok = kvdb_conf:write(T, {<<"a*b">>, [], <<"1">>}),
154 | ok = kvdb_conf:write(T, {<<"a*c">>, [], <<"2">>}),
155 | ok = kvdb_conf:write(T, {<<"a*c*1">>, [], <<"3">>}),
156 | ok = kvdb_conf:write(T, {<<"a*d">>, [], <<"4">>}),
157 | {ok, <<"a*b">>} = kvdb_conf:first_child(T, <<"a">>),
158 | {ok, <<"a*d">>} = kvdb_conf:last_child(T, <<"a">>),
159 | {ok, <<"a*c">>} = kvdb_conf:next_child(T, <<"a*b">>),
160 | {ok, <<"a*d">>} = kvdb_conf:next_child(T, <<"a*c">>),
161 | done = kvdb_conf:next_child(T, <<"a*d">>),
162 | ok = kvdb_conf:delete_table(T).
163 |
164 | fold_list() ->
165 | ?mktab(T, []),
166 | ok = kvdb_conf:write(T, {<<"a*b*b">>, [], <<>>}),
167 | ok = kvdb_conf:write(T, {<<"a*b*c[00000001]">>, [], <<>>}),
168 | ok = kvdb_conf:write(T, {<<"a*b*c[00000005]*d">>, [], <<>>}),
169 | ok = kvdb_conf:write(T, {<<"a*b*c[00000005]*e">>, [], <<>>}),
170 | ok = kvdb_conf:write(T, {<<"a*b*c[0000000A]">>, [], <<>>}),
171 | ok = kvdb_conf:write(T, {<<"a*b*d">>, [], <<>>}),
172 | [{10, <<"a*b*c[0000000A]">>},
173 | {5, <<"a*b*c[00000005]">>},
174 | {1, <<"a*b*c[00000001]">>}] =
175 | kvdb_conf:fold_list(T, fun(I,K,Acc) ->
176 | [{I,K}|Acc]
177 | end, [], <<"a*b*c">>),
178 | ok = kvdb_conf:delete_table(T).
179 |
180 | fold_children() ->
181 | ?mktab(T, []),
182 | ok = kvdb_conf:write(T, {<<"a*b">>, [], <<>>}),
183 | ok = kvdb_conf:write(T, {<<"a*b*b">>, [], <<>>}),
184 | ok = kvdb_conf:write(T, {<<"a*b*c">>, [], <<>>}),
185 | ok = kvdb_conf:write(T, {<<"a*b*d">>, [], <<>>}),
186 | ok = kvdb_conf:write(T, {<<"a*c">>, [], <<>>}),
187 | [<<"a*b*d">>,
188 | <<"a*b*c">>,
189 | <<"a*b*b">>] =
190 | kvdb_conf:fold_children(T, fun(K,Acc) ->
191 | [K|Acc]
192 | end, [], <<"a*b">>),
193 | ok = kvdb_conf:delete_table(T).
194 |
195 |
196 | -endif.
197 |
--------------------------------------------------------------------------------
/test/kvdb_lib_proper.erl:
--------------------------------------------------------------------------------
1 | -module(kvdb_lib_proper).
2 |
3 | -ifdef(PROPER).
4 |
5 | -export([q_raw_keyed/0,
6 | q_raw_keyed_sorts/0,
7 | escape_unescape/0,
8 | escape_sorts/0]).
9 |
10 | -include_lib("proper/include/proper.hrl").
11 |
12 |
13 | -type raw_key() :: binary().
14 |
15 | q_raw_keyed() ->
16 | T = {keyed,fifo},
17 | ?FORALL(Key, raw_key(),
18 | begin
19 | {Raw,QK} =
20 | kvdb_lib:actual_key(raw, T, <<"q">>, Key),
21 | QK == kvdb_lib:split_queue_key(raw, T, Raw)
22 | end).
23 |
24 | q_raw_keyed_sorts() ->
25 | T = {keyed,fifo},
26 | ?FORALL({K1,K2}, {raw_key(), raw_key()},
27 | begin
28 | {Raw1,_} = kvdb_lib:actual_key(raw, T, <<"q">>, K1),
29 | {Raw2,_} = kvdb_lib:actual_key(raw, T, <<"q">>, K2),
30 | (Raw1 > Raw2) == (K1 > K2)
31 | end).
32 |
33 | escape_sorts() ->
34 | ?FORALL({A,B}, {raw_key(), raw_key()},
35 | begin
36 | X = kvdb_lib:escape(A),
37 | Y = kvdb_lib:escape(B),
38 | (A < B) == (X < Y)
39 | end).
40 |
41 | escape_unescape() ->
42 | ?FORALL(K, raw_key(),
43 | K == kvdb_lib:unescape(kvdb_lib:escape(K))).
44 |
45 | -endif. % PROPER
46 |
--------------------------------------------------------------------------------
/test/kvdb_trans_tests.erl:
--------------------------------------------------------------------------------
1 | %%%---- BEGIN COPYRIGHT -------------------------------------------------------
2 | %%%
3 | %%% Copyright (C) 2012 Feuerlabs, Inc. All rights reserved.
4 | %%%
5 | %%% This Source Code Form is subject to the terms of the Mozilla Public
6 | %%% License, v. 2.0. If a copy of the MPL was not distributed with this
7 | %%% file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 | %%%
9 | %%%---- END COPYRIGHT ---------------------------------------------------------
10 | %%% @author Ulf Wiger
11 | -module(kvdb_trans_tests).
12 |
13 | -ifdef(TEST).
14 |
15 | -include_lib("eunit/include/eunit.hrl").
16 |
17 | -define(_t(E), {timeout,60000,
18 | [?_test(try E catch error:_R_ ->
19 | error({_R_, erlang:get_stacktrace()})
20 | end)]}).
21 |
22 | -define(tab, iolist_to_binary([<<"t_">>, integer_to_list(?LINE)])).
23 |
24 | -define(dbg(E),
25 | (fun() ->
26 | try (E) of
27 | __V ->
28 | ?debugFmt(<<"~s = ~P">>, [(??E), __V, 15]),
29 | __V
30 | catch
31 | error:__Err ->
32 | io:fwrite(user,
33 | "FAIL: test = ~s~n"
34 | "Error = ~p~n"
35 | "Trace = ~p~n", [(??E), __Err,
36 | erlang:get_stacktrace()]),
37 | error(__Err)
38 | end
39 | end)()).
40 |
41 |
42 | -define(trace(Mods, Expr), begin dbg:tracer(),
43 | lists:foreach(
44 | fun(_M_) when is_atom(_M_) ->
45 | dbg:tpl(_M_,x);
46 | ({_M_,_F_}) ->
47 | dbg:tpl(_M_,_F_,x)
48 | end, Mods),
49 | dbg:p(all,[c]),
50 | try Expr
51 | after
52 | lists:foreach(
53 | fun(_M_) when is_atom(_M_) ->
54 | dbg:ctpl(_M_);
55 | ({_M_,_F_}) ->
56 | dbg:ctpl(_M_,_F_)
57 | end, Mods),
58 | dbg:stop()
59 | end
60 | end).
61 | fill_test_() ->
62 | {setup,
63 | fun() ->
64 | ?debugVal(application:start(gproc)),
65 | ?debugVal(application:start(kvdb)),
66 | ok
67 | end,
68 | fun(_) ->
69 | ?debugVal(application:stop(kvdb)),
70 | ?debugVal(application:stop(gproc))
71 | end,
72 | {foreachx,
73 | fun({N,Opts,D}) ->
74 | delete_files(Opts),
75 | open_db(N, Opts, D)
76 | end,
77 | [{{N,Opts,D}, fun(_, Db) ->
78 | [?_t(?dbg(fill_db(N, Db, Opts, D)))
79 | , ?_t(?dbg(add_tab_index_get(N)))
80 | , ?_t(?dbg(write_del_write(N)))
81 | , ?_t(?dbg(update_counter(N)))
82 | , ?_t(?dbg(first_next(N)))
83 | , ?_t(?dbg(get_attrs(N)))
84 | , ?_t(?dbg(q(N)))
85 | , ?_t(?dbg(q_push_pop(N)))
86 | , ?_t(?dbg(q_push_prel_pop(N)))
87 | , ?_t(?dbg(q_extract(N)))
88 | , ?_t(?dbg(q_mark_blocking(N)))
89 | , ?_t(?dbg(q_mark_inactive(N)))
90 | , ?_t(?dbg(index_get(N)))
91 | , ?_t(?dbg(index_keys(N)))
92 | , ?_t(?dbg(prefix_match(N)))
93 | , ?_t(?dbg(prefix_match_rel(N)))
94 | , ?_t(?dbg(no_lingering_monitors(N)))
95 | ]
96 | end} ||
97 | {N,Opts,D} <- [new_opts(foo_10, 10)]]
98 | }}.
99 |
100 |
101 | delete_files(_Opts) -> ok.
102 | %% {_, Dir} = lists:keyfind(log_dir, 1, Opts),
103 | %% {_, File} = lists:keyfind(file, 1, Opts),
104 | %% ?debugVal({os:cmd("rm -r " ++ Dir),
105 | %% file:delete(File)}).
106 |
107 | open_db(N, Opts, D) ->
108 | {ok, Db} = kvdb:open(N, [{log_dir, filename:join(D, "kvdb.log")},
109 | {file, filename:join(D, "kvdb.tab")}
110 | | Opts]),
111 | Db.
112 |
113 | new_opts(Name, N) ->
114 | {Name, [{backend, ets},
115 | {log_threshold, [{writes,N}]}], dirname(Name)}.
116 |
117 | dirname(Name) ->
118 | {_,S,U} = erlang:now(),
119 | filename:join("/tmp", atom_to_list(Name)
120 | ++ "." ++ integer_to_list(S) ++ "." ++ integer_to_list(U)).
121 |
122 |
123 | fill_db(Name, _Db, Opts, D) ->
124 | %% Numbers are set pretty low right now to speed things up.
125 | %% The risks are: (1) sleep is set too low so that log switch doesn't
126 | %% occur within the batch, and (2) the number of writes is too low - really
127 | %% at *least* 2-3 log switches should take place before we end the test.
128 | ?assertMatch(ok, kvdb:add_table(Name, t, [{encoding,sext}])),
129 | Objs = [{N,a} || N <- lists:seq(1,30)],
130 | lists:foreach(
131 | fun(Obj) ->
132 | timer:sleep(30),
133 | kvdb:put(Name, t, Obj)
134 | end, Objs),
135 | ?debugFmt("closing ~p...~n", [Name]),
136 | kvdb:close(Name),
137 | ?debugFmt("DB closed. Trying to reopen...~n", []),
138 | timer:sleep(500),
139 | open_db(Name, Opts, D),
140 | Found = kvdb:prefix_match(Name, t, '_', infinity),
141 | %% io:fwrite(user, "Objs = ~p~n", [Objs]),
142 | ?assertMatch({Objs, _}, Found).
143 |
144 | add_tab_index_get(Name) ->
145 | T1 = ?tab,
146 | kvdb:in_transaction(
147 | Name,
148 | fun(_) ->
149 | ok = kvdb:add_table(Name, T1, [{type,set},
150 | {encoding, {sext,term,term}},
151 | {index, [a]}]),
152 | [] = kvdb:index_get(Name, T1, a, v1),
153 | [] = kvdb:index_keys(Name, T1, a, v1),
154 | ok = kvdb:put(Name, T1, {a, [{a, v1}], 1}),
155 | ok = kvdb:put(Name, T1, {b, [{a, v1}], 2}),
156 | [{a,[{a,v1}],1},{b,[{a,v1}],2}] = kvdb:index_get(Name, T1, a, v1),
157 | [a, b] = kvdb:index_keys(Name, T1, a, v1),
158 | ok
159 | end).
160 |
161 | write_del_write(Name) ->
162 | T1 = ?tab,
163 | write_del_write(Name, T1).
164 |
165 | write_del_write(Name, T1) ->
166 | ok = kvdb:add_table(Name, T1, [{type,set},
167 | {encoding, {sext,term,sext}}]),
168 | ok = kvdb:put(Name, T1, {x, [], 1}),
169 | kvdb:in_transaction(
170 | Name,
171 | fun(_) ->
172 | ok = kvdb:delete(Name, T1, x),
173 | ok = kvdb:put(Name, T1, {x,[],1}),
174 | {ok, {x,[],1}} = kvdb:get(Name, T1, x)
175 | end),
176 | {ok, {x,[],1}} = kvdb:get(Name, T1,x),
177 | ok.
178 |
179 | update_counter(Name) ->
180 | T1 = ?tab,
181 | T2 = ?tab,
182 | ok = kvdb:add_table(Name, T1, [{type, set},
183 | {encoding, sext}]),
184 | ok = kvdb:add_table(Name, T2, [{type, set},
185 | {encoding, raw}]),
186 | ok = kvdb:put(Name, T1, {c, 1}),
187 | ok = kvdb:put(Name, T2, {<<"c">>, <<1>>}),
188 | Res1 = kvdb_trans:run(
189 | Name,
190 | fun(_) ->
191 | R1 = kvdb:update_counter(Name, T1, c, 1),
192 | R2 = kvdb:update_counter(Name, T1, c, 1),
193 | R3 = kvdb:update_counter(Name, T2, <<"c">>, 1),
194 | R4 = kvdb:update_counter(Name, T2, <<"c">>, 1),
195 | [R1,R2,R3,R4]
196 | end),
197 | ?assertMatch([2,3,<<2>>,<<3>>], Res1),
198 | ok = kvdb:delete_table(Name, T1),
199 | ok = kvdb:delete_table(Name, T2).
200 |
201 | first_next(Name) ->
202 | T1 = ?tab,
203 | T2 = ?tab,
204 | T3 = ?tab,
205 | [ok,ok,ok] =
206 | [kvdb:add_table(Name, T, [{type,set},{encoding,sext}]) ||
207 | T <- [T1,T2,T3]],
208 | [kvdb:put(Name, T, Obj) || {T, Obj} <- [{T1, {t11,a}},
209 | {T2, {t21,a}},
210 | {T2, {t22,a}},
211 | {T3, {t31,a}}]],
212 | Res = kvdb_trans:run(
213 | Name,
214 | fun(_) ->
215 | {ok,{t21,a}} = kvdb:first(Name, T2),
216 | {ok,{t22,a}} = kvdb:next(Name, T2, t21),
217 | {ok,{t21,a}} = kvdb:prev(Name, T2, t22),
218 | {ok,{t22,a}} = kvdb:last(Name, T2),
219 | done = kvdb:next(Name, T2, t22),
220 | done = kvdb:prev(Name, T2, t21)
221 | end),
222 | [ok,ok,ok] =
223 | [kvdb:delete_table(Name, T) || T <- [T1,T2,T3]],
224 | ok.
225 |
226 | get_attrs(Name) ->
227 | T1 = ?tab,
228 | T2 = ?tab,
229 | ok = kvdb:add_table(Name, T1, [{encoding,{raw,sext,term}}]),
230 | ok = kvdb:add_table(Name, T2, [{encoding,{sext,sext,term}}]),
231 | As = [{a,1}, {b,2}, {c,3}],
232 | ok = kvdb:put(Name, T1,{<<"a">>,As,1}),
233 | ok = kvdb:put(Name, T2,{a,As,1}),
234 | kvdb:transaction(
235 | Name,
236 | fun(_) ->
237 | {ok, [{a,1},{b,2}]} = kvdb:get_attrs(Name,T1,<<"a">>,[a,b]),
238 | {ok, [{a,1},{b,2}]} = kvdb:get_attrs(Name,T2,a,[a,b]),
239 | {ok, [{a,1},{b,2},{c,3}]} =
240 | kvdb:get_attrs(Name,T1,<<"a">>,all),
241 | {ok, [{a,1},{b,2},{c,3}]} =
242 | kvdb:get_attrs(Name,T2,a,all),
243 | ok = kvdb:put(Name, T1, {<<"a">>,[{a,10}],1}),
244 | ok = kvdb:put(Name, T2, {a,[{a,10}],1}),
245 | {ok, [{a,10}]} = kvdb:get_attrs(Name,T1,<<"a">>,[a,b]),
246 | {ok, [{a,10}]} = kvdb:get_attrs(Name,T2,a,[a,b])
247 | end),
248 | ok = kvdb:delete_table(Name, T1),
249 | ok = kvdb:delete_table(Name, T2).
250 |
251 |
252 | q(Name) ->
253 | T = ?tab,
254 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
255 | kvdb:push(Name, T, <<>>, {1,a}),
256 | kvdb:push(Name, T, <<>>, {2,b}),
257 | Res1 = kvdb_trans:run(
258 | Name, fun(_) ->
259 | kvdb:list_queue(Name, T, <<>>)
260 | end),
261 | ?assertMatch({[{1,a},{2,b}], _}, Res1),
262 | kvdb_trans:run(
263 | Name, fun(_) ->
264 | kvdb:push(Name, T, <<>>, {3,c}),
265 | kvdb:push(Name, T, <<>>, {4,d})
266 | end),
267 | Res2 = kvdb_trans:run(
268 | Name, fun(_) ->
269 | kvdb:list_queue(Name, T, <<>>)
270 | end),
271 | ?assertMatch({[{1,a},{2,b},{3,c},{4,d}], _}, Res2),
272 | ok = kvdb:delete_table(Name, T).
273 |
274 | q_push_pop(Name) ->
275 | T = ?tab,
276 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
277 | kvdb:push(Name, T, q, {1,a}),
278 | kvdb:push(Name, T, q, {2,b}),
279 | Res1 = kvdb_trans:run(
280 | Name, fun(_) ->
281 | kvdb:pop(Name, T, q)
282 | end),
283 | ?assertMatch({ok, {1,a}}, Res1),
284 | ok = kvdb:delete_table(Name, T).
285 |
286 | q_push_prel_pop(Name) ->
287 | T = ?tab,
288 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
289 | {ok, QKey1} = kvdb:push(Name, T, q, {1,a}),
290 | kvdb:push(Name, T, q, {2,b}),
291 | Res1 = kvdb_trans:run(
292 | Name, fun(_) ->
293 | ?assertMatch({ok,{1,a},QKey1},
294 | kvdb:prel_pop(Name, T, q)),
295 | kvdb:pop(Name, T, q)
296 | end),
297 | ?assertMatch(blocked, Res1),
298 | ok = kvdb:delete_table(Name, T).
299 |
300 | q_extract(Name) ->
301 | T = ?tab,
302 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
303 | {ok, QKey} = kvdb:push(Name, T, q, {1,a}),
304 | kvdb:push(Name, T, q, {2,b}),
305 | Res1 = kvdb_trans:run(
306 | Name, fun(_) ->
307 | ?assertMatch({ok, {1,a}},
308 | kvdb:extract(Name, T, QKey)),
309 | kvdb:pop(Name, T, q)
310 | end),
311 | ?assertMatch({ok, {2,b}}, Res1),
312 | ok = kvdb:delete_table(Name, T).
313 |
314 | q_mark_blocking(Name) ->
315 | T = ?tab,
316 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
317 | {ok, QK1} = kvdb:push(Name, T, q, {1,a}),
318 | Res1 =
319 | kvdb_trans:run(
320 | Name, fun(DbT) ->
321 | kvdb:mark_queue_object(Name, T, QK1, blocking),
322 | %% io:fwrite(user, "Tstore = ~p~n",
323 | %% [kvdb_trans:tstore_to_list(DbT)]),
324 | kvdb:pop(Name, T, q)
325 | end),
326 | ?assertMatch(blocked, Res1),
327 | Res2 = kvdb:pop(Name, T, q),
328 | ?assertMatch(blocked, Res2),
329 | ok = kvdb:delete_table(Name, T).
330 |
331 | q_mark_inactive(Name) ->
332 | T = ?tab,
333 | ok = kvdb:add_table(Name, T, [{type, fifo}, {encoding, sext}]),
334 | {ok, QK1} = kvdb:push(Name, T, q, {1,a}),
335 | {ok, _} = kvdb:push(Name, T, q, {2,b}),
336 | Res1 =
337 | kvdb_trans:run(
338 | Name, fun(DbT) ->
339 | kvdb:mark_queue_object(Name, T, QK1, inactive),
340 | kvdb:pop(Name, T, q)
341 | end),
342 | ?assertMatch({ok,{2,b}}, Res1),
343 | Res2 = kvdb:pop(Name, T, q),
344 | ?assertMatch(done, Res2),
345 | ok = kvdb:delete_table(Name, T).
346 |
347 | index_get(Name) ->
348 | T = ?tab,
349 | ok = kvdb:add_table(Name, T, [{type, set},
350 | {encoding, {sext,sext,sext}},
351 | {index, [a]}]),
352 | ok = kvdb:put(Name, T, {1, [{a, 1}], a}),
353 | ok = kvdb:put(Name, T, {2, [{a, 1}], b}),
354 | Res1 = kvdb_trans:run(
355 | Name,
356 | fun(_) ->
357 | ok = kvdb:put(Name, T, {3, [{a, 1}], c}),
358 | ok = kvdb:put(Name, T, {4, [{a, 1}], d}),
359 | ok = kvdb:put(Name, T, {5, [], e}),
360 | kvdb:index_get(Name, T, a, 1)
361 | end),
362 | ?assertMatch([{1,[{a,1}],a},
363 | {2,[{a,1}],b},
364 | {3,[{a,1}],c},
365 | {4,[{a,1}],d}], Res1),
366 | ok = kvdb:delete_table(Name, T).
367 |
368 | index_keys(Name) ->
369 | T = ?tab,
370 | ok = kvdb:add_table(Name, T, [{type, set},
371 | {encoding, {sext,sext,sext}},
372 | {index, [a]}]),
373 | ok = kvdb:put(Name, T, {1, [{a, 1}], a}),
374 | ok = kvdb:put(Name, T, {2, [{a, 1}], b}),
375 | Res1 = kvdb_trans:run(
376 | Name,
377 | fun(_) ->
378 | ok = kvdb:put(Name, T, {3, [{a, 1}], c}),
379 | ok = kvdb:put(Name, T, {4, [{a, 1}], d}),
380 | ok = kvdb:put(Name, T, {5, [], e}),
381 | kvdb:index_keys(Name, T, a, 1)
382 | end),
383 | ?assertMatch([1,2,3,4], Res1),
384 | ok = kvdb:delete_table(Name, T).
385 |
386 | prefix_match(Name) ->
387 | T = ?tab,
388 | ok = kvdb:add_table(Name, T, [{type, set}, {encoding, sext}]),
389 | [ok,ok] = [kvdb:put(Name, T, Obj) || Obj <- [{2,b}, {3,c}]],
390 | Res = kvdb:transaction(
391 | Name,
392 | fun(_) ->
393 | {[{2,b},{3,c}], _} =
394 | kvdb:prefix_match(Name, T, '_', infinity),
395 | [ok,ok] =
396 | [kvdb:put(Name, T, Obj) || Obj <- [{1,a}, {4,d}]],
397 | {[{1,a}, {2,b}], C1} =
398 | kvdb:prefix_match(Name, T, '_', 2),
399 | {[{3,c}, {4,d}], C2} = C1(),
400 | done = C2()
401 | end),
402 | ok.
403 |
404 | prefix_match_rel(_) ->
405 | ok.
406 |
407 | no_lingering_monitors(Name) ->
408 | {monitored_by, Bef} = process_info(self(), monitored_by),
409 | io:fwrite(user, "monitored by Bef = ~p~n",
410 | [[process_info(P) || P <- Bef]]),
411 | T = ?tab,
412 | write_del_write(Name, T),
413 | {monitored_by, Aft} = process_info(self(), monitored_by),
414 | io:fwrite(user, "monitored by Aft = ~p~n",
415 | [[process_info(P) || P <- Aft]]),
416 | [] = Aft -- Bef.
417 |
418 | -endif.
419 |
--------------------------------------------------------------------------------
/test/kvdb_usb_ets.erl:
--------------------------------------------------------------------------------
1 | -module(kvdb_usb_ets).
2 |
3 | -compile(export_all).
4 |
5 | -define(SET_SZ, 5).
6 | -define(N, 5).
7 | -define(M, 50).
8 | -define(PAR, 6).
9 |
10 | run() ->
11 | run("/Volumes/USB20FD/t1").
12 |
13 | run2() ->
14 | run("/Users/uwiger/tmp/usb_test/t1").
15 |
16 | run3() ->
17 | run_mnesia("/Users/uwiger/tmp/usb_test/m1").
18 |
19 | run(Dir) ->
20 | init(),
21 | create(usb_test, [{writes, 10}], Dir),
22 | write(usb_test, ?N, ?M).
23 |
24 | run_mnesia(Dir) ->
25 | application:load(mnesia),
26 | application:set_env(mnesia, dir, Dir),
27 | application:set_env(mnesia, dump_log_write_threshold, 10),
28 | mnesia:create_schema([node()]),
29 | mnesia:start(),
30 | mnesia:create_table(t, [{disc_copies, [node()]}]),
31 | mnesia:wait_for_tables([t], 10000),
32 | ets:new(?MODULE, [ordered_set, named_table, public]),
33 | write_mnesia(?N, ?M).
34 |
35 | init() ->
36 | ets:new(?MODULE, [ordered_set, named_table, public]),
37 | kvdb:start().
38 |
39 | create(Name, Threshold, Dir) ->
40 | Base = filename:join(Dir, kvdb_lib:good_string(Name)),
41 | kvdb:open(
42 | Name, [{file, Base ++ ".db"},
43 | {log_dir, Base ++ ".log"},
44 | {backend, ets},
45 | {save_mode, [on_close, on_switch]},
46 | {log_threshold, Threshold}]).
47 |
48 | write(Name, N, M) ->
49 | pforeach(
50 | fun({N1,M1}) ->
51 | case M1 rem 2 of
52 | 0 ->
53 | {T, ok} = timer:tc(?MODULE, write_set, [Name, N1]),
54 | log(T, ins);
55 | _ ->
56 | {T, ok} = timer:tc(?MODULE, delete_set, [Name, N1]),
57 | log(T, del)
58 | end
59 | end, [{Nx, Mx} || Mx <- lists:seq(1,M),
60 | Nx <- lists:seq(1,N)]).
61 |
62 | pforeach(F, L) ->
63 | Split = split(?PAR, L),
64 | pforeach_(F, Split).
65 |
66 | pforeach_(F, [L|T]) ->
67 | Pids = [spawn_monitor(fun() -> exit({ok, F(X)}) end) || X <- L],
68 | collect(Pids),
69 | pforeach_(F, T);
70 | pforeach_(_, []) ->
71 | ok.
72 |
73 | split(_, []) ->
74 | [];
75 | split(N, L) ->
76 | case length(L) of
77 | Len when Len > N ->
78 | {A,B} = lists:split(N, L),
79 | [A | split(N, B)];
80 | _ ->
81 | [L]
82 | end.
83 |
84 |
85 | collect([{_,Ref}|T]) ->
86 | receive
87 | {'DOWN', Ref, _, _, Res} ->
88 | case Res of
89 | {ok, _} ->
90 | collect(T);
91 | Other ->
92 | [exit(P,kill) || {P,_} <- T],
93 | error(Other)
94 | end
95 | end;
96 | collect([]) ->
97 | ok.
98 |
99 |
100 |
101 | write_mnesia(N, M) ->
102 | lists:foreach(
103 | fun({N1,M1}) ->
104 | case M1 rem 2 of
105 | 0 ->
106 | {T, ok} = timer:tc(?MODULE, write_set_m, [N1]),
107 | log(T, ins);
108 | _ ->
109 | {T, ok} = timer:tc(?MODULE, delete_set_m, [N1]),
110 | log(T, del)
111 | end
112 | end, [{Nx, Mx} || Mx <- lists:seq(1,M),
113 | Nx <- lists:seq(1,N)]).
114 |
115 | log(T, Op) ->
116 | ets:insert(?MODULE, {erlang:now(), {T, Op}}),
117 | io:fwrite("~p: ~8w | ~.3f~n", [Op, T, T/?SET_SZ]).
118 |
119 | write_set(Name, N) ->
120 | kvdb_trans:run(
121 | Name, fun(_) ->
122 | kvdb:add_table(Name, Tab = "t_" ++ integer_to_list(N),
123 | [{encoding,sext}]),
124 | R = [kvdb:put(Name, Tab, {N1, a}) ||
125 | N1 <- lists:seq(1,?SET_SZ)],
126 | true = lists:all(fun(X) -> X == ok end, R),
127 | ok
128 | end).
129 | write_set_m(N) ->
130 | mnesia:activity(
131 | transaction,
132 | fun() ->
133 | Tab = iolist_to_binary(["t_", integer_to_list(N)]),
134 | R = [mnesia:write({t, {Tab,N1}, a}) ||
135 | N1 <- lists:seq(1,?SET_SZ)],
136 | true = lists:all(fun(X) -> X == ok end, R),
137 | ok
138 | end).
139 |
140 | delete_set(Name, N) ->
141 | kvdb_trans:run(
142 | Name, fun(_) ->
143 | kvdb:delete_table(Name, "t_" ++ integer_to_list(N))
144 | end).
145 |
146 | delete_set_m(N) ->
147 | mnesia:activity(
148 | transaction,
149 | fun() ->
150 | Tab = iolist_to_binary(["t_", integer_to_list(N)]),
151 | Objs = mnesia:match_object({t, {Tab,'_'}, '_'}),
152 | [mnesia:delete({t, element(1,O)}) || O <- Objs],
153 | ok
154 | end).
155 |
--------------------------------------------------------------------------------
/tetrapak/config.ini:
--------------------------------------------------------------------------------
1 | [build]
2 | version = "~t.~o~~~c"
3 |
4 | [package]
5 | maintainer = "Ulf Wiger "
6 | exclude = "\\.gitignore|README.md"
7 | architecture = host
8 |
9 | [xref]
10 |
11 | ignore_undef = [
12 | { eleveldb, delete, 3 },
13 | { eleveldb, get, 3 },
14 | { eleveldb, iterator, 2 },
15 | { eleveldb, iterator_close, 1 },
16 | { eleveldb, iterator_move, 2 },
17 | { eleveldb, open, 2 },
18 | { eleveldb, put, 4 },
19 | { eleveldb, write, 3 },
20 | { setup, find_env_vars, 1 },
21 | { resource, notify_when_destroyed, 2 },
22 | { sext, decode, 1 },
23 | { sext, decode_next, 1 },
24 | { sext, encode, 1 },
25 | { sext, prefix, 1 },
26 | { sqlite3, bind, 3 },
27 | { sqlite3, close, 1 },
28 | { sqlite3, create_table, 3 },
29 | { sqlite3, delete, 3 },
30 | { sqlite3, drop_table, 2 },
31 | { sqlite3, finalize, 2 },
32 | { sqlite3, list_tables, 1 },
33 | { sqlite3, next, 2 },
34 | { sqlite3, open, 2 },
35 | { sqlite3, prepare, 2 },
36 | { sqlite3, read, 3 },
37 | { sqlite3, sql_exec, 2 },
38 | { sqlite3, sql_exec, 3 },
39 | { sqlite3, sql_exec_script, 2 },
40 | { sqlite3_lib, delete_sql, 3 },
41 | { sqlite3_lib, value_to_sql, 1 },
42 | { sqlite3_lib, write_col_sql, 1 },
43 | { sqlite3_lib, write_value_sql, 1 }
44 | ]
45 |
--------------------------------------------------------------------------------