├── ChangeLog ├── docs ├── mdm-slides.pdf └── 20120829-LinuxCon-MDB-txt.pdf ├── .gitignore ├── start.sh ├── tools ├── update-mdb ├── lmdb.config └── basho_bench_driver_lmdb.erl ├── src ├── lmdb.app.src ├── async_nif.hrl ├── lmdb_sup.erl └── lmdb.erl ├── rebar.config ├── Makefile ├── include └── lmdb.hrl ├── c_src ├── common.h ├── midl.h ├── midl.c ├── khash.h ├── queue.h ├── async_nif.h ├── lmdb_nif.c └── lmdb.h ├── LICENSE └── README.md /ChangeLog: -------------------------------------------------------------------------------- 1 | o Sep 29th, 2012 2 | initial release 3 | 4 | -------------------------------------------------------------------------------- /docs/mdm-slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gburd/lmdb/HEAD/docs/mdm-slides.pdf -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | .eunit 3 | ebin 4 | c_src/*.o 5 | deps/ 6 | priv/ 7 | *~ 8 | .rebar 9 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd `dirname $0` 4 | exec erl -pa $PWD/ebin -pa +B -- $@ 5 | -------------------------------------------------------------------------------- /docs/20120829-LinuxCon-MDB-txt.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gburd/lmdb/HEAD/docs/20120829-LinuxCon-MDB-txt.pdf -------------------------------------------------------------------------------- /tools/update-mdb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for file in lmdb.h mdb.c midl.h midl.c; do 4 | curl -O https://raw.githubusercontent.com/LMDB/lmdb/mdb.master/libraries/liblmdb/$file 5 | done 6 | 7 | -------------------------------------------------------------------------------- /src/lmdb.app.src: -------------------------------------------------------------------------------- 1 | {application, lmdb, 2 | [ 3 | {description, "OpenLDAP's Lightning Memory-Mapped Database BTREE"}, 4 | {vsn, "1.0.0"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {env, []} 11 | ]}. 12 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | 4 | {require_otp_vsn, "R1[56]|1[78]"}. 5 | 6 | {cover_enabled, true}. 7 | 8 | {erl_opts, [%{d,'DEBUG',true}, 9 | debug_info, 10 | warn_unused_vars, 11 | warn_export_all, 12 | warn_shadow_vars, 13 | warn_unused_import, 14 | warn_unused_function, 15 | warn_bif_clash, 16 | warn_unused_record, 17 | warn_deprecated_function, 18 | warn_obsolete_guard, 19 | warn_export_vars, 20 | warn_exported_vars, 21 | warn_untyped_record 22 | %warn_missing_spec, 23 | %strict_validation, 24 | %fail_on_warning 25 | ]}. 26 | 27 | {xref_checks, [undefined_function_calls, deprecated_function_calls]}. 28 | 29 | {eunit_opts, [verbose, {report, {eunit_surefire, [{dir, "."}]}}]}. 30 | 31 | {port_specs, [ 32 | {"priv/lmdb.so", ["c_src/*.c"]} 33 | ]}. 34 | 35 | {port_env, [ 36 | {"DRV_CFLAGS", "$DRV_CFLAGS -O3 -fPIC -march=native -mtune=native -Wall -Wextra"} 37 | ]}. 38 | 39 | % for debugging use 40 | % {"DRV_CFLAGS", "$DRV_CFLAGS -DMDB_DEBUG=2 -DMDB_PARANOID -DDEBUG -O0 -fPIC -Wall -Wextra -Werror"} 41 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULE = lmdb 2 | 3 | DIALYZER = dialyzer 4 | REBAR = rebar 5 | 6 | .PHONY: compile clean 7 | 8 | all: ebin priv compile 9 | 10 | ebin: 11 | @mkdir -p $@ 12 | 13 | priv: 14 | @mkdir -p $@ 15 | 16 | compile: 17 | @$(REBAR) compile 18 | 19 | clean: 20 | @$(REBAR) clean 21 | @rm -f *~ */*~ erl_crash.dump 22 | @rm -rf ebin priv 23 | 24 | xref: 25 | @$(REBAR) xref skip_deps=true 26 | 27 | test: eunit 28 | 29 | eunit: compile-for-eunit 30 | @$(REBAR) eunit skip_deps=true 31 | 32 | eqc: compile-for-eqc 33 | @$(REBAR) eqc skip_deps=true 34 | 35 | proper: compile-for-proper 36 | @echo "rebar does not implement a 'proper' command" && false 37 | 38 | triq: compile-for-triq 39 | @$(REBAR) triq skip_deps=true 40 | 41 | compile-for-eunit: 42 | @$(REBAR) compile eunit compile_only=true 43 | 44 | compile-for-eqc: 45 | @$(REBAR) -D QC -D QC_EQC compile eqc compile_only=true 46 | 47 | compile-for-eqcmini: 48 | @$(REBAR) -D QC -D QC_EQCMINI compile eqc compile_only=true 49 | 50 | compile-for-proper: 51 | @$(REBAR) -D QC -D QC_PROPER compile eqc compile_only=true 52 | 53 | compile-for-triq: 54 | @$(REBAR) -D QC -D QC_TRIQ compile triq compile_only=true 55 | 56 | plt: compile 57 | @$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa --apps kernel stdlib 58 | 59 | analyze: compile 60 | @$(DIALYZER) --plt .$(TARGET).plt 61 | 62 | repl: 63 | @$(ERL) exec erl -pa $PWD/ebin -pa +B 64 | -------------------------------------------------------------------------------- /tools/lmdb.config: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | 4 | %% How to: 5 | %% * put the lmdb.config file into basho_bench/examples 6 | %% * put the basho_bench_driver_lmdb.erl into basho_bench/src 7 | %% * make clean in basho_bench, then make 8 | %% * edit examples/lmdb.config 9 | %% - change {code_paths, ["../lmdb"]}. to be a relative path to your 10 | %% lmdb directory 11 | %% - change {lmdb_dir, "/home/gburd/ws/basho_bench/data"}. to a fully 12 | %% qualified location for your test data files (mkdir that directory 13 | %% yourself, if it doesn't exist the test will fail 'enoent') 14 | %% * to run, replace this path with the proper path on your system: 15 | %% LD_LIBRARY_PATH=/home/you/lmdb/priv ./basho_bench examples/lmdb.config 16 | %% * the test should run for 10 minutes (as it is configured right now) 17 | %% with 4 concurrent workers accessing the same table 18 | %% 19 | %% Note: 20 | %% There are two config sections in wt.config {lmdb, [ ... ]}. and 21 | %% {lmdb_, [ ... ]}. The one being used is named "lmdb" the other 22 | %% config is ignored. I setup an LSM and BTREE config and to choose 23 | %% which is run you just rename those two sections (turn one off by 24 | %% adding a "_" to the name and take the "_" out of the other's name). 25 | 26 | {mode, max}. 27 | {duration, 480}. 28 | {concurrent, 32}. 29 | {driver, basho_bench_driver_lmdb}. 30 | {key_generator, {int_to_bin_littleendian,{uniform_int, 5000000000}}}. 31 | {value_generator, {highly_compressible_bin, 2048}}. 32 | %{value_generator, {fixed_bin, 1024}}. 33 | {operations, [{get, 25}, {put, 70}, {delete, 5}]}. 34 | %{operations, [{put, 1}]}. 35 | {code_paths, ["../lmdb"]}. 36 | {lmdb_dir, "/home/gburd/ws/basho_bench/data"}. 37 | 38 | {lmdb, [ ]}. 39 | -------------------------------------------------------------------------------- /src/async_nif.hrl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% async_nif: An async thread-pool layer for Erlang's NIF API 4 | %% 5 | %% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 6 | %% Author: Gregory Burd 7 | %% 8 | %% This file is provided to you under the Apache License, 9 | %% Version 2.0 (the "License"); you may not use this file 10 | %% except in compliance with the License. You may obtain 11 | %% a copy of the License at 12 | %% 13 | %% http://www.apache.org/licenses/LICENSE-2.0 14 | %% 15 | %% Unless required by applicable law or agreed to in writing, 16 | %% software distributed under the License is distributed on an 17 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 | %% KIND, either express or implied. See the License for the 19 | %% specific language governing permissions and limitations 20 | %% under the License. 21 | %% 22 | %% ------------------------------------------------------------------- 23 | 24 | -define(ASYNC_NIF_CALL(Fun, Args), 25 | F = fun(F, T) -> 26 | R = erlang:make_ref(), 27 | case erlang:apply(Fun, [R|Args]) of 28 | {ok, {enqueued, PctBusy}} -> 29 | if 30 | PctBusy > 0.25 andalso PctBusy =< 1.0 -> 31 | erlang:bump_reductions(erlang:trunc(2000 * PctBusy)); 32 | true -> 33 | ok 34 | end, 35 | receive 36 | {R, {error, shutdown}=Error} -> 37 | %% Work unit was queued, but not executed. 38 | Error; 39 | {R, {error, _Reason}=Error} -> 40 | %% Work unit returned an error. 41 | Error; 42 | {R, Reply} -> 43 | Reply 44 | end; 45 | {error, eagain} -> 46 | case T of 47 | 3 -> not_found; 48 | _ -> F(F, T + 1) 49 | end; 50 | Other -> 51 | Other 52 | end 53 | end, 54 | F(F, 1)). 55 | -------------------------------------------------------------------------------- /include/lmdb.hrl: -------------------------------------------------------------------------------- 1 | %%------------------------------------------------------------------- 2 | %% This file is part of LMDB - Erlang Lightning MDB API 3 | %% 4 | %% Copyright (c) 2012 by Aleph Archives. All rights reserved. 5 | %% Copyright (c) 2013 by Basho Technologies, Inc. All rights reserved. 6 | %% 7 | %%------------------------------------------------------------------- 8 | %% Redistribution and use in source and binary forms, with or without 9 | %% modification, are permitted only as authorized by the OpenLDAP 10 | %% Public License. 11 | %% 12 | %% A copy of this license is available in the file LICENSE in the 13 | %% top-level directory of the distribution or, alternatively, at 14 | %% . 15 | %% 16 | %% Permission to use, copy, modify, and distribute this software for any 17 | %% purpose with or without fee is hereby granted, provided that the above 18 | %% copyright notice and this permission notice appear in all copies. 19 | %% 20 | %% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 21 | %% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 22 | %% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 23 | %% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 24 | %% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 25 | %% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 26 | %% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 27 | %%------------------------------------------------------------------- 28 | 29 | 30 | %% Environment Flags 31 | -define(MDB_FIXEDMAP, 16#01). 32 | -define(MDB_NOSUBDIR, 16#02). 33 | -define(MDB_NOSYNC, 16#10000). 34 | -define(MDB_RDONLY, 16#20000). 35 | -define(MDB_NOMETASYNC, 16#40000). 36 | -define(MDB_WRITEMAP, 16#80000). 37 | -define(MDB_MAPASYNC, 16#100000). 38 | -------------------------------------------------------------------------------- /c_src/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 3 | * Author: Gregory Burd 4 | * 5 | * This file is provided to you under the Apache License, 6 | * Version 2.0 (the "License"); you may not use this file 7 | * except in compliance with the License. You may obtain 8 | * a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef __COMMON_H__ 21 | #define __COMMON_H__ 22 | 23 | #if defined(__cplusplus) 24 | extern "C" { 25 | #endif 26 | 27 | #if !(__STDC_VERSION__ >= 199901L || defined(__GNUC__)) 28 | # undef DEBUG 29 | # define DEBUG 0 30 | # define DPRINTF (void) /* Vararg macros may be unsupported */ 31 | #elif DEBUG 32 | #include 33 | #include 34 | #define DPRINTF(fmt, ...) \ 35 | do { \ 36 | fprintf(stderr, "%s:%d " fmt "\n", __FILE__, __LINE__, __VA_ARGS__); \ 37 | fflush(stderr); \ 38 | } while(0) 39 | #define DPUTS(arg) DPRINTF("%s", arg) 40 | #else 41 | #define DPRINTF(fmt, ...) ((void) 0) 42 | #define DPUTS(arg) ((void) 0) 43 | #endif 44 | 45 | #ifndef __UNUSED 46 | #define __UNUSED(v) ((void)(v)) 47 | #endif 48 | 49 | #ifndef COMPQUIET 50 | #define COMPQUIET(n, v) do { \ 51 | (n) = (v); \ 52 | (n) = (n); \ 53 | } while (0) 54 | #endif 55 | 56 | #ifdef __APPLE__ 57 | #define PRIuint64(x) (x) 58 | #else 59 | #define PRIuint64(x) (unsigned long long)(x) 60 | #endif 61 | 62 | #if defined(__cplusplus) 63 | } 64 | #endif 65 | 66 | #endif // __COMMON_H__ 67 | -------------------------------------------------------------------------------- /tools/basho_bench_driver_lmdb.erl: -------------------------------------------------------------------------------- 1 | -module(basho_bench_driver_lmdb). 2 | 3 | -record(state, { 4 | handle 5 | }). 6 | 7 | -export([new/1, 8 | run/4]). 9 | 10 | -include_lib("basho_bench/include/basho_bench.hrl"). 11 | 12 | 13 | %% ==================================================================== 14 | %% API 15 | %% ==================================================================== 16 | 17 | new(1) -> 18 | %% Make sure lmdb is available 19 | case code:which(lmdb) of 20 | non_existing -> 21 | ?FAIL_MSG("~s requires lmdb to be available on code path.\n", 22 | [?MODULE]); 23 | _ -> 24 | ok 25 | end, 26 | %{ok, _} = lmdb_sup:start_link(), 27 | setup(1); 28 | new(Id) -> 29 | setup(Id). 30 | 31 | setup(_Id) -> 32 | %% Get the target directory 33 | Dir = basho_bench_config:get(lmdb_dir, "/tmp"), 34 | %Config = basho_bench_config:get(lmdb, []), 35 | 36 | %% Start Lightning MDB 37 | case lmdb:open(Dir, 32212254720, 16#10000 bor 16#40000 bor 16#80000) of 38 | {ok, H} -> 39 | {ok, #state{handle=H}}; 40 | {error, Reason} -> 41 | ?FAIL_MSG("Failed to establish a Lightning MDB connection, lmdb backend unable to start: ~p\n", [Reason]), 42 | {error, Reason} 43 | end. 44 | 45 | run(get, KeyGen, _ValueGen, #state{handle=Handle}=State) -> 46 | case lmdb:get(Handle, KeyGen()) of 47 | {ok, _Value} -> 48 | {ok, State}; 49 | not_found -> 50 | {ok, State}; 51 | {error, Reason} -> 52 | {error, Reason} 53 | end; 54 | run(put, KeyGen, ValueGen, #state{handle=Handle}=State) -> 55 | Key = KeyGen(), 56 | Val = ValueGen(), 57 | case lmdb:upd(Handle, Key, Val) of 58 | ok -> 59 | {ok, State}; 60 | {error, Reason} -> 61 | {error, Reason} 62 | end; 63 | run(delete, KeyGen, _ValueGen, #state{handle=Handle}=State) -> 64 | case lmdb:del(Handle, KeyGen()) of 65 | ok -> 66 | {ok, State}; 67 | not_found -> 68 | {ok, State}; 69 | {error, Reason} -> 70 | {error, Reason} 71 | end. 72 | 73 | %% config_value(Key, Config, Default) -> 74 | %% case proplists:get_value(Key, Config) of 75 | %% undefined -> 76 | %% Default; 77 | %% Value -> 78 | %% Value 79 | %% end. 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------- 2 | The OpenLDAP Public License 3 | Version 2.8, 17 August 2003 4 | 5 | Redistribution and use of this software and associated documentation 6 | ("Software"), with or without modification, are permitted provided 7 | that the following conditions are met: 8 | 9 | 1. Redistributions in source form must retain copyright statements 10 | and notices, 11 | 12 | 2. Redistributions in binary form must reproduce applicable copyright 13 | statements and notices, this list of conditions, and the following 14 | disclaimer in the documentation and/or other materials provided 15 | with the distribution, and 16 | 17 | 3. Redistributions must contain a verbatim copy of this document. 18 | 19 | The OpenLDAP Foundation may revise this license from time to time. 20 | Each revision is distinguished by a version number. You may use 21 | this Software under terms of this license revision or under the 22 | terms of any subsequent revision of the license. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS 25 | CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, 26 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 27 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 28 | SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) 29 | OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, 30 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 31 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 32 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 33 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 35 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 | POSSIBILITY OF SUCH DAMAGE. 37 | 38 | The names of the authors and copyright holders must not be used in 39 | advertising or otherwise to promote the sale, use or other dealing 40 | in this Software without specific, written prior permission. Title 41 | to copyright in this Software shall at all times remain with copyright 42 | holders. 43 | 44 | OpenLDAP is a registered trademark of the OpenLDAP Foundation. 45 | 46 | Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, 47 | California, USA. All Rights Reserved. Permission to copy and 48 | distribute verbatim copies of this document is granted. 49 | ------------------------------------------------------------------------- 50 | -------------------------------------------------------------------------------- /src/lmdb_sup.erl: -------------------------------------------------------------------------------- 1 | %%------------------------------------------------------------------- 2 | %% This file is part of LMDB - Erlang Lightning MDB API 3 | %% 4 | %% Copyright (c) 2013 by Basho Technologies, Inc. All rights reserved. 5 | %% 6 | %%------------------------------------------------------------------- 7 | %% Redistribution and use in source and binary forms, with or without 8 | %% modification, are permitted only as authorized by the OpenLDAP 9 | %% Public License. 10 | %% 11 | %% A copy of this license is available in the file LICENSE in the 12 | %% top-level directory of the distribution or, alternatively, at 13 | %% . 14 | %% 15 | %% Permission to use, copy, modify, and distribute this software for any 16 | %% purpose with or without fee is hereby granted, provided that the above 17 | %% copyright notice and this permission notice appear in all copies. 18 | %% 19 | %% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 20 | %% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 21 | %% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 22 | %% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 23 | %% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 24 | %% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 25 | %% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 26 | %%------------------------------------------------------------------- 27 | 28 | -module(lmdb_sup). 29 | 30 | -behaviour(supervisor). 31 | 32 | %%==================================================================== 33 | %% EXPORTS 34 | %%==================================================================== 35 | -export([start_link/0]). %% API 36 | -export([init/1]). %% Supervisor callbacks 37 | 38 | %%==================================================================== 39 | %% MACROS 40 | %%==================================================================== 41 | 42 | %% Helper macro for declaring children of supervisor 43 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 44 | 45 | %% =================================================================== 46 | %% API functions 47 | %% =================================================================== 48 | start_link() -> 49 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 50 | 51 | 52 | %% =================================================================== 53 | %% Supervisor callbacks 54 | %% =================================================================== 55 | init([]) -> 56 | {ok, {{one_for_one, 5, 10}, [?CHILD(lmdb, worker)]}}. 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | An Erlang NIF for LMDB 2 | ====================== 3 | 4 | This is an Erlang NIF for OpenLDAP's Lightning Memory-Mapped Database (LMDB) database library. LMDB is an ultra-fast, ultra-compact key-value data store developed by Symas for the OpenLDAP Project. It uses memory-mapped files, so it has the read performance of a pure in-memory database while still offering the persistence of standard disk-based databases, and is only limited to the size of the virtual address space, (it is not limited to the size of physical RAM). LMDB was originally called MDB, but was renamed to avoid confusion with other software associated with the name MDB 5 | 6 | Quick Overview 7 | -------------- 8 | 9 | LMDB is a tiny database with some excellent properties: 10 | * Ordered-map interface (keys are always sorted, supports range lookups) 11 | * Reader/writer transactions: readers don't block writers and writers don't block readers. Writers are fully serialized, so writes are always deadlock-free. 12 | * Read transactions are extremely cheap, and can be performed using no mallocs or any other blocking calls. 13 | * Supports multi-thread and multi-process concurrency, environments may be opened by multiple processes on the same host. 14 | * Multiple sub-databases may be created with transactions covering all sub-databases. 15 | * Memory-mapped, allowing for zero-copy lookup and iteration. 16 | * Maintenance-free, no external process or background cleanup/compaction required. 17 | * No application-level caching. LMDB fully exploits the operating system's buffer cache. 18 | * 32KB of object code and 6KLOC of C. 19 | 20 | The main purpose of this integration is to provide Erlang programmers access to this excellent, and open source friendly, BTREE implementation. 21 | 22 | Requirements 23 | ------------ 24 | * Erlang R14B04+ 25 | * Clang, GCC 4.2+ or MS VisualStudio 2010+ 26 | 27 | Build 28 | ----- 29 | 30 | $ make 31 | 32 | 33 | API 34 | --- 35 | 36 | * `open/1`: equivalent to `lmdb:open(DirName, 10485760)`. 37 | * `open/2`: equivalent to `lmdb:open(DirName, 10485760, 0)`. 38 | * `open/3`: creates a new MDB database. This call also re-open an already existing one. Arguments are: 39 | * DirName: database directory name 40 | * MapSize: database map size (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h)) 41 | * EnvFlags: database environment flags (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h)). The possible values are defined in **lmdb.hrl**. 42 | * `close/2`: closes the database 43 | * `put/2`: inserts Key with value Val into the database. Assumes that the key is not present, 'key_exit' is returned otherwise. 44 | * `get/1`: retrieves the value stored with Key in the database. 45 | * `del/1`: Removes the key-value with key Key from database. 46 | * `update/2` or `upd/2`: inserts Key with value Val into the database if the key is not present, otherwise updates Key to value Val. 47 | * `drop/1`: deletes all key-value pairs in the database. 48 | 49 | 50 | Usage 51 | ----- 52 | 53 | ``` 54 | $ make 55 | $ ./start.sh 56 | %% create a new database 57 | 1> {ok, Handle} = lmdb:open("/tmp/lmdb1"). 58 | 59 | %% insert the key <<"a">> with value <<"1">> 60 | 2> ok = lmdb:put(Handle, <<"a">>, <<"1">>). 61 | 62 | %% try to re-insert the same key <<"a">> 63 | 3> key_exist = lmdb:put(Handle, <<"a">>, <<"2">>). 64 | 65 | %% add a new key-value pair 66 | 4> ok = lmdb:put(Handle, <<"b">>, <<"2">>). 67 | 68 | %% search a non-existing key <<"c">> 69 | 5> none = lmdb:get(Handle, <<"c">>). 70 | 71 | %% retrieve the value for key <<"b">> 72 | 6> {ok, <<"2">>} = lmdb:get(Handle, <<"b">>). 73 | 74 | %% retrieve the value for key <<"a">> 75 | 7> {ok, <<"1">>} = lmdb:get(Handle, <<"a">>). 76 | 77 | %% delete key <<"b">> 78 | 8> ok = lmdb:del(Handle, <<"b">>). 79 | 80 | %% search a non-existing key <<"b">> 81 | 9> none = lmdb:get(Handle, <<"b">>). 82 | 83 | %% delete a non-existing key <<"z">> 84 | 10> none = lmdb:del(Handle, <<"z">>). 85 | 86 | %% ensure key <<"a">>'s value is still <<"1">> 87 | 11> {ok, <<"1">>} = lmdb:get(Handle, <<"a">>). 88 | 89 | %% update the value for key <<"a">> 90 | 12> ok = lmdb:update(Handle, <<"a">>, <<"7">>). 91 | 92 | %% check the new value for key <<"a">> 93 | 13> {ok, <<"7">>} = lmdb:get(Handle, <<"a">>). 94 | 95 | %% delete all key-value pairs in the database 96 | 14> ok = lmdb:drop(Handle). 97 | 98 | %% try to retrieve key <<"a">> value 99 | 15> none = lmdb:get(Handle, <<"a">>). 100 | 101 | %% close the database 102 | 16> ok = lmdb:close(Handle). 103 | ``` 104 | 105 | #### Note: 106 | The code below creates a new database with **80GB** MapSize, **avoids fsync** after each commit (for an "ACI" but not "D" database we trade durability for speed) and uses the experimental **MDB_FIXEDMAP**. 107 | 108 | ``` 109 | {ok, Handle} = lmdb:open("/tmp/lmdb2", 85899345920, ?MDB_NOSYNC bor ?MDB_FIXEDMAP). 110 | ``` 111 | 112 | Performance 113 | ----------- 114 | 115 | See the [microbench](http://highlandsun.com/hyc/mdb/microbench/) against: 116 | * Google's LevelDB (which is slower and can stall unlike Basho's fork of LevelDB) 117 | * SQLite3 118 | * Kyoto TreeDB 119 | * BerkeleyDB 5.x 120 | * btree2n.c (?) 121 | * WiredTiger (?) 122 | 123 | MDB will mmap the entire database into memory which means that if your dataset is larger than 2^32 bytes you'll need to run on a 64-bit arch system. 124 | 125 | 126 | Supported Operating Systems 127 | -------------- 128 | 129 | Should work on: 130 | 131 | * Linux 132 | * Mac OS/X 133 | * *BSD 134 | * Windows 135 | 136 | TODO 137 | ---- 138 | 139 | * Fold over keys and/or values 140 | * Unit tests 141 | * PropEr testing 142 | * Bulk "writing" 143 | * basho_bench driver 144 | * EQC, PULSE testing 145 | * Key expirey 146 | * improve stats 147 | * txn API 148 | * cursor API 149 | * config 150 | * use async_nif affinity 151 | 152 | Other Ideas 153 | ----------- 154 | 155 | * Create a Riak/KV [backend](http://wiki.basho.com/Storage-Backends.html) 156 | * use dups for siblings 157 | * use txn for 2i 158 | * Create a Riak/KV AAE (riak_kv/src/hashtree.erl) alternative using LMDB 159 | 160 | Status 161 | ------ 162 | 163 | Work in progress, not production quality and not supported by Basho Technologies. This is an experiment at this time, nothing more. You are encouraged to contribute code, tests, etc. as you see fit. 164 | 165 | LICENSE 166 | ------- 167 | 168 | LMDB is Copyright (C) 2012 by Aleph Archives and (C) 2013 by Basho Technologies, Inc., and released under the [OpenLDAP](http://www.OpenLDAP.org/license.html) License. 169 | -------------------------------------------------------------------------------- /c_src/midl.h: -------------------------------------------------------------------------------- 1 | /** @file midl.h 2 | * @brief LMDB ID List header file. 3 | * 4 | * This file was originally part of back-bdb but has been 5 | * modified for use in libmdb. Most of the macros defined 6 | * in this file are unused, just left over from the original. 7 | * 8 | * This file is only used internally in libmdb and its definitions 9 | * are not exposed publicly. 10 | */ 11 | /* $OpenLDAP$ */ 12 | /* This work is part of OpenLDAP Software . 13 | * 14 | * Copyright 2000-2016 The OpenLDAP Foundation. 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted only as authorized by the OpenLDAP 19 | * Public License. 20 | * 21 | * A copy of this license is available in the file LICENSE in the 22 | * top-level directory of the distribution or, alternatively, at 23 | * . 24 | */ 25 | 26 | #ifndef _MDB_MIDL_H_ 27 | #define _MDB_MIDL_H_ 28 | 29 | #include 30 | #include 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | /** @defgroup internal LMDB Internals 37 | * @{ 38 | */ 39 | 40 | /** @defgroup idls ID List Management 41 | * @{ 42 | */ 43 | /** A generic unsigned ID number. These were entryIDs in back-bdb. 44 | * Preferably it should have the same size as a pointer. 45 | */ 46 | #ifdef MDB_VL32 47 | typedef uint64_t MDB_ID; 48 | #else 49 | typedef size_t MDB_ID; 50 | #endif 51 | 52 | /** An IDL is an ID List, a sorted array of IDs. The first 53 | * element of the array is a counter for how many actual 54 | * IDs are in the list. In the original back-bdb code, IDLs are 55 | * sorted in ascending order. For libmdb IDLs are sorted in 56 | * descending order. 57 | */ 58 | typedef MDB_ID *MDB_IDL; 59 | 60 | /* IDL sizes - likely should be even bigger 61 | * limiting factors: sizeof(ID), thread stack size 62 | */ 63 | #ifdef MDB_VL32 64 | #define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */ 65 | #else 66 | #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ 67 | #endif 68 | #define MDB_IDL_DB_SIZE (1<. 5 | * 6 | * Copyright 2000-2016 The OpenLDAP Foundation. 7 | * All rights reserved. 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted only as authorized by the OpenLDAP 11 | * Public License. 12 | * 13 | * A copy of this license is available in the file LICENSE in the 14 | * top-level directory of the distribution or, alternatively, at 15 | * . 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "midl.h" 24 | 25 | /** @defgroup internal LMDB Internals 26 | * @{ 27 | */ 28 | /** @defgroup idls ID List Management 29 | * @{ 30 | */ 31 | #define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) 32 | 33 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) 34 | { 35 | /* 36 | * binary search of id in ids 37 | * if found, returns position of id 38 | * if not found, returns first position greater than id 39 | */ 40 | unsigned base = 0; 41 | unsigned cursor = 1; 42 | int val = 0; 43 | unsigned n = ids[0]; 44 | 45 | while( 0 < n ) { 46 | unsigned pivot = n >> 1; 47 | cursor = base + pivot + 1; 48 | val = CMP( ids[cursor], id ); 49 | 50 | if( val < 0 ) { 51 | n = pivot; 52 | 53 | } else if ( val > 0 ) { 54 | base = cursor; 55 | n -= pivot + 1; 56 | 57 | } else { 58 | return cursor; 59 | } 60 | } 61 | 62 | if( val > 0 ) { 63 | ++cursor; 64 | } 65 | return cursor; 66 | } 67 | 68 | #if 0 /* superseded by append/sort */ 69 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) 70 | { 71 | unsigned x, i; 72 | 73 | x = mdb_midl_search( ids, id ); 74 | assert( x > 0 ); 75 | 76 | if( x < 1 ) { 77 | /* internal error */ 78 | return -2; 79 | } 80 | 81 | if ( x <= ids[0] && ids[x] == id ) { 82 | /* duplicate */ 83 | assert(0); 84 | return -1; 85 | } 86 | 87 | if ( ++ids[0] >= MDB_IDL_DB_MAX ) { 88 | /* no room */ 89 | --ids[0]; 90 | return -2; 91 | 92 | } else { 93 | /* insert id */ 94 | for (i=ids[0]; i>x; i--) 95 | ids[i] = ids[i-1]; 96 | ids[x] = id; 97 | } 98 | 99 | return 0; 100 | } 101 | #endif 102 | 103 | MDB_IDL mdb_midl_alloc(int num) 104 | { 105 | MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); 106 | if (ids) { 107 | *ids++ = num; 108 | *ids = 0; 109 | } 110 | return ids; 111 | } 112 | 113 | void mdb_midl_free(MDB_IDL ids) 114 | { 115 | if (ids) 116 | free(ids-1); 117 | } 118 | 119 | void mdb_midl_shrink( MDB_IDL *idp ) 120 | { 121 | MDB_IDL ids = *idp; 122 | if (*(--ids) > MDB_IDL_UM_MAX && 123 | (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) 124 | { 125 | *ids++ = MDB_IDL_UM_MAX; 126 | *idp = ids; 127 | } 128 | } 129 | 130 | static int mdb_midl_grow( MDB_IDL *idp, int num ) 131 | { 132 | MDB_IDL idn = *idp-1; 133 | /* grow it */ 134 | idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); 135 | if (!idn) 136 | return ENOMEM; 137 | *idn++ += num; 138 | *idp = idn; 139 | return 0; 140 | } 141 | 142 | int mdb_midl_need( MDB_IDL *idp, unsigned num ) 143 | { 144 | MDB_IDL ids = *idp; 145 | num += ids[0]; 146 | if (num > ids[-1]) { 147 | num = (num + num/4 + (256 + 2)) & -256; 148 | if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) 149 | return ENOMEM; 150 | *ids++ = num - 2; 151 | *idp = ids; 152 | } 153 | return 0; 154 | } 155 | 156 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) 157 | { 158 | MDB_IDL ids = *idp; 159 | /* Too big? */ 160 | if (ids[0] >= ids[-1]) { 161 | if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) 162 | return ENOMEM; 163 | ids = *idp; 164 | } 165 | ids[0]++; 166 | ids[ids[0]] = id; 167 | return 0; 168 | } 169 | 170 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) 171 | { 172 | MDB_IDL ids = *idp; 173 | /* Too big? */ 174 | if (ids[0] + app[0] >= ids[-1]) { 175 | if (mdb_midl_grow(idp, app[0])) 176 | return ENOMEM; 177 | ids = *idp; 178 | } 179 | memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); 180 | ids[0] += app[0]; 181 | return 0; 182 | } 183 | 184 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) 185 | { 186 | MDB_ID *ids = *idp, len = ids[0]; 187 | /* Too big? */ 188 | if (len + n > ids[-1]) { 189 | if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) 190 | return ENOMEM; 191 | ids = *idp; 192 | } 193 | ids[0] = len + n; 194 | ids += len; 195 | while (n) 196 | ids[n--] = id++; 197 | return 0; 198 | } 199 | 200 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) 201 | { 202 | MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; 203 | idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ 204 | old_id = idl[j]; 205 | while (i) { 206 | merge_id = merge[i--]; 207 | for (; old_id < merge_id; old_id = idl[--j]) 208 | idl[k--] = old_id; 209 | idl[k--] = merge_id; 210 | } 211 | idl[0] = total; 212 | } 213 | 214 | /* Quicksort + Insertion sort for small arrays */ 215 | 216 | #define SMALL 8 217 | #define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } 218 | 219 | void 220 | mdb_midl_sort( MDB_IDL ids ) 221 | { 222 | /* Max possible depth of int-indexed tree * 2 items/level */ 223 | int istack[sizeof(int)*CHAR_BIT * 2]; 224 | int i,j,k,l,ir,jstack; 225 | MDB_ID a, itmp; 226 | 227 | ir = (int)ids[0]; 228 | l = 1; 229 | jstack = 0; 230 | for(;;) { 231 | if (ir - l < SMALL) { /* Insertion sort */ 232 | for (j=l+1;j<=ir;j++) { 233 | a = ids[j]; 234 | for (i=j-1;i>=1;i--) { 235 | if (ids[i] >= a) break; 236 | ids[i+1] = ids[i]; 237 | } 238 | ids[i+1] = a; 239 | } 240 | if (jstack == 0) break; 241 | ir = istack[jstack--]; 242 | l = istack[jstack--]; 243 | } else { 244 | k = (l + ir) >> 1; /* Choose median of left, center, right */ 245 | MIDL_SWAP(ids[k], ids[l+1]); 246 | if (ids[l] < ids[ir]) { 247 | MIDL_SWAP(ids[l], ids[ir]); 248 | } 249 | if (ids[l+1] < ids[ir]) { 250 | MIDL_SWAP(ids[l+1], ids[ir]); 251 | } 252 | if (ids[l] < ids[l+1]) { 253 | MIDL_SWAP(ids[l], ids[l+1]); 254 | } 255 | i = l+1; 256 | j = ir; 257 | a = ids[l+1]; 258 | for(;;) { 259 | do i++; while(ids[i] > a); 260 | do j--; while(ids[j] < a); 261 | if (j < i) break; 262 | MIDL_SWAP(ids[i],ids[j]); 263 | } 264 | ids[l+1] = ids[j]; 265 | ids[j] = a; 266 | jstack += 2; 267 | if (ir-i+1 >= j-l) { 268 | istack[jstack] = ir; 269 | istack[jstack-1] = i; 270 | ir = j-1; 271 | } else { 272 | istack[jstack] = j-1; 273 | istack[jstack-1] = l; 274 | l = i; 275 | } 276 | } 277 | } 278 | } 279 | 280 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) 281 | { 282 | /* 283 | * binary search of id in ids 284 | * if found, returns position of id 285 | * if not found, returns first position greater than id 286 | */ 287 | unsigned base = 0; 288 | unsigned cursor = 1; 289 | int val = 0; 290 | unsigned n = (unsigned)ids[0].mid; 291 | 292 | while( 0 < n ) { 293 | unsigned pivot = n >> 1; 294 | cursor = base + pivot + 1; 295 | val = CMP( id, ids[cursor].mid ); 296 | 297 | if( val < 0 ) { 298 | n = pivot; 299 | 300 | } else if ( val > 0 ) { 301 | base = cursor; 302 | n -= pivot + 1; 303 | 304 | } else { 305 | return cursor; 306 | } 307 | } 308 | 309 | if( val > 0 ) { 310 | ++cursor; 311 | } 312 | return cursor; 313 | } 314 | 315 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) 316 | { 317 | unsigned x, i; 318 | 319 | x = mdb_mid2l_search( ids, id->mid ); 320 | 321 | if( x < 1 ) { 322 | /* internal error */ 323 | return -2; 324 | } 325 | 326 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) { 327 | /* duplicate */ 328 | return -1; 329 | } 330 | 331 | if ( ids[0].mid >= MDB_IDL_UM_MAX ) { 332 | /* too big */ 333 | return -2; 334 | 335 | } else { 336 | /* insert id */ 337 | ids[0].mid++; 338 | for (i=(unsigned)ids[0].mid; i>x; i--) 339 | ids[i] = ids[i-1]; 340 | ids[x] = *id; 341 | } 342 | 343 | return 0; 344 | } 345 | 346 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) 347 | { 348 | /* Too big? */ 349 | if (ids[0].mid >= MDB_IDL_UM_MAX) { 350 | return -2; 351 | } 352 | ids[0].mid++; 353 | ids[ids[0].mid] = *id; 354 | return 0; 355 | } 356 | 357 | #ifdef MDB_VL32 358 | unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id ) 359 | { 360 | /* 361 | * binary search of id in ids 362 | * if found, returns position of id 363 | * if not found, returns first position greater than id 364 | */ 365 | unsigned base = 0; 366 | unsigned cursor = 1; 367 | int val = 0; 368 | unsigned n = (unsigned)ids[0].mid; 369 | 370 | while( 0 < n ) { 371 | unsigned pivot = n >> 1; 372 | cursor = base + pivot + 1; 373 | val = CMP( id, ids[cursor].mid ); 374 | 375 | if( val < 0 ) { 376 | n = pivot; 377 | 378 | } else if ( val > 0 ) { 379 | base = cursor; 380 | n -= pivot + 1; 381 | 382 | } else { 383 | return cursor; 384 | } 385 | } 386 | 387 | if( val > 0 ) { 388 | ++cursor; 389 | } 390 | return cursor; 391 | } 392 | 393 | int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id ) 394 | { 395 | unsigned x, i; 396 | 397 | x = mdb_mid3l_search( ids, id->mid ); 398 | 399 | if( x < 1 ) { 400 | /* internal error */ 401 | return -2; 402 | } 403 | 404 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) { 405 | /* duplicate */ 406 | return -1; 407 | } 408 | 409 | /* insert id */ 410 | ids[0].mid++; 411 | for (i=(unsigned)ids[0].mid; i>x; i--) 412 | ids[i] = ids[i-1]; 413 | ids[x] = *id; 414 | 415 | return 0; 416 | } 417 | #endif /* MDB_VL32 */ 418 | 419 | /** @} */ 420 | /** @} */ 421 | -------------------------------------------------------------------------------- /src/lmdb.erl: -------------------------------------------------------------------------------- 1 | %%------------------------------------------------------------------- 2 | %% This file is part of LMDB - Erlang Lightning MDB API 3 | %% 4 | %% Copyright (c) 2012 by Aleph Archives. All rights reserved. 5 | %% Copyright (c) 2013 by Basho Technologies, Inc. All rights reserved. 6 | %% 7 | %%------------------------------------------------------------------- 8 | %% Redistribution and use in source and binary forms, with or without 9 | %% modification, are permitted only as authorized by the OpenLDAP 10 | %% Public License. 11 | %% 12 | %% A copy of this license is available in the file LICENSE in the 13 | %% top-level directory of the distribution or, alternatively, at 14 | %% . 15 | %% 16 | %% Permission to use, copy, modify, and distribute this software for any 17 | %% purpose with or without fee is hereby granted, provided that the above 18 | %% copyright notice and this permission notice appear in all copies. 19 | %% 20 | %% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 21 | %% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 22 | %% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 23 | %% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 24 | %% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 25 | %% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 26 | %% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 27 | %%------------------------------------------------------------------- 28 | 29 | -module(lmdb). 30 | 31 | %%==================================================================== 32 | %% EXPORTS 33 | %%==================================================================== 34 | -export([ 35 | open/1, 36 | open/2, 37 | open/3, 38 | 39 | close/1, 40 | 41 | put/3, 42 | get/2, 43 | txn_begin/1, 44 | txn_commit/1, 45 | txn_abort/1, 46 | del/2, 47 | update/3, upd/3, 48 | 49 | drop/1 50 | ]). 51 | 52 | 53 | %% internal export (ex. spawn, apply) 54 | -on_load(init/0). 55 | 56 | %% config for testing 57 | -ifdef(TEST). 58 | -ifdef(EQC). 59 | -include_lib("eqc/include/eqc.hrl"). 60 | -define(QC_OUT(P), eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 61 | -endif. 62 | -include_lib("eunit/include/eunit.hrl"). 63 | -endif. 64 | 65 | %%==================================================================== 66 | %% Includes 67 | %%==================================================================== 68 | -include("lmdb.hrl"). 69 | -include("async_nif.hrl"). 70 | 71 | %%==================================================================== 72 | %% MACROS 73 | %%==================================================================== 74 | -define(LMDB_DRIVER_NAME, "lmdb"). 75 | -define(NOT_LOADED, not_loaded(?LINE)). 76 | -define(MDB_MAP_SIZE, 2147483648). %% 2GB in bytes 77 | 78 | %%==================================================================== 79 | %% PUBLIC API 80 | %%==================================================================== 81 | 82 | %%-------------------------------------------------------------------- 83 | %% @doc Create a new MDB database 84 | %% @end 85 | %%-------------------------------------------------------------------- 86 | open(DirName) -> 87 | open(DirName, ?MDB_MAP_SIZE). 88 | open(DirName, MapSize) 89 | when is_integer(MapSize) 90 | andalso MapSize > 0 -> 91 | open(DirName, MapSize, 0). 92 | open(DirName, MapSize, EnvFlags) 93 | when is_integer(MapSize) andalso MapSize > 0 andalso 94 | is_integer(EnvFlags) andalso EnvFlags >= 0 -> 95 | %% ensure directory exists 96 | ok = filelib:ensure_dir(filename:join([DirName, "x"])), 97 | ?ASYNC_NIF_CALL(fun open/4, [DirName, MapSize, EnvFlags]). 98 | 99 | open(_AsyncRef, _DirName, _MapSize, _EnvFlags) -> 100 | ?NOT_LOADED. 101 | 102 | close(Handle) -> 103 | ?ASYNC_NIF_CALL(fun close/2, [Handle]). 104 | 105 | close(_AsyncRef, _Handle) -> 106 | ?NOT_LOADED. 107 | 108 | put(Handle, Key, Val) 109 | when is_binary(Key) andalso is_binary(Val) -> 110 | ?ASYNC_NIF_CALL(fun put/4, [Handle, Key, Val]). 111 | 112 | put(_AsyncRef, _Handle, _Key, _Val) -> 113 | ?NOT_LOADED. 114 | 115 | get(Handle, Key) 116 | when is_binary(Key) -> 117 | ?ASYNC_NIF_CALL(fun get/3, [Handle, Key]). 118 | 119 | get(_AsyncRef, _Handle, _Key) -> 120 | ?NOT_LOADED. 121 | 122 | txn_begin(Handle) -> 123 | ?ASYNC_NIF_CALL(fun txn_begin/2, [Handle]). 124 | 125 | txn_begin(_AsyncRef, _Handle) -> 126 | ?NOT_LOADED. 127 | 128 | txn_commit(Handle) -> 129 | ?ASYNC_NIF_CALL(fun txn_commit/2, [Handle]). 130 | 131 | txn_commit(_AsyncRef, _Handle) -> 132 | ?NOT_LOADED. 133 | 134 | txn_abort(Handle) -> 135 | ?ASYNC_NIF_CALL(fun txn_abort/2, [Handle]). 136 | 137 | txn_abort(_AsyncRef, _Handle) -> 138 | ?NOT_LOADED. 139 | 140 | del(Handle, Key) 141 | when is_binary(Key) -> 142 | ?ASYNC_NIF_CALL(fun del/3, [Handle, Key]). 143 | 144 | del(_AsyncRef, _Handle, _Key) -> 145 | ?NOT_LOADED. 146 | 147 | upd(Handle, Key, Val) -> 148 | update(Handle, Key, Val). 149 | 150 | update(Handle, Key, Val) 151 | when is_binary(Key) andalso is_binary(Val) -> 152 | ?ASYNC_NIF_CALL(fun update/4, [Handle, Key, Val]). 153 | 154 | update(_AsyncRef, _Handle, _Key, _Val) -> 155 | ?NOT_LOADED. 156 | 157 | drop(Handle) -> 158 | ?ASYNC_NIF_CALL(fun drop/2, [Handle]). 159 | 160 | drop(_AsyncRef, _Handle) -> 161 | ?NOT_LOADED. 162 | 163 | %%==================================================================== 164 | %% PRIVATE API 165 | %%==================================================================== 166 | 167 | %%-------------------------------------------------------------------- 168 | %% @doc 169 | %% @end 170 | %%-------------------------------------------------------------------- 171 | init() -> 172 | PrivDir = case code:priv_dir(?MODULE) of 173 | {error, _} -> 174 | EbinDir = filename:dirname(code:which(?MODULE)), 175 | AppPath = filename:dirname(EbinDir), 176 | filename:join(AppPath, "priv"); 177 | Path -> 178 | Path 179 | end, 180 | erlang:load_nif(filename:join(PrivDir, ?LMDB_DRIVER_NAME), 0). 181 | 182 | 183 | %%-------------------------------------------------------------------- 184 | %% @doc 185 | %% @end 186 | %%-------------------------------------------------------------------- 187 | not_loaded(Line) -> 188 | erlang:nif_error({not_loaded, [{module, ?MODULE}, {line, Line}]}). 189 | 190 | %% =================================================================== 191 | %% EUnit tests 192 | %% =================================================================== 193 | -ifdef(TEST). 194 | 195 | open_test_db() -> 196 | {ok, CWD} = file:get_cwd(), 197 | DataDir = filename:join([CWD, "test", "eunit"]), 198 | ?cmd("rm -rf " ++ DataDir), 199 | ?assertMatch(ok, filelib:ensure_dir(filename:join([DataDir, "x"]))), 200 | {ok, Handle} = ?MODULE:open(DataDir, 2147483648), 201 | [?MODULE:upd(Handle, crypto:hash(sha, <>), 202 | crypto:rand_bytes(crypto:rand_uniform(128, 4096))) || 203 | X <- lists:seq(1, 10)], 204 | Handle. 205 | 206 | basics_test_() -> 207 | {setup, 208 | fun() -> 209 | open_test_db() 210 | end, 211 | fun(Handle) -> 212 | ok = ?MODULE:close(Handle) 213 | end, 214 | fun(Handle) -> 215 | {inorder, 216 | [{"open and close a database", 217 | fun() -> 218 | Handle = open_test_db() 219 | end}, 220 | {"create, then drop an empty database", 221 | fun() -> 222 | Handle = open_test_db(), 223 | ?assertMatch(ok, ?MODULE:drop(Handle)) 224 | end}, 225 | {"create, put an item, get it, then drop the database", 226 | fun() -> 227 | Handle = open_test_db(), 228 | ?assertMatch(ok, ?MODULE:put(Handle, <<"a">>, <<"apple">>)), 229 | ?assertMatch(ok, ?MODULE:put(Handle, <<"b">>, <<"boy">>)), 230 | ?assertMatch(ok, ?MODULE:put(Handle, <<"c">>, <<"cat">>)), 231 | ?assertMatch({ok, <<"apple">>}, ?MODULE:get(Handle, <<"a">>)), 232 | ?assertMatch(ok, ?MODULE:update(Handle, <<"a">>, <<"ant">>)), 233 | ?assertMatch({ok, <<"ant">>}, ?MODULE:get(Handle, <<"a">>)), 234 | ?assertMatch(ok, ?MODULE:del(Handle, <<"a">>)), 235 | ?assertMatch(not_found, ?MODULE:get(Handle, <<"a">>)), 236 | ?assertMatch(ok, ?MODULE:drop(Handle)) 237 | end} 238 | ]} 239 | end}. 240 | 241 | -ifdef(EQC). 242 | 243 | qc(P) -> 244 | ?assert(eqc:quickcheck(?QC_OUT(P))). 245 | 246 | keys() -> 247 | eqc_gen:non_empty(list(eqc_gen:non_empty(binary()))). 248 | 249 | values() -> 250 | eqc_gen:non_empty(list(binary())). 251 | 252 | ops(Keys, Values) -> 253 | {oneof([put, delete]), oneof(Keys), oneof(Values)}. 254 | 255 | apply_kv_ops([], _Handle, Acc0) -> 256 | Acc0; 257 | apply_kv_ops([{put, K, V} | Rest], Handle, Acc0) -> 258 | ok = ?MODULE:put(Handle, K, V), 259 | apply_kv_ops(Rest, Handle, orddict:store(K, V, Acc0)); 260 | apply_kv_ops([{del, K, _} | Rest], Handle, Acc0) -> 261 | ok = case ?MODULE:del(Handle, K) of 262 | ok -> 263 | ok; 264 | not_found -> 265 | ok; 266 | Else -> 267 | Else 268 | end, 269 | apply_kv_ops(Rest, Handle, orddict:store(K, deleted, Acc0)). 270 | 271 | prop_put_delete() -> 272 | ?LET({Keys, Values}, {keys(), values()}, 273 | ?FORALL(Ops, eqc_gen:non_empty(list(ops(Keys, Values))), 274 | begin 275 | {ok, CWD} = file:get_cwd(), 276 | DataDir = filename:join([CWD, "test", "eqc"]), 277 | ?cmd("rm -rf " ++ DataDir), 278 | ok = filelib:ensure_dir(filename:join([DataDir, "x"])), 279 | {ok, Handle} = ?MODULE:open(DataDir, 2147483648), 280 | try 281 | Model = apply_kv_ops(Ops, Handle, []), 282 | 283 | %% Validate that all deleted values return not_found 284 | F = fun({K, deleted}) -> 285 | ?assertEqual(not_found, ?MODULE:get(Handle, K)); 286 | ({K, V}) -> 287 | ?assertEqual({ok, V}, ?MODULE:get(Handle, K)) 288 | end, 289 | lists:map(F, Model), 290 | true 291 | after 292 | ?MODULE:close(Handle) 293 | end 294 | end)). 295 | 296 | prop_put_delete_test_() -> 297 | {timeout, 3*60, fun() -> qc(prop_put_delete()) end}. 298 | 299 | -endif. 300 | -endif. 301 | -------------------------------------------------------------------------------- /c_src/khash.h: -------------------------------------------------------------------------------- 1 | /* The MIT License 2 | 3 | Copyright (c) 2008, 2009, 2011 by Attractive Chaos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 20 | BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 21 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | */ 25 | 26 | /* 27 | An example: 28 | 29 | #include "khash.h" 30 | KHASH_MAP_INIT_INT(32, char) 31 | int main() { 32 | int ret, is_missing; 33 | khiter_t k; 34 | khash_t(32) *h = kh_init(32); 35 | k = kh_put(32, h, 5, &ret); 36 | kh_value(h, k) = 10; 37 | k = kh_get(32, h, 10); 38 | is_missing = (k == kh_end(h)); 39 | k = kh_get(32, h, 5); 40 | kh_del(32, h, k); 41 | for (k = kh_begin(h); k != kh_end(h); ++k) 42 | if (kh_exist(h, k)) kh_value(h, k) = 1; 43 | kh_destroy(32, h); 44 | return 0; 45 | } 46 | */ 47 | 48 | /* 49 | 2011-12-29 (0.2.7): 50 | 51 | * Minor code clean up; no actual effect. 52 | 53 | 2011-09-16 (0.2.6): 54 | 55 | * The capacity is a power of 2. This seems to dramatically improve the 56 | speed for simple keys. Thank Zilong Tan for the suggestion. Reference: 57 | 58 | - http://code.google.com/p/ulib/ 59 | - http://nothings.org/computer/judy/ 60 | 61 | * Allow to optionally use linear probing which usually has better 62 | performance for random input. Double hashing is still the default as it 63 | is more robust to certain non-random input. 64 | 65 | * Added Wang's integer hash function (not used by default). This hash 66 | function is more robust to certain non-random input. 67 | 68 | 2011-02-14 (0.2.5): 69 | 70 | * Allow to declare global functions. 71 | 72 | 2009-09-26 (0.2.4): 73 | 74 | * Improve portability 75 | 76 | 2008-09-19 (0.2.3): 77 | 78 | * Corrected the example 79 | * Improved interfaces 80 | 81 | 2008-09-11 (0.2.2): 82 | 83 | * Improved speed a little in kh_put() 84 | 85 | 2008-09-10 (0.2.1): 86 | 87 | * Added kh_clear() 88 | * Fixed a compiling error 89 | 90 | 2008-09-02 (0.2.0): 91 | 92 | * Changed to token concatenation which increases flexibility. 93 | 94 | 2008-08-31 (0.1.2): 95 | 96 | * Fixed a bug in kh_get(), which has not been tested previously. 97 | 98 | 2008-08-31 (0.1.1): 99 | 100 | * Added destructor 101 | */ 102 | 103 | 104 | #ifndef __AC_KHASH_H 105 | #define __AC_KHASH_H 106 | 107 | /*! 108 | @header 109 | 110 | Generic hash table library. 111 | */ 112 | 113 | #define AC_VERSION_KHASH_H "0.2.6" 114 | 115 | #include 116 | #include 117 | #include 118 | 119 | /* compiler specific configuration */ 120 | 121 | #if UINT_MAX == 0xffffffffu 122 | typedef unsigned int khint32_t; 123 | #elif ULONG_MAX == 0xffffffffu 124 | typedef unsigned long khint32_t; 125 | #endif 126 | 127 | #if ULONG_MAX == ULLONG_MAX 128 | typedef unsigned long khint64_t; 129 | #else 130 | typedef unsigned long long khint64_t; 131 | #endif 132 | 133 | #ifdef _MSC_VER 134 | #define kh_inline __inline 135 | #else 136 | #define kh_inline inline 137 | #endif 138 | 139 | typedef khint32_t khint_t; 140 | typedef khint_t khiter_t; 141 | 142 | #define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) 143 | #define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) 144 | #define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) 145 | #define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) 146 | #define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) 147 | #define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) 148 | #define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) 149 | 150 | #ifdef KHASH_LINEAR 151 | #define __ac_inc(k, m) 1 152 | #else 153 | #define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m) 154 | #endif 155 | 156 | #define __ac_fsize(m) ((m) < 16? 1 : (m)>>4) 157 | 158 | #ifndef kroundup32 159 | #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) 160 | #endif 161 | 162 | #ifndef kcalloc 163 | #define kcalloc(N,Z) calloc(N,Z) 164 | #endif 165 | #ifndef kmalloc 166 | #define kmalloc(Z) malloc(Z) 167 | #endif 168 | #ifndef krealloc 169 | #define krealloc(P,Z) realloc(P,Z) 170 | #endif 171 | #ifndef kfree 172 | #define kfree(P) free(P) 173 | #endif 174 | 175 | static const double __ac_HASH_UPPER = 0.77; 176 | 177 | #define __KHASH_TYPE(name, khkey_t, khval_t) \ 178 | typedef struct { \ 179 | khint_t n_buckets, size, n_occupied, upper_bound; \ 180 | khint32_t *flags; \ 181 | khkey_t *keys; \ 182 | khval_t *vals; \ 183 | } kh_##name##_t; 184 | 185 | #define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \ 186 | extern kh_##name##_t *kh_init_##name(void); \ 187 | extern void kh_destroy_##name(kh_##name##_t *h); \ 188 | extern void kh_clear_##name(kh_##name##_t *h); \ 189 | extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ 190 | extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ 191 | extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ 192 | extern void kh_del_##name(kh_##name##_t *h, khint_t x); 193 | 194 | #define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ 195 | SCOPE kh_##name##_t *kh_init_##name(void) { \ 196 | return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \ 197 | } \ 198 | SCOPE void kh_destroy_##name(kh_##name##_t *h) \ 199 | { \ 200 | if (h) { \ 201 | kfree((void *)h->keys); kfree(h->flags); \ 202 | kfree((void *)h->vals); \ 203 | kfree(h); \ 204 | } \ 205 | } \ 206 | SCOPE void kh_clear_##name(kh_##name##_t *h) \ 207 | { \ 208 | if (h && h->flags) { \ 209 | memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \ 210 | h->size = h->n_occupied = 0; \ 211 | } \ 212 | } \ 213 | SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ 214 | { \ 215 | if (h->n_buckets) { \ 216 | khint_t inc, k, i, last, mask; \ 217 | mask = h->n_buckets - 1; \ 218 | k = __hash_func(key); i = k & mask; \ 219 | inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \ 220 | while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ 221 | i = (i + inc) & mask; \ 222 | if (i == last) return h->n_buckets; \ 223 | } \ 224 | return __ac_iseither(h->flags, i)? h->n_buckets : i; \ 225 | } else return 0; \ 226 | } \ 227 | SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ 228 | { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ 229 | khint32_t *new_flags = 0; \ 230 | khint_t j = 1; \ 231 | { \ 232 | kroundup32(new_n_buckets); \ 233 | if (new_n_buckets < 4) new_n_buckets = 4; \ 234 | if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \ 235 | else { /* hash table size to be changed (shrink or expand); rehash */ \ 236 | new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ 237 | if (!new_flags) return -1; \ 238 | memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ 239 | if (h->n_buckets < new_n_buckets) { /* expand */ \ 240 | khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ 241 | if (!new_keys) return -1; \ 242 | h->keys = new_keys; \ 243 | if (kh_is_map) { \ 244 | khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ 245 | if (!new_vals) return -1; \ 246 | h->vals = new_vals; \ 247 | } \ 248 | } /* otherwise shrink */ \ 249 | } \ 250 | } \ 251 | if (j) { /* rehashing is needed */ \ 252 | for (j = 0; j != h->n_buckets; ++j) { \ 253 | if (__ac_iseither(h->flags, j) == 0) { \ 254 | khkey_t key = h->keys[j]; \ 255 | khval_t val; \ 256 | khint_t new_mask; \ 257 | new_mask = new_n_buckets - 1; \ 258 | if (kh_is_map) val = h->vals[j]; \ 259 | __ac_set_isdel_true(h->flags, j); \ 260 | while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \ 261 | khint_t inc, k, i; \ 262 | k = __hash_func(key); \ 263 | i = k & new_mask; \ 264 | inc = __ac_inc(k, new_mask); \ 265 | while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \ 266 | __ac_set_isempty_false(new_flags, i); \ 267 | if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \ 268 | { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ 269 | if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ 270 | __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \ 271 | } else { /* write the element and jump out of the loop */ \ 272 | h->keys[i] = key; \ 273 | if (kh_is_map) h->vals[i] = val; \ 274 | break; \ 275 | } \ 276 | } \ 277 | } \ 278 | } \ 279 | if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \ 280 | h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ 281 | if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ 282 | } \ 283 | kfree(h->flags); /* free the working space */ \ 284 | h->flags = new_flags; \ 285 | h->n_buckets = new_n_buckets; \ 286 | h->n_occupied = h->size; \ 287 | h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ 288 | } \ 289 | return 0; \ 290 | } \ 291 | SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ 292 | { \ 293 | khint_t x; \ 294 | if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \ 295 | if (h->n_buckets > (h->size<<1)) { \ 296 | if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \ 297 | *ret = -1; return h->n_buckets; \ 298 | } \ 299 | } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \ 300 | *ret = -1; return h->n_buckets; \ 301 | } \ 302 | } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \ 303 | { \ 304 | khint_t inc, k, i, site, last, mask = h->n_buckets - 1; \ 305 | x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \ 306 | if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \ 307 | else { \ 308 | inc = __ac_inc(k, mask); last = i; \ 309 | while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ 310 | if (__ac_isdel(h->flags, i)) site = i; \ 311 | i = (i + inc) & mask; \ 312 | if (i == last) { x = site; break; } \ 313 | } \ 314 | if (x == h->n_buckets) { \ 315 | if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ 316 | else x = i; \ 317 | } \ 318 | } \ 319 | } \ 320 | if (__ac_isempty(h->flags, x)) { /* not present at all */ \ 321 | h->keys[x] = key; \ 322 | __ac_set_isboth_false(h->flags, x); \ 323 | ++h->size; ++h->n_occupied; \ 324 | *ret = 1; \ 325 | } else if (__ac_isdel(h->flags, x)) { /* deleted */ \ 326 | h->keys[x] = key; \ 327 | __ac_set_isboth_false(h->flags, x); \ 328 | ++h->size; \ 329 | *ret = 2; \ 330 | } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \ 331 | return x; \ 332 | } \ 333 | SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ 334 | { \ 335 | if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ 336 | __ac_set_isdel_true(h->flags, x); \ 337 | --h->size; \ 338 | } \ 339 | } 340 | 341 | #define KHASH_DECLARE(name, khkey_t, khval_t) \ 342 | __KHASH_TYPE(name, khkey_t, khval_t) \ 343 | __KHASH_PROTOTYPES(name, khkey_t, khval_t) 344 | 345 | #define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ 346 | __KHASH_TYPE(name, khkey_t, khval_t) \ 347 | __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) 348 | 349 | #define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ 350 | KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) 351 | 352 | /* --- BEGIN OF HASH FUNCTIONS --- */ 353 | 354 | /*! @function 355 | @abstract Integer hash function 356 | @param key The integer [khint32_t] 357 | @return The hash value [khint_t] 358 | */ 359 | #define kh_int_hash_func(key) (khint32_t)(key) 360 | /*! @function 361 | @abstract Integer comparison function 362 | */ 363 | #define kh_int_hash_equal(a, b) ((a) == (b)) 364 | /*! @function 365 | @abstract 64-bit integer hash function 366 | @param key The integer [khint64_t] 367 | @return The hash value [khint_t] 368 | */ 369 | #define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) 370 | /*! @function 371 | @abstract 64-bit integer comparison function 372 | */ 373 | #define kh_int64_hash_equal(a, b) ((a) == (b)) 374 | /*! @function 375 | @abstract Pointer hash function 376 | @param key The integer void * 377 | @return The hash value [khint_t] 378 | */ 379 | #define kh_ptr_hash_func(key) (khint32_t)(key) 380 | /*! @function 381 | @abstract Pointer comparison function 382 | */ 383 | #define kh_ptr_hash_equal(a, b) ((a) == (b)) 384 | /*! @function 385 | @abstract 64-bit pointer hash function 386 | @param key The integer void * 387 | @return The hash value [khint_t] 388 | */ 389 | #define kh_ptr64_hash_func(key) (khint32_t)(((khint64_t)key)>>33^((khint64_t)key)^((khint64_t)key)<<11) 390 | /*! @function 391 | @abstract 64-bit pointer comparison function 392 | */ 393 | #define kh_ptr64_hash_equal(a, b) ((a) == (b)) 394 | /*! @function 395 | @abstract const char* hash function 396 | @param s Pointer to a null terminated string 397 | @return The hash value 398 | */ 399 | static kh_inline khint_t __ac_X31_hash_string(const char *s) 400 | { 401 | khint_t h = (khint_t)*s; 402 | if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s; 403 | return h; 404 | } 405 | /*! @function 406 | @abstract Another interface to const char* hash function 407 | @param key Pointer to a null terminated string [const char*] 408 | @return The hash value [khint_t] 409 | */ 410 | #define kh_str_hash_func(key) __ac_X31_hash_string(key) 411 | /*! @function 412 | @abstract Const char* comparison function 413 | */ 414 | #define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) 415 | 416 | static kh_inline khint_t __ac_Wang_hash(khint_t key) 417 | { 418 | key += ~(key << 15); 419 | key ^= (key >> 10); 420 | key += (key << 3); 421 | key ^= (key >> 6); 422 | key += ~(key << 11); 423 | key ^= (key >> 16); 424 | return key; 425 | } 426 | #define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key) 427 | 428 | /* --- END OF HASH FUNCTIONS --- */ 429 | 430 | /* Other convenient macros... */ 431 | 432 | /*! 433 | @abstract Type of the hash table. 434 | @param name Name of the hash table [symbol] 435 | */ 436 | #define khash_t(name) kh_##name##_t 437 | 438 | /*! @function 439 | @abstract Initiate a hash table. 440 | @param name Name of the hash table [symbol] 441 | @return Pointer to the hash table [khash_t(name)*] 442 | */ 443 | #define kh_init(name) kh_init_##name() 444 | 445 | /*! @function 446 | @abstract Destroy a hash table. 447 | @param name Name of the hash table [symbol] 448 | @param h Pointer to the hash table [khash_t(name)*] 449 | */ 450 | #define kh_destroy(name, h) kh_destroy_##name(h) 451 | 452 | /*! @function 453 | @abstract Reset a hash table without deallocating memory. 454 | @param name Name of the hash table [symbol] 455 | @param h Pointer to the hash table [khash_t(name)*] 456 | */ 457 | #define kh_clear(name, h) kh_clear_##name(h) 458 | 459 | /*! @function 460 | @abstract Resize a hash table. 461 | @param name Name of the hash table [symbol] 462 | @param h Pointer to the hash table [khash_t(name)*] 463 | @param s New size [khint_t] 464 | */ 465 | #define kh_resize(name, h, s) kh_resize_##name(h, s) 466 | 467 | /*! @function 468 | @abstract Insert a key to the hash table. 469 | @param name Name of the hash table [symbol] 470 | @param h Pointer to the hash table [khash_t(name)*] 471 | @param k Key [type of keys] 472 | @param r Extra return code: 0 if the key is present in the hash table; 473 | 1 if the bucket is empty (never used); 2 if the element in 474 | the bucket has been deleted [int*] 475 | @return Iterator to the inserted element [khint_t] 476 | */ 477 | #define kh_put(name, h, k, r) kh_put_##name(h, k, r) 478 | 479 | /*! @function 480 | @abstract Retrieve a key from the hash table. 481 | @param name Name of the hash table [symbol] 482 | @param h Pointer to the hash table [khash_t(name)*] 483 | @param k Key [type of keys] 484 | @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t] 485 | */ 486 | #define kh_get(name, h, k) kh_get_##name(h, k) 487 | 488 | /*! @function 489 | @abstract Remove a key from the hash table. 490 | @param name Name of the hash table [symbol] 491 | @param h Pointer to the hash table [khash_t(name)*] 492 | @param k Iterator to the element to be deleted [khint_t] 493 | */ 494 | #define kh_del(name, h, k) kh_del_##name(h, k) 495 | 496 | /*! @function 497 | @abstract Test whether a bucket contains data. 498 | @param h Pointer to the hash table [khash_t(name)*] 499 | @param x Iterator to the bucket [khint_t] 500 | @return 1 if containing data; 0 otherwise [int] 501 | */ 502 | #define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) 503 | 504 | /*! @function 505 | @abstract Get key given an iterator 506 | @param h Pointer to the hash table [khash_t(name)*] 507 | @param x Iterator to the bucket [khint_t] 508 | @return Key [type of keys] 509 | */ 510 | #define kh_key(h, x) ((h)->keys[x]) 511 | 512 | /*! @function 513 | @abstract Get value given an iterator 514 | @param h Pointer to the hash table [khash_t(name)*] 515 | @param x Iterator to the bucket [khint_t] 516 | @return Value [type of values] 517 | @discussion For hash sets, calling this results in segfault. 518 | */ 519 | #define kh_val(h, x) ((h)->vals[x]) 520 | 521 | /*! @function 522 | @abstract Alias of kh_val() 523 | */ 524 | #define kh_value(h, x) ((h)->vals[x]) 525 | 526 | /*! @function 527 | @abstract Get the start iterator 528 | @param h Pointer to the hash table [khash_t(name)*] 529 | @return The start iterator [khint_t] 530 | */ 531 | #define kh_begin(h) (khint_t)(0) 532 | 533 | /*! @function 534 | @abstract Get the end iterator 535 | @param h Pointer to the hash table [khash_t(name)*] 536 | @return The end iterator [khint_t] 537 | */ 538 | #define kh_end(h) ((h)->n_buckets) 539 | 540 | /*! @function 541 | @abstract Get the number of elements in the hash table 542 | @param h Pointer to the hash table [khash_t(name)*] 543 | @return Number of elements in the hash table [khint_t] 544 | */ 545 | #define kh_size(h) ((h)->size) 546 | 547 | /*! @function 548 | @abstract Get the number of buckets in the hash table 549 | @param h Pointer to the hash table [khash_t(name)*] 550 | @return Number of buckets in the hash table [khint_t] 551 | */ 552 | #define kh_n_buckets(h) ((h)->n_buckets) 553 | 554 | /*! @function 555 | @abstract Iterate over the entries in the hash table 556 | @param h Pointer to the hash table [khash_t(name)*] 557 | @param kvar Variable to which key will be assigned 558 | @param vvar Variable to which value will be assigned 559 | @param code Block of code to execute 560 | */ 561 | #define kh_foreach(h, kvar, vvar, code) { khint_t __i; \ 562 | for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ 563 | if (!kh_exist(h,__i)) continue; \ 564 | (kvar) = kh_key(h,__i); \ 565 | (vvar) = kh_val(h,__i); \ 566 | code; \ 567 | } } 568 | 569 | /*! @function 570 | @abstract Iterate over the values in the hash table 571 | @param h Pointer to the hash table [khash_t(name)*] 572 | @param vvar Variable to which value will be assigned 573 | @param code Block of code to execute 574 | */ 575 | #define kh_foreach_value(h, vvar, code) { khint_t __i; \ 576 | for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ 577 | if (!kh_exist(h,__i)) continue; \ 578 | (vvar) = kh_val(h,__i); \ 579 | code; \ 580 | } } 581 | 582 | /* More conenient interfaces */ 583 | 584 | /*! @function 585 | @abstract Instantiate a hash map containing (void *) keys 586 | @param name Name of the hash table [symbol] 587 | @param khval_t Type of values [type] 588 | */ 589 | #ifdef __x86_64__ 590 | #define KHASH_MAP_INIT_PTR(name, khval_t) \ 591 | KHASH_INIT(name, void*, khval_t, 1, kh_ptr64_hash_func, kh_ptr64_hash_equal) 592 | #else 593 | #define KHASH_MAP_INIT_PTR(name, khval_t) \ 594 | KHASH_INIT(name, void*, khval_t, 1, kh_ptr_hash_func, kh_ptr_hash_equal) 595 | #endif 596 | 597 | /*! @function 598 | @abstract Instantiate a hash set containing integer keys 599 | @param name Name of the hash table [symbol] 600 | */ 601 | #define KHASH_SET_INIT_INT(name) \ 602 | KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) 603 | 604 | /*! @function 605 | @abstract Instantiate a hash map containing integer keys 606 | @param name Name of the hash table [symbol] 607 | @param khval_t Type of values [type] 608 | */ 609 | #define KHASH_MAP_INIT_INT(name, khval_t) \ 610 | KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) 611 | 612 | /*! @function 613 | @abstract Instantiate a hash map containing 64-bit integer keys 614 | @param name Name of the hash table [symbol] 615 | */ 616 | #define KHASH_SET_INIT_INT64(name) \ 617 | KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) 618 | 619 | /*! @function 620 | @abstract Instantiate a hash map containing 64-bit integer keys 621 | @param name Name of the hash table [symbol] 622 | @param khval_t Type of values [type] 623 | */ 624 | #define KHASH_MAP_INIT_INT64(name, khval_t) \ 625 | KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) 626 | 627 | typedef const char *kh_cstr_t; 628 | /*! @function 629 | @abstract Instantiate a hash map containing const char* keys 630 | @param name Name of the hash table [symbol] 631 | */ 632 | #define KHASH_SET_INIT_STR(name) \ 633 | KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) 634 | 635 | /*! @function 636 | @abstract Instantiate a hash map containing const char* keys 637 | @param name Name of the hash table [symbol] 638 | @param khval_t Type of values [type] 639 | */ 640 | #define KHASH_MAP_INIT_STR(name, khval_t) \ 641 | KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) 642 | 643 | #endif /* __AC_KHASH_H */ 644 | -------------------------------------------------------------------------------- /c_src/queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 1991, 1993 3 | * The Regents of the University of California. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 4. Neither the name of the University nor the names of its contributors 14 | * may be used to endorse or promote products derived from this software 15 | * without specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 | * SUCH DAMAGE. 28 | * 29 | * @(#)queue.h 8.5 (Berkeley) 8/20/94 30 | * $FreeBSD: src/sys/sys/queue.h,v 1.54 2002/08/05 05:18:43 alfred Exp $ 31 | */ 32 | 33 | #ifndef _DB_QUEUE_H_ 34 | #define _DB_QUEUE_H_ 35 | 36 | #ifndef __offsetof 37 | #define __offsetof(st, m) \ 38 | ((size_t) ( (char *)&((st *)0)->m - (char *)0 )) 39 | #endif 40 | 41 | #ifndef __containerof 42 | #define __containerof(ptr, type, member) ({ \ 43 | const typeof( ((type *)0)->member ) *__mptr = (ptr); \ 44 | (type *)( (char *)__mptr - __offsetof(type,member) );}) 45 | #endif 46 | 47 | #if defined(__cplusplus) 48 | extern "C" { 49 | #endif 50 | 51 | /* 52 | * This file defines four types of data structures: singly-linked lists, 53 | * singly-linked tail queues, lists and tail queues. 54 | * 55 | * A singly-linked list is headed by a single forward pointer. The elements 56 | * are singly linked for minimum space and pointer manipulation overhead at 57 | * the expense of O(n) removal for arbitrary elements. New elements can be 58 | * added to the list after an existing element or at the head of the list. 59 | * Elements being removed from the head of the list should use the explicit 60 | * macro for this purpose for optimum efficiency. A singly-linked list may 61 | * only be traversed in the forward direction. Singly-linked lists are ideal 62 | * for applications with large datasets and few or no removals or for 63 | * implementing a LIFO queue. 64 | * 65 | * A singly-linked tail queue is headed by a pair of pointers, one to the 66 | * head of the list and the other to the tail of the list. The elements are 67 | * singly linked for minimum space and pointer manipulation overhead at the 68 | * expense of O(n) removal for arbitrary elements. New elements can be added 69 | * to the list after an existing element, at the head of the list, or at the 70 | * end of the list. Elements being removed from the head of the tail queue 71 | * should use the explicit macro for this purpose for optimum efficiency. 72 | * A singly-linked tail queue may only be traversed in the forward direction. 73 | * Singly-linked tail queues are ideal for applications with large datasets 74 | * and few or no removals or for implementing a FIFO queue. 75 | * 76 | * A list is headed by a single forward pointer (or an array of forward 77 | * pointers for a hash table header). The elements are doubly linked 78 | * so that an arbitrary element can be removed without a need to 79 | * traverse the list. New elements can be added to the list before 80 | * or after an existing element or at the head of the list. A list 81 | * may only be traversed in the forward direction. 82 | * 83 | * A tail queue is headed by a pair of pointers, one to the head of the 84 | * list and the other to the tail of the list. The elements are doubly 85 | * linked so that an arbitrary element can be removed without a need to 86 | * traverse the list. New elements can be added to the list before or 87 | * after an existing element, at the head of the list, or at the end of 88 | * the list. A tail queue may be traversed in either direction. 89 | * 90 | * For details on the use of these macros, see the queue(3) manual page. 91 | * 92 | * 93 | * SLIST LIST STAILQ TAILQ 94 | * _HEAD + + + + 95 | * _HEAD_INITIALIZER + + + + 96 | * _ENTRY + + + + 97 | * _INIT + + + + 98 | * _EMPTY + + + + 99 | * _FIRST + + + + 100 | * _NEXT + + + + 101 | * _PREV - - - + 102 | * _LAST - - + + 103 | * _FOREACH + + + + 104 | * _FOREACH_REVERSE - - - + 105 | * _INSERT_HEAD + + + + 106 | * _INSERT_BEFORE - + - + 107 | * _INSERT_AFTER + + + + 108 | * _INSERT_TAIL - - + + 109 | * _CONCAT - - + + 110 | * _REMOVE_HEAD + - + - 111 | * _REMOVE + + + + 112 | * 113 | */ 114 | 115 | /* 116 | * XXX 117 | * We #undef all of the macros because there are incompatible versions of this 118 | * file and these macros on various systems. What makes the problem worse is 119 | * they are included and/or defined by system include files which we may have 120 | * already loaded into Berkeley DB before getting here. For example, FreeBSD's 121 | * includes its system , and VxWorks UnixLib.h defines 122 | * several of the LIST_XXX macros. Visual C.NET 7.0 also defines some of these 123 | * same macros in Vc7\PlatformSDK\Include\WinNT.h. Make sure we use ours. 124 | */ 125 | #undef LIST_EMPTY 126 | #undef LIST_ENTRY 127 | #undef LIST_FIRST 128 | #undef LIST_FOREACH 129 | #undef LIST_HEAD 130 | #undef LIST_HEAD_INITIALIZER 131 | #undef LIST_INIT 132 | #undef LIST_INSERT_AFTER 133 | #undef LIST_INSERT_BEFORE 134 | #undef LIST_INSERT_HEAD 135 | #undef LIST_NEXT 136 | #undef LIST_REMOVE 137 | #undef QMD_TRACE_ELEM 138 | #undef QMD_TRACE_HEAD 139 | #undef QUEUE_MACRO_DEBUG 140 | #undef SLIST_EMPTY 141 | #undef SLIST_ENTRY 142 | #undef SLIST_FIRST 143 | #undef SLIST_FOREACH 144 | #undef SLIST_FOREACH_PREVPTR 145 | #undef SLIST_HEAD 146 | #undef SLIST_HEAD_INITIALIZER 147 | #undef SLIST_INIT 148 | #undef SLIST_INSERT_AFTER 149 | #undef SLIST_INSERT_HEAD 150 | #undef SLIST_NEXT 151 | #undef SLIST_REMOVE 152 | #undef SLIST_REMOVE_HEAD 153 | #undef STAILQ_CONCAT 154 | #undef STAILQ_EMPTY 155 | #undef STAILQ_ENTRY 156 | #undef STAILQ_FIRST 157 | #undef STAILQ_FOREACH 158 | #undef STAILQ_HEAD 159 | #undef STAILQ_HEAD_INITIALIZER 160 | #undef STAILQ_INIT 161 | #undef STAILQ_INSERT_AFTER 162 | #undef STAILQ_INSERT_HEAD 163 | #undef STAILQ_INSERT_TAIL 164 | #undef STAILQ_LAST 165 | #undef STAILQ_NEXT 166 | #undef STAILQ_REMOVE 167 | #undef STAILQ_REMOVE_HEAD 168 | #undef STAILQ_REMOVE_HEAD_UNTIL 169 | #undef TAILQ_CONCAT 170 | #undef TAILQ_EMPTY 171 | #undef TAILQ_ENTRY 172 | #undef TAILQ_FIRST 173 | #undef TAILQ_FOREACH 174 | #undef TAILQ_FOREACH_REVERSE 175 | #undef TAILQ_HEAD 176 | #undef TAILQ_HEAD_INITIALIZER 177 | #undef TAILQ_INIT 178 | #undef TAILQ_INSERT_AFTER 179 | #undef TAILQ_INSERT_BEFORE 180 | #undef TAILQ_INSERT_HEAD 181 | #undef TAILQ_INSERT_TAIL 182 | #undef TAILQ_LAST 183 | #undef TAILQ_NEXT 184 | #undef TAILQ_PREV 185 | #undef TAILQ_REMOVE 186 | #undef TRACEBUF 187 | #undef TRASHIT 188 | 189 | #define QUEUE_MACRO_DEBUG 0 190 | #if QUEUE_MACRO_DEBUG 191 | /* Store the last 2 places the queue element or head was altered */ 192 | struct qm_trace { 193 | char * lastfile; 194 | int lastline; 195 | char * prevfile; 196 | int prevline; 197 | }; 198 | 199 | #define TRACEBUF struct qm_trace trace; 200 | #define TRASHIT(x) do {(x) = (void *)-1;} while (0) 201 | 202 | #define QMD_TRACE_HEAD(head) do { \ 203 | (head)->trace.prevline = (head)->trace.lastline; \ 204 | (head)->trace.prevfile = (head)->trace.lastfile; \ 205 | (head)->trace.lastline = __LINE__; \ 206 | (head)->trace.lastfile = __FILE__; \ 207 | } while (0) 208 | 209 | #define QMD_TRACE_ELEM(elem) do { \ 210 | (elem)->trace.prevline = (elem)->trace.lastline; \ 211 | (elem)->trace.prevfile = (elem)->trace.lastfile; \ 212 | (elem)->trace.lastline = __LINE__; \ 213 | (elem)->trace.lastfile = __FILE__; \ 214 | } while (0) 215 | 216 | #else 217 | #define QMD_TRACE_ELEM(elem) 218 | #define QMD_TRACE_HEAD(head) 219 | #define TRACEBUF 220 | #define TRASHIT(x) 221 | #endif /* QUEUE_MACRO_DEBUG */ 222 | 223 | /* 224 | * Singly-linked List declarations. 225 | */ 226 | #define SLIST_HEAD(name, type) \ 227 | struct name { \ 228 | struct type *slh_first; /* first element */ \ 229 | } 230 | 231 | #define SLIST_HEAD_INITIALIZER(head) \ 232 | { NULL } 233 | 234 | #define SLIST_ENTRY(type) \ 235 | struct { \ 236 | struct type *sle_next; /* next element */ \ 237 | } 238 | 239 | /* 240 | * Singly-linked List functions. 241 | */ 242 | #define SLIST_EMPTY(head) ((head)->slh_first == NULL) 243 | 244 | #define SLIST_FIRST(head) ((head)->slh_first) 245 | 246 | #define SLIST_FOREACH(var, head, field) \ 247 | for ((var) = SLIST_FIRST((head)); \ 248 | (var); \ 249 | (var) = SLIST_NEXT((var), field)) 250 | 251 | #define SLIST_FOREACH_PREVPTR(var, varp, head, field) \ 252 | for ((varp) = &SLIST_FIRST((head)); \ 253 | ((var) = *(varp)) != NULL; \ 254 | (varp) = &SLIST_NEXT((var), field)) 255 | 256 | #define SLIST_INIT(head) do { \ 257 | SLIST_FIRST((head)) = NULL; \ 258 | } while (0) 259 | 260 | #define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ 261 | SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ 262 | SLIST_NEXT((slistelm), field) = (elm); \ 263 | } while (0) 264 | 265 | #define SLIST_INSERT_HEAD(head, elm, field) do { \ 266 | SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ 267 | SLIST_FIRST((head)) = (elm); \ 268 | } while (0) 269 | 270 | #define SLIST_NEXT(elm, field) ((elm)->field.sle_next) 271 | 272 | #define SLIST_REMOVE(head, elm, type, field) do { \ 273 | if (SLIST_FIRST((head)) == (elm)) { \ 274 | SLIST_REMOVE_HEAD((head), field); \ 275 | } \ 276 | else { \ 277 | struct type *curelm = SLIST_FIRST((head)); \ 278 | while (SLIST_NEXT(curelm, field) != (elm)) \ 279 | curelm = SLIST_NEXT(curelm, field); \ 280 | SLIST_NEXT(curelm, field) = \ 281 | SLIST_NEXT(SLIST_NEXT(curelm, field), field); \ 282 | } \ 283 | } while (0) 284 | 285 | #define SLIST_REMOVE_HEAD(head, field) do { \ 286 | SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ 287 | } while (0) 288 | 289 | /* 290 | * Singly-linked Tail queue declarations. 291 | */ 292 | #define STAILQ_HEAD(name, type) \ 293 | struct name { \ 294 | struct type *stqh_first;/* first element */ \ 295 | struct type **stqh_last;/* addr of last next element */ \ 296 | } 297 | 298 | #define STAILQ_HEAD_INITIALIZER(head) \ 299 | { NULL, &(head).stqh_first } 300 | 301 | #define STAILQ_ENTRY(type) \ 302 | struct { \ 303 | struct type *stqe_next; /* next element */ \ 304 | } 305 | 306 | /* 307 | * Singly-linked Tail queue functions. 308 | */ 309 | #define STAILQ_CONCAT(head1, head2) do { \ 310 | if (!STAILQ_EMPTY((head2))) { \ 311 | *(head1)->stqh_last = (head2)->stqh_first; \ 312 | (head1)->stqh_last = (head2)->stqh_last; \ 313 | STAILQ_INIT((head2)); \ 314 | } \ 315 | } while (0) 316 | 317 | #define STAILQ_EMPTY(head) ((head)->stqh_first == NULL) 318 | 319 | #define STAILQ_FIRST(head) ((head)->stqh_first) 320 | 321 | #define STAILQ_FOREACH(var, head, field) \ 322 | for ((var) = STAILQ_FIRST((head)); \ 323 | (var); \ 324 | (var) = STAILQ_NEXT((var), field)) 325 | 326 | #define STAILQ_INIT(head) do { \ 327 | STAILQ_FIRST((head)) = NULL; \ 328 | (head)->stqh_last = &STAILQ_FIRST((head)); \ 329 | } while (0) 330 | 331 | #define STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ 332 | if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ 333 | (head)->stqh_last = &STAILQ_NEXT((elm), field); \ 334 | STAILQ_NEXT((tqelm), field) = (elm); \ 335 | } while (0) 336 | 337 | #define STAILQ_INSERT_HEAD(head, elm, field) do { \ 338 | if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ 339 | (head)->stqh_last = &STAILQ_NEXT((elm), field); \ 340 | STAILQ_FIRST((head)) = (elm); \ 341 | } while (0) 342 | 343 | #define STAILQ_INSERT_TAIL(head, elm, field) do { \ 344 | STAILQ_NEXT((elm), field) = NULL; \ 345 | *(head)->stqh_last = (elm); \ 346 | (head)->stqh_last = &STAILQ_NEXT((elm), field); \ 347 | } while (0) 348 | 349 | #define STAILQ_LAST(head, type, field) \ 350 | (STAILQ_EMPTY((head)) ? \ 351 | NULL : \ 352 | ((struct type *) \ 353 | ((char *)((head)->stqh_last) - __offsetof(struct type, field)))) 354 | 355 | #define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) 356 | 357 | #define STAILQ_REMOVE(head, elm, type, field) do { \ 358 | if (STAILQ_FIRST((head)) == (elm)) { \ 359 | STAILQ_REMOVE_HEAD((head), field); \ 360 | } \ 361 | else { \ 362 | struct type *curelm = STAILQ_FIRST((head)); \ 363 | while (STAILQ_NEXT(curelm, field) != (elm)) \ 364 | curelm = STAILQ_NEXT(curelm, field); \ 365 | if ((STAILQ_NEXT(curelm, field) = \ 366 | STAILQ_NEXT(STAILQ_NEXT(curelm, field), field)) == NULL)\ 367 | (head)->stqh_last = &STAILQ_NEXT((curelm), field);\ 368 | } \ 369 | } while (0) 370 | 371 | #define STAILQ_REMOVE_HEAD(head, field) do { \ 372 | if ((STAILQ_FIRST((head)) = \ 373 | STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ 374 | (head)->stqh_last = &STAILQ_FIRST((head)); \ 375 | } while (0) 376 | 377 | #define STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do { \ 378 | if ((STAILQ_FIRST((head)) = STAILQ_NEXT((elm), field)) == NULL) \ 379 | (head)->stqh_last = &STAILQ_FIRST((head)); \ 380 | } while (0) 381 | 382 | /* 383 | * List declarations. 384 | */ 385 | #define LIST_HEAD(name, type) \ 386 | struct name { \ 387 | struct type *lh_first; /* first element */ \ 388 | } 389 | 390 | #define LIST_HEAD_INITIALIZER(head) \ 391 | { NULL } 392 | 393 | #define LIST_ENTRY(type) \ 394 | struct { \ 395 | struct type *le_next; /* next element */ \ 396 | struct type **le_prev; /* address of previous next element */ \ 397 | } 398 | 399 | /* 400 | * List functions. 401 | */ 402 | 403 | #define LIST_EMPTY(head) ((head)->lh_first == NULL) 404 | 405 | #define LIST_FIRST(head) ((head)->lh_first) 406 | 407 | #define LIST_FOREACH(var, head, field) \ 408 | for ((var) = LIST_FIRST((head)); \ 409 | (var); \ 410 | (var) = LIST_NEXT((var), field)) 411 | 412 | #define LIST_INIT(head) do { \ 413 | LIST_FIRST((head)) = NULL; \ 414 | } while (0) 415 | 416 | #define LIST_INSERT_AFTER(listelm, elm, field) do { \ 417 | if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ 418 | LIST_NEXT((listelm), field)->field.le_prev = \ 419 | &LIST_NEXT((elm), field); \ 420 | LIST_NEXT((listelm), field) = (elm); \ 421 | (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ 422 | } while (0) 423 | 424 | #define LIST_INSERT_BEFORE(listelm, elm, field) do { \ 425 | (elm)->field.le_prev = (listelm)->field.le_prev; \ 426 | LIST_NEXT((elm), field) = (listelm); \ 427 | *(listelm)->field.le_prev = (elm); \ 428 | (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ 429 | } while (0) 430 | 431 | #define LIST_INSERT_HEAD(head, elm, field) do { \ 432 | if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ 433 | LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ 434 | LIST_FIRST((head)) = (elm); \ 435 | (elm)->field.le_prev = &LIST_FIRST((head)); \ 436 | } while (0) 437 | 438 | #define LIST_NEXT(elm, field) ((elm)->field.le_next) 439 | 440 | #define LIST_REMOVE(elm, field) do { \ 441 | if (LIST_NEXT((elm), field) != NULL) \ 442 | LIST_NEXT((elm), field)->field.le_prev = \ 443 | (elm)->field.le_prev; \ 444 | *(elm)->field.le_prev = LIST_NEXT((elm), field); \ 445 | } while (0) 446 | 447 | /* 448 | * Tail queue declarations. 449 | */ 450 | #define TAILQ_HEAD(name, type) \ 451 | struct name { \ 452 | struct type *tqh_first; /* first element */ \ 453 | struct type **tqh_last; /* addr of last next element */ \ 454 | TRACEBUF \ 455 | } 456 | 457 | #define TAILQ_HEAD_INITIALIZER(head) \ 458 | { NULL, &(head).tqh_first } 459 | 460 | #define TAILQ_ENTRY(type) \ 461 | struct { \ 462 | struct type *tqe_next; /* next element */ \ 463 | struct type **tqe_prev; /* address of previous next element */ \ 464 | TRACEBUF \ 465 | } 466 | 467 | /* 468 | * Tail queue functions. 469 | */ 470 | #define TAILQ_CONCAT(head1, head2, field) do { \ 471 | if (!TAILQ_EMPTY(head2)) { \ 472 | *(head1)->tqh_last = (head2)->tqh_first; \ 473 | (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ 474 | (head1)->tqh_last = (head2)->tqh_last; \ 475 | TAILQ_INIT((head2)); \ 476 | QMD_TRACE_HEAD(head); \ 477 | QMD_TRACE_HEAD(head2); \ 478 | } \ 479 | } while (0) 480 | 481 | #define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) 482 | 483 | #define TAILQ_FIRST(head) ((head)->tqh_first) 484 | 485 | #define TAILQ_FOREACH(var, head, field) \ 486 | for ((var) = TAILQ_FIRST((head)); \ 487 | (var); \ 488 | (var) = TAILQ_NEXT((var), field)) 489 | 490 | #define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ 491 | for ((var) = TAILQ_LAST((head), headname); \ 492 | (var); \ 493 | (var) = TAILQ_PREV((var), headname, field)) 494 | 495 | #define TAILQ_INIT(head) do { \ 496 | TAILQ_FIRST((head)) = NULL; \ 497 | (head)->tqh_last = &TAILQ_FIRST((head)); \ 498 | QMD_TRACE_HEAD(head); \ 499 | } while (0) 500 | 501 | #define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ 502 | if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ 503 | TAILQ_NEXT((elm), field)->field.tqe_prev = \ 504 | &TAILQ_NEXT((elm), field); \ 505 | else { \ 506 | (head)->tqh_last = &TAILQ_NEXT((elm), field); \ 507 | QMD_TRACE_HEAD(head); \ 508 | } \ 509 | TAILQ_NEXT((listelm), field) = (elm); \ 510 | (elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field); \ 511 | QMD_TRACE_ELEM(&(elm)->field); \ 512 | QMD_TRACE_ELEM(&listelm->field); \ 513 | } while (0) 514 | 515 | #define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ 516 | (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ 517 | TAILQ_NEXT((elm), field) = (listelm); \ 518 | *(listelm)->field.tqe_prev = (elm); \ 519 | (listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field); \ 520 | QMD_TRACE_ELEM(&(elm)->field); \ 521 | QMD_TRACE_ELEM(&listelm->field); \ 522 | } while (0) 523 | 524 | #define TAILQ_INSERT_HEAD(head, elm, field) do { \ 525 | if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ 526 | TAILQ_FIRST((head))->field.tqe_prev = \ 527 | &TAILQ_NEXT((elm), field); \ 528 | else \ 529 | (head)->tqh_last = &TAILQ_NEXT((elm), field); \ 530 | TAILQ_FIRST((head)) = (elm); \ 531 | (elm)->field.tqe_prev = &TAILQ_FIRST((head)); \ 532 | QMD_TRACE_HEAD(head); \ 533 | QMD_TRACE_ELEM(&(elm)->field); \ 534 | } while (0) 535 | 536 | #define TAILQ_INSERT_TAIL(head, elm, field) do { \ 537 | TAILQ_NEXT((elm), field) = NULL; \ 538 | (elm)->field.tqe_prev = (head)->tqh_last; \ 539 | *(head)->tqh_last = (elm); \ 540 | (head)->tqh_last = &TAILQ_NEXT((elm), field); \ 541 | QMD_TRACE_HEAD(head); \ 542 | QMD_TRACE_ELEM(&(elm)->field); \ 543 | } while (0) 544 | 545 | #define TAILQ_LAST(head, headname) \ 546 | (*(((struct headname *)((head)->tqh_last))->tqh_last)) 547 | 548 | #define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) 549 | 550 | #define TAILQ_PREV(elm, headname, field) \ 551 | (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) 552 | 553 | #define TAILQ_REMOVE(head, elm, field) do { \ 554 | if ((TAILQ_NEXT((elm), field)) != NULL) \ 555 | TAILQ_NEXT((elm), field)->field.tqe_prev = \ 556 | (elm)->field.tqe_prev; \ 557 | else { \ 558 | (head)->tqh_last = (elm)->field.tqe_prev; \ 559 | QMD_TRACE_HEAD(head); \ 560 | } \ 561 | *(elm)->field.tqe_prev = TAILQ_NEXT((elm), field); \ 562 | TRASHIT((elm)->field.tqe_next); \ 563 | TRASHIT((elm)->field.tqe_prev); \ 564 | QMD_TRACE_ELEM(&(elm)->field); \ 565 | } while (0) 566 | 567 | /* 568 | * Circular queue definitions. 569 | */ 570 | #define CIRCLEQ_HEAD(name, type) \ 571 | struct name { \ 572 | struct type *cqh_first; /* first element */ \ 573 | struct type *cqh_last; /* last element */ \ 574 | } 575 | 576 | #define CIRCLEQ_HEAD_INITIALIZER(head) \ 577 | { (void *)&head, (void *)&head } 578 | 579 | #define CIRCLEQ_ENTRY(type) \ 580 | struct { \ 581 | struct type *cqe_next; /* next element */ \ 582 | struct type *cqe_prev; /* previous element */ \ 583 | } 584 | 585 | /* 586 | * Circular queue functions. 587 | */ 588 | #define CIRCLEQ_INIT(head) do { \ 589 | (head)->cqh_first = (void *)(head); \ 590 | (head)->cqh_last = (void *)(head); \ 591 | } while (/*CONSTCOND*/0) 592 | 593 | #define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ 594 | (elm)->field.cqe_next = (listelm)->field.cqe_next; \ 595 | (elm)->field.cqe_prev = (listelm); \ 596 | if ((listelm)->field.cqe_next == (void *)(head)) \ 597 | (head)->cqh_last = (elm); \ 598 | else \ 599 | (listelm)->field.cqe_next->field.cqe_prev = (elm); \ 600 | (listelm)->field.cqe_next = (elm); \ 601 | } while (/*CONSTCOND*/0) 602 | 603 | #define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do { \ 604 | (elm)->field.cqe_next = (listelm); \ 605 | (elm)->field.cqe_prev = (listelm)->field.cqe_prev; \ 606 | if ((listelm)->field.cqe_prev == (void *)(head)) \ 607 | (head)->cqh_first = (elm); \ 608 | else \ 609 | (listelm)->field.cqe_prev->field.cqe_next = (elm); \ 610 | (listelm)->field.cqe_prev = (elm); \ 611 | } while (/*CONSTCOND*/0) 612 | 613 | #define CIRCLEQ_INSERT_HEAD(head, elm, field) do { \ 614 | (elm)->field.cqe_next = (head)->cqh_first; \ 615 | (elm)->field.cqe_prev = (void *)(head); \ 616 | if ((head)->cqh_last == (void *)(head)) \ 617 | (head)->cqh_last = (elm); \ 618 | else \ 619 | (head)->cqh_first->field.cqe_prev = (elm); \ 620 | (head)->cqh_first = (elm); \ 621 | } while (/*CONSTCOND*/0) 622 | 623 | #define CIRCLEQ_INSERT_TAIL(head, elm, field) do { \ 624 | (elm)->field.cqe_next = (void *)(head); \ 625 | (elm)->field.cqe_prev = (head)->cqh_last; \ 626 | if ((head)->cqh_first == (void *)(head)) \ 627 | (head)->cqh_first = (elm); \ 628 | else \ 629 | (head)->cqh_last->field.cqe_next = (elm); \ 630 | (head)->cqh_last = (elm); \ 631 | } while (/*CONSTCOND*/0) 632 | 633 | #define CIRCLEQ_REMOVE(head, elm, field) do { \ 634 | if ((elm)->field.cqe_next == (void *)(head)) \ 635 | (head)->cqh_last = (elm)->field.cqe_prev; \ 636 | else \ 637 | (elm)->field.cqe_next->field.cqe_prev = \ 638 | (elm)->field.cqe_prev; \ 639 | if ((elm)->field.cqe_prev == (void *)(head)) \ 640 | (head)->cqh_first = (elm)->field.cqe_next; \ 641 | else \ 642 | (elm)->field.cqe_prev->field.cqe_next = \ 643 | (elm)->field.cqe_next; \ 644 | } while (/*CONSTCOND*/0) 645 | 646 | #define CIRCLEQ_FOREACH(var, head, field) \ 647 | for ((var) = ((head)->cqh_first); \ 648 | (var) != (const void *)(head); \ 649 | (var) = ((var)->field.cqe_next)) 650 | 651 | #define CIRCLEQ_FOREACH_REVERSE(var, head, field) \ 652 | for ((var) = ((head)->cqh_last); \ 653 | (var) != (const void *)(head); \ 654 | (var) = ((var)->field.cqe_prev)) 655 | 656 | /* 657 | * Circular queue access methods. 658 | */ 659 | #define CIRCLEQ_EMPTY(head) ((head)->cqh_first == (void *)(head)) 660 | #define CIRCLEQ_FIRST(head) ((head)->cqh_first) 661 | #define CIRCLEQ_LAST(head) ((head)->cqh_last) 662 | #define CIRCLEQ_NEXT(elm, field) ((elm)->field.cqe_next) 663 | #define CIRCLEQ_PREV(elm, field) ((elm)->field.cqe_prev) 664 | 665 | #define CIRCLEQ_LOOP_NEXT(head, elm, field) \ 666 | (((elm)->field.cqe_next == (void *)(head)) \ 667 | ? ((head)->cqh_first) \ 668 | : (elm->field.cqe_next)) 669 | #define CIRCLEQ_LOOP_PREV(head, elm, field) \ 670 | (((elm)->field.cqe_prev == (void *)(head)) \ 671 | ? ((head)->cqh_last) \ 672 | : (elm->field.cqe_prev)) 673 | 674 | 675 | #if defined(__cplusplus) 676 | } 677 | #endif 678 | #endif /* !_DB_QUEUE_H_ */ 679 | -------------------------------------------------------------------------------- /c_src/async_nif.h: -------------------------------------------------------------------------------- 1 | /* 2 | * async_nif: An async thread-pool layer for Erlang's NIF API 3 | * 4 | * Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 5 | * Author: Gregory Burd 6 | * 7 | * This file is provided to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at: 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 16 | * License for the specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef __ASYNC_NIF_H__ 21 | #define __ASYNC_NIF_H__ 22 | 23 | #if defined(__cplusplus) 24 | extern "C" { 25 | #endif 26 | 27 | #include 28 | 29 | #include "queue.h" 30 | 31 | #ifndef UNUSED 32 | #define UNUSED(v) ((void)(v)) 33 | #endif 34 | 35 | #define ASYNC_NIF_MAX_WORKERS 1024 36 | #define ASYNC_NIF_MIN_WORKERS 2 37 | #define ASYNC_NIF_WORKER_QUEUE_SIZE 8192 38 | #define ASYNC_NIF_MAX_QUEUED_REQS ASYNC_NIF_WORKER_QUEUE_SIZE * ASYNC_NIF_MAX_WORKERS 39 | 40 | /* Atoms (initialized in on_load) */ 41 | static ERL_NIF_TERM ATOM_EAGAIN; 42 | static ERL_NIF_TERM ATOM_ENOMEM; 43 | static ERL_NIF_TERM ATOM_ENQUEUED; 44 | static ERL_NIF_TERM ATOM_ERROR; 45 | static ERL_NIF_TERM ATOM_OK; 46 | static ERL_NIF_TERM ATOM_SHUTDOWN; 47 | 48 | 49 | struct async_nif_req_entry { 50 | ERL_NIF_TERM ref; 51 | ErlNifEnv *env; 52 | ErlNifPid pid; 53 | void *args; 54 | void (*fn_work)(ErlNifEnv*, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *); 55 | void (*fn_post)(void *); 56 | STAILQ_ENTRY(async_nif_req_entry) entries; 57 | }; 58 | 59 | 60 | struct async_nif_work_queue { 61 | unsigned int num_workers; 62 | unsigned int depth; 63 | ErlNifMutex *reqs_mutex; 64 | ErlNifCond *reqs_cnd; 65 | struct async_nif_work_queue *next; 66 | STAILQ_HEAD(reqs, async_nif_req_entry) reqs; 67 | }; 68 | 69 | struct async_nif_worker_entry { 70 | ErlNifTid tid; 71 | unsigned int worker_id; 72 | struct async_nif_state *async_nif; 73 | struct async_nif_work_queue *q; 74 | SLIST_ENTRY(async_nif_worker_entry) entries; 75 | }; 76 | 77 | struct async_nif_state { 78 | unsigned int shutdown; 79 | ErlNifMutex *we_mutex; 80 | unsigned int we_active; 81 | SLIST_HEAD(joining, async_nif_worker_entry) we_joining; 82 | unsigned int num_queues; 83 | unsigned int next_q; 84 | STAILQ_HEAD(recycled_reqs, async_nif_req_entry) recycled_reqs; 85 | unsigned int num_reqs; 86 | ErlNifMutex *recycled_req_mutex; 87 | struct async_nif_work_queue queues[]; 88 | }; 89 | 90 | #define ASYNC_NIF_DECL(decl, frame, pre_block, work_block, post_block) \ 91 | struct decl ## _args frame; \ 92 | static void fn_work_ ## decl (ErlNifEnv *env, ERL_NIF_TERM ref, ErlNifPid *pid, unsigned int worker_id, struct decl ## _args *args) { \ 93 | UNUSED(worker_id); \ 94 | DPRINTF("async_nif: calling \"%s\"", __func__); \ 95 | do work_block while(0); \ 96 | DPRINTF("async_nif: returned from \"%s\"", __func__); \ 97 | } \ 98 | static void fn_post_ ## decl (struct decl ## _args *args) { \ 99 | UNUSED(args); \ 100 | DPRINTF("async_nif: calling \"fn_post_%s\"", #decl); \ 101 | do post_block while(0); \ 102 | DPRINTF("async_nif: returned from \"fn_post_%s\"", #decl); \ 103 | } \ 104 | static ERL_NIF_TERM decl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv_in[]) { \ 105 | struct decl ## _args on_stack_args; \ 106 | struct decl ## _args *args = &on_stack_args; \ 107 | struct decl ## _args *copy_of_args; \ 108 | struct async_nif_req_entry *req = NULL; \ 109 | unsigned int affinity = 0; \ 110 | ErlNifEnv *new_env = NULL; \ 111 | /* argv[0] is a ref used for selective recv */ \ 112 | const ERL_NIF_TERM *argv = argv_in + 1; \ 113 | argc -= 1; \ 114 | /* Note: !!! this assumes that the first element of priv_data is ours */ \ 115 | struct async_nif_state *async_nif = *(struct async_nif_state**)enif_priv_data(env); \ 116 | if (async_nif->shutdown) \ 117 | return enif_make_tuple2(env, ATOM_ERROR, ATOM_SHUTDOWN); \ 118 | req = async_nif_reuse_req(async_nif); \ 119 | if (!req) \ 120 | return enif_make_tuple2(env, ATOM_ERROR, ATOM_ENOMEM); \ 121 | new_env = req->env; \ 122 | DPRINTF("async_nif: calling \"%s\"", __func__); \ 123 | do pre_block while(0); \ 124 | DPRINTF("async_nif: returned from \"%s\"", __func__); \ 125 | copy_of_args = (struct decl ## _args *)malloc(sizeof(struct decl ## _args)); \ 126 | if (!copy_of_args) { \ 127 | fn_post_ ## decl (args); \ 128 | async_nif_recycle_req(req, async_nif); \ 129 | return enif_make_tuple2(env, ATOM_ERROR, ATOM_ENOMEM); \ 130 | } \ 131 | memcpy(copy_of_args, args, sizeof(struct decl ## _args)); \ 132 | req->ref = enif_make_copy(new_env, argv_in[0]); \ 133 | enif_self(env, &req->pid); \ 134 | req->args = (void*)copy_of_args; \ 135 | req->fn_work = (void (*)(ErlNifEnv *, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *))fn_work_ ## decl ; \ 136 | req->fn_post = (void (*)(void *))fn_post_ ## decl; \ 137 | int h = -1; \ 138 | if (affinity) \ 139 | h = ((unsigned int)affinity) % async_nif->num_queues; \ 140 | ERL_NIF_TERM reply = async_nif_enqueue_req(async_nif, req, h); \ 141 | if (!reply) { \ 142 | fn_post_ ## decl (args); \ 143 | async_nif_recycle_req(req, async_nif); \ 144 | free(copy_of_args); \ 145 | return enif_make_tuple2(env, ATOM_ERROR, ATOM_EAGAIN); \ 146 | } \ 147 | return reply; \ 148 | } 149 | 150 | #define ASYNC_NIF_INIT(name) \ 151 | static ErlNifMutex *name##_async_nif_coord = NULL; 152 | 153 | #define ASYNC_NIF_LOAD(name, env, priv) do { \ 154 | if (!name##_async_nif_coord) \ 155 | name##_async_nif_coord = enif_mutex_create("nif_coord load"); \ 156 | enif_mutex_lock(name##_async_nif_coord); \ 157 | priv = async_nif_load(env); \ 158 | enif_mutex_unlock(name##_async_nif_coord); \ 159 | } while(0); 160 | #define ASYNC_NIF_UNLOAD(name, env, priv) do { \ 161 | if (!name##_async_nif_coord) \ 162 | name##_async_nif_coord = enif_mutex_create("nif_coord unload"); \ 163 | enif_mutex_lock(name##_async_nif_coord); \ 164 | async_nif_unload(env, priv); \ 165 | enif_mutex_unlock(name##_async_nif_coord); \ 166 | enif_mutex_destroy(name##_async_nif_coord); \ 167 | name##_async_nif_coord = NULL; \ 168 | } while(0); 169 | #define ASYNC_NIF_UPGRADE(name, env) do { \ 170 | if (!name##_async_nif_coord) \ 171 | name##_async_nif_coord = enif_mutex_create("nif_coord upgrade"); \ 172 | enif_mutex_lock(name##_async_nif_coord); \ 173 | async_nif_upgrade(env); \ 174 | enif_mutex_unlock(name##_async_nif_coord); \ 175 | } while(0); 176 | 177 | #define ASYNC_NIF_RETURN_BADARG() do { \ 178 | async_nif_recycle_req(req, async_nif); \ 179 | return enif_make_badarg(env); \ 180 | } while(0); 181 | #define ASYNC_NIF_WORK_ENV new_env 182 | 183 | #define ASYNC_NIF_REPLY(msg) enif_send(NULL, pid, env, enif_make_tuple2(env, ref, msg)) 184 | 185 | /** 186 | * Return a request structure from the recycled req queue if one exists, 187 | * otherwise create one. 188 | */ 189 | struct async_nif_req_entry * 190 | async_nif_reuse_req(struct async_nif_state *async_nif) 191 | { 192 | struct async_nif_req_entry *req = NULL; 193 | ErlNifEnv *env = NULL; 194 | 195 | enif_mutex_lock(async_nif->recycled_req_mutex); 196 | if (STAILQ_EMPTY(&async_nif->recycled_reqs)) { 197 | if (async_nif->num_reqs < ASYNC_NIF_MAX_QUEUED_REQS) { 198 | req = malloc(sizeof(struct async_nif_req_entry)); 199 | if (req) { 200 | memset(req, 0, sizeof(struct async_nif_req_entry)); 201 | env = enif_alloc_env(); 202 | if (env) { 203 | req->env = env; 204 | __sync_fetch_and_add(&async_nif->num_reqs, 1); 205 | } else { 206 | free(req); 207 | req = NULL; 208 | } 209 | } 210 | } 211 | } else { 212 | req = STAILQ_FIRST(&async_nif->recycled_reqs); 213 | STAILQ_REMOVE(&async_nif->recycled_reqs, req, async_nif_req_entry, entries); 214 | } 215 | enif_mutex_unlock(async_nif->recycled_req_mutex); 216 | return req; 217 | } 218 | 219 | /** 220 | * Store the request for future re-use. 221 | * 222 | * req a request entry with an ErlNifEnv* which will be cleared 223 | * before reuse, but not until then. 224 | * async_nif a handle to our state so that we can find and use the mutex 225 | */ 226 | void 227 | async_nif_recycle_req(struct async_nif_req_entry *req, struct async_nif_state *async_nif) 228 | { 229 | ErlNifEnv *env = NULL; 230 | enif_mutex_lock(async_nif->recycled_req_mutex); 231 | enif_clear_env(req->env); 232 | env = req->env; 233 | memset(req, 0, sizeof(struct async_nif_req_entry)); 234 | req->env = env; 235 | STAILQ_INSERT_TAIL(&async_nif->recycled_reqs, req, entries); 236 | enif_mutex_unlock(async_nif->recycled_req_mutex); 237 | } 238 | 239 | static void *async_nif_worker_fn(void *); 240 | 241 | /** 242 | * Start up a worker thread. 243 | */ 244 | static int 245 | async_nif_start_worker(struct async_nif_state *async_nif, struct async_nif_work_queue *q) 246 | { 247 | struct async_nif_worker_entry *we; 248 | 249 | if (0 == q) 250 | return EINVAL; 251 | 252 | enif_mutex_lock(async_nif->we_mutex); 253 | 254 | we = SLIST_FIRST(&async_nif->we_joining); 255 | while(we != NULL) { 256 | struct async_nif_worker_entry *n = SLIST_NEXT(we, entries); 257 | SLIST_REMOVE(&async_nif->we_joining, we, async_nif_worker_entry, entries); 258 | void *exit_value = 0; /* We ignore the thread_join's exit value. */ 259 | enif_thread_join(we->tid, &exit_value); 260 | free(we); 261 | async_nif->we_active--; 262 | we = n; 263 | } 264 | 265 | if (async_nif->we_active == ASYNC_NIF_MAX_WORKERS) { 266 | enif_mutex_unlock(async_nif->we_mutex); 267 | return EAGAIN; 268 | } 269 | 270 | we = malloc(sizeof(struct async_nif_worker_entry)); 271 | if (!we) { 272 | enif_mutex_unlock(async_nif->we_mutex); 273 | return ENOMEM; 274 | } 275 | memset(we, 0, sizeof(struct async_nif_worker_entry)); 276 | we->worker_id = async_nif->we_active++; 277 | we->async_nif = async_nif; 278 | we->q = q; 279 | 280 | enif_mutex_unlock(async_nif->we_mutex); 281 | return enif_thread_create(NULL,&we->tid, &async_nif_worker_fn, (void*)we, 0); 282 | } 283 | 284 | /** 285 | * Enqueue a request for processing by a worker thread. 286 | * 287 | * Places the request into a work queue determined either by the 288 | * provided affinity or by iterating through the available queues. 289 | */ 290 | static ERL_NIF_TERM 291 | async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_entry *req, int hint) 292 | { 293 | /* Identify the most appropriate worker for this request. */ 294 | unsigned int i, last_qid, qid = 0; 295 | struct async_nif_work_queue *q = NULL; 296 | double avg_depth = 0.0; 297 | 298 | /* Either we're choosing a queue based on some affinity/hinted value or we 299 | need to select the next queue in the rotation and atomically update that 300 | global value (next_q is shared across worker threads) . */ 301 | if (hint >= 0) { 302 | qid = (unsigned int)hint; 303 | } else { 304 | do { 305 | last_qid = __sync_fetch_and_add(&async_nif->next_q, 0); 306 | qid = (last_qid + 1) % async_nif->num_queues; 307 | } while (!__sync_bool_compare_and_swap(&async_nif->next_q, last_qid, qid)); 308 | } 309 | 310 | /* Now we inspect and interate across the set of queues trying to select one 311 | that isn't too full or too slow. */ 312 | for (i = 0; i < async_nif->num_queues; i++) { 313 | /* Compute the average queue depth not counting queues which are empty or 314 | the queue we're considering right now. */ 315 | unsigned int j, n = 0; 316 | for (j = 0; j < async_nif->num_queues; j++) { 317 | if (j != qid && async_nif->queues[j].depth != 0) { 318 | n++; 319 | avg_depth += async_nif->queues[j].depth; 320 | } 321 | } 322 | if (avg_depth) avg_depth /= n; 323 | 324 | /* Lock this queue under consideration, then check for shutdown. While 325 | we hold this lock either a) we're shutting down so exit now or b) this 326 | queue will be valid until we release the lock. */ 327 | q = &async_nif->queues[qid]; 328 | enif_mutex_lock(q->reqs_mutex); 329 | 330 | /* Try not to enqueue a request into a queue that isn't keeping up with 331 | the request volume. */ 332 | if (q->depth <= avg_depth) break; 333 | else { 334 | enif_mutex_unlock(q->reqs_mutex); 335 | qid = (qid + 1) % async_nif->num_queues; 336 | } 337 | } 338 | 339 | /* If the for loop finished then we didn't find a suitable queue for this 340 | request, meaning we're backed up so trigger eagain. Note that if we left 341 | the loop in this way we hold no lock. */ 342 | if (i == async_nif->num_queues) return 0; 343 | 344 | /* Add the request to the queue. */ 345 | STAILQ_INSERT_TAIL(&q->reqs, req, entries); 346 | __sync_fetch_and_add(&q->depth, 1); 347 | 348 | /* We've selected a queue for this new request now check to make sure there are 349 | enough workers actively processing requests on this queue. */ 350 | while (q->depth > q->num_workers) { 351 | switch(async_nif_start_worker(async_nif, q)) { 352 | case EINVAL: case ENOMEM: default: return 0; 353 | case EAGAIN: continue; 354 | case 0: __sync_fetch_and_add(&q->num_workers, 1); goto done; 355 | } 356 | }done:; 357 | 358 | /* Build the term before releasing the lock so as not to race on the use of 359 | the req pointer (which will soon become invalid in another thread 360 | performing the request). */ 361 | double pct_full = (double)avg_depth / (double)ASYNC_NIF_WORKER_QUEUE_SIZE; 362 | ERL_NIF_TERM reply = enif_make_tuple2(req->env, ATOM_OK, 363 | enif_make_tuple2(req->env, ATOM_ENQUEUED, 364 | enif_make_double(req->env, pct_full))); 365 | enif_cond_signal(q->reqs_cnd); 366 | enif_mutex_unlock(q->reqs_mutex); 367 | return reply; 368 | } 369 | 370 | /** 371 | * Worker threads execute this function. Here each worker pulls requests of 372 | * their respective queues, executes that work and continues doing that until 373 | * they see the shutdown flag is set at which point they exit. 374 | */ 375 | static void * 376 | async_nif_worker_fn(void *arg) 377 | { 378 | struct async_nif_worker_entry *we = (struct async_nif_worker_entry *)arg; 379 | unsigned int worker_id = we->worker_id; 380 | struct async_nif_state *async_nif = we->async_nif; 381 | struct async_nif_work_queue *q = we->q; 382 | struct async_nif_req_entry *req = NULL; 383 | unsigned int tries = async_nif->num_queues; 384 | 385 | for(;;) { 386 | /* Examine the request queue, are there things to be done? */ 387 | enif_mutex_lock(q->reqs_mutex); 388 | check_again_for_work: 389 | if (async_nif->shutdown) { 390 | enif_mutex_unlock(q->reqs_mutex); 391 | break; 392 | } 393 | if (STAILQ_EMPTY(&q->reqs)) { 394 | /* Queue is empty so we wait for more work to arrive. */ 395 | enif_mutex_unlock(q->reqs_mutex); 396 | if (tries == 0 && q == we->q) { 397 | if (q->num_workers > ASYNC_NIF_MIN_WORKERS) { 398 | /* At this point we've tried to find/execute work on all queues 399 | * and there are at least MIN_WORKERS on this queue so we 400 | * leaving this loop (break) which leads to a thread exit/join. */ 401 | break; 402 | } else { 403 | enif_mutex_lock(q->reqs_mutex); 404 | enif_cond_wait(q->reqs_cnd, q->reqs_mutex); 405 | goto check_again_for_work; 406 | } 407 | } else { 408 | tries--; 409 | __sync_fetch_and_add(&q->num_workers, -1); 410 | q = q->next; 411 | __sync_fetch_and_add(&q->num_workers, 1); 412 | continue; // try next queue 413 | } 414 | } else { 415 | /* At this point the next req is ours to process and we hold the 416 | reqs_mutex lock. Take the request off the queue. */ 417 | req = STAILQ_FIRST(&q->reqs); 418 | STAILQ_REMOVE(&q->reqs, req, async_nif_req_entry, entries); 419 | __sync_fetch_and_add(&q->depth, -1); 420 | 421 | /* Wake up other worker thread watching this queue to help process work. */ 422 | enif_cond_signal(q->reqs_cnd); 423 | enif_mutex_unlock(q->reqs_mutex); 424 | 425 | /* Perform the work. */ 426 | req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args); 427 | 428 | /* Now call the post-work cleanup function. */ 429 | req->fn_post(req->args); 430 | 431 | /* Clean up req for reuse. */ 432 | req->ref = 0; 433 | req->fn_work = 0; 434 | req->fn_post = 0; 435 | free(req->args); 436 | req->args = NULL; 437 | async_nif_recycle_req(req, async_nif); 438 | req = NULL; 439 | } 440 | } 441 | enif_mutex_lock(async_nif->we_mutex); 442 | SLIST_INSERT_HEAD(&async_nif->we_joining, we, entries); 443 | enif_mutex_unlock(async_nif->we_mutex); 444 | __sync_fetch_and_add(&q->num_workers, -1); 445 | enif_thread_exit(0); 446 | return 0; 447 | } 448 | 449 | static void 450 | async_nif_unload(ErlNifEnv *env, struct async_nif_state *async_nif) 451 | { 452 | unsigned int i; 453 | unsigned int num_queues = async_nif->num_queues; 454 | struct async_nif_work_queue *q = NULL; 455 | struct async_nif_req_entry *req = NULL; 456 | struct async_nif_worker_entry *we = NULL; 457 | UNUSED(env); 458 | 459 | /* Signal the worker threads, stop what you're doing and exit. To ensure 460 | that we don't race with the enqueue() process we first lock all the worker 461 | queues, then set shutdown to true, then unlock. The enqueue function will 462 | take the queue mutex, then test for shutdown condition, then enqueue only 463 | if not shutting down. */ 464 | for (i = 0; i < num_queues; i++) { 465 | q = &async_nif->queues[i]; 466 | enif_mutex_lock(q->reqs_mutex); 467 | } 468 | /* Set the shutdown flag so that worker threads will no continue 469 | executing requests. */ 470 | async_nif->shutdown = 1; 471 | for (i = 0; i < num_queues; i++) { 472 | q = &async_nif->queues[i]; 473 | enif_mutex_unlock(q->reqs_mutex); 474 | } 475 | 476 | /* Join for the now exiting worker threads. */ 477 | while(async_nif->we_active > 0) { 478 | for (i = 0; i < num_queues; i++) 479 | enif_cond_broadcast(async_nif->queues[i].reqs_cnd); 480 | enif_mutex_lock(async_nif->we_mutex); 481 | we = SLIST_FIRST(&async_nif->we_joining); 482 | while(we != NULL) { 483 | struct async_nif_worker_entry *n = SLIST_NEXT(we, entries); 484 | SLIST_REMOVE(&async_nif->we_joining, we, async_nif_worker_entry, entries); 485 | void *exit_value = 0; /* We ignore the thread_join's exit value. */ 486 | enif_thread_join(we->tid, &exit_value); 487 | free(we); 488 | async_nif->we_active--; 489 | we = n; 490 | } 491 | enif_mutex_unlock(async_nif->we_mutex); 492 | } 493 | enif_mutex_destroy(async_nif->we_mutex); 494 | 495 | /* Cleanup in-flight requests, mutexes and conditions in each work queue. */ 496 | for (i = 0; i < num_queues; i++) { 497 | q = &async_nif->queues[i]; 498 | 499 | /* Worker threads are stopped, now toss anything left in the queue. */ 500 | req = NULL; 501 | req = STAILQ_FIRST(&q->reqs); 502 | while(req != NULL) { 503 | struct async_nif_req_entry *n = STAILQ_NEXT(req, entries); 504 | enif_clear_env(req->env); 505 | enif_send(NULL, &req->pid, req->env, 506 | enif_make_tuple2(req->env, ATOM_ERROR, ATOM_SHUTDOWN)); 507 | req->fn_post(req->args); 508 | enif_free_env(req->env); 509 | free(req->args); 510 | free(req); 511 | req = n; 512 | } 513 | enif_mutex_destroy(q->reqs_mutex); 514 | enif_cond_destroy(q->reqs_cnd); 515 | } 516 | 517 | /* Free any req structures sitting unused on the recycle queue. */ 518 | enif_mutex_lock(async_nif->recycled_req_mutex); 519 | req = NULL; 520 | req = STAILQ_FIRST(&async_nif->recycled_reqs); 521 | while(req != NULL) { 522 | struct async_nif_req_entry *n = STAILQ_NEXT(req, entries); 523 | enif_free_env(req->env); 524 | free(req); 525 | req = n; 526 | } 527 | 528 | enif_mutex_unlock(async_nif->recycled_req_mutex); 529 | enif_mutex_destroy(async_nif->recycled_req_mutex); 530 | memset(async_nif, 0, sizeof(struct async_nif_state) + (sizeof(struct async_nif_work_queue) * async_nif->num_queues)); 531 | free(async_nif); 532 | } 533 | 534 | static void * 535 | async_nif_load(ErlNifEnv *env) 536 | { 537 | static int has_init = 0; 538 | unsigned int i, num_queues; 539 | ErlNifSysInfo info; 540 | struct async_nif_state *async_nif; 541 | 542 | /* Don't init more than once. */ 543 | if (has_init) return 0; 544 | else has_init = 1; 545 | 546 | /* Init some static references to commonly used atoms. */ 547 | ATOM_EAGAIN = enif_make_atom(env, "eagain"); 548 | ATOM_ENOMEM = enif_make_atom(env, "enomem"); 549 | ATOM_ENQUEUED = enif_make_atom(env, "enqueued"); 550 | ATOM_ERROR = enif_make_atom(env, "error"); 551 | ATOM_OK = enif_make_atom(env, "ok"); 552 | ATOM_SHUTDOWN = enif_make_atom(env, "shutdown"); 553 | 554 | /* Find out how many schedulers there are. */ 555 | enif_system_info(&info, sizeof(ErlNifSysInfo)); 556 | 557 | /* Size the number of work queues according to schedulers. */ 558 | if (info.scheduler_threads > ASYNC_NIF_MAX_WORKERS / 2) { 559 | num_queues = ASYNC_NIF_MAX_WORKERS / 2; 560 | } else { 561 | int remainder = ASYNC_NIF_MAX_WORKERS % info.scheduler_threads; 562 | if (remainder != 0) 563 | num_queues = info.scheduler_threads - remainder; 564 | else 565 | num_queues = info.scheduler_threads; 566 | if (num_queues < 2) 567 | num_queues = 2; 568 | } 569 | 570 | /* Init our portion of priv_data's module-specific state. */ 571 | async_nif = malloc(sizeof(struct async_nif_state) + 572 | sizeof(struct async_nif_work_queue) * num_queues); 573 | if (!async_nif) 574 | return NULL; 575 | memset(async_nif, 0, sizeof(struct async_nif_state) + 576 | sizeof(struct async_nif_work_queue) * num_queues); 577 | 578 | async_nif->num_queues = num_queues; 579 | async_nif->we_active = 0; 580 | async_nif->next_q = 0; 581 | async_nif->shutdown = 0; 582 | STAILQ_INIT(&async_nif->recycled_reqs); 583 | async_nif->recycled_req_mutex = enif_mutex_create("recycled_req"); 584 | async_nif->we_mutex = enif_mutex_create("we"); 585 | SLIST_INIT(&async_nif->we_joining); 586 | 587 | for (i = 0; i < async_nif->num_queues; i++) { 588 | struct async_nif_work_queue *q = &async_nif->queues[i]; 589 | STAILQ_INIT(&q->reqs); 590 | q->reqs_mutex = enif_mutex_create("reqs"); 591 | q->reqs_cnd = enif_cond_create("reqs"); 592 | q->next = &async_nif->queues[(i + 1) % num_queues]; 593 | } 594 | return async_nif; 595 | } 596 | 597 | static void 598 | async_nif_upgrade(ErlNifEnv *env) 599 | { 600 | UNUSED(env); 601 | // TODO: 602 | } 603 | 604 | 605 | #if defined(__cplusplus) 606 | } 607 | #endif 608 | 609 | #endif // __ASYNC_NIF_H__ 610 | -------------------------------------------------------------------------------- /c_src/lmdb_nif.c: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------------- 2 | * This file is part of LMDB - Erlang Lightning MDB API 3 | * 4 | * Copyright (c) 2012 by Aleph Archives. All rights reserved. 5 | * Copyright (c) 2013 by Basho Technologies, Inc. All rights reserved. 6 | * 7 | * ------------------------------------------------------------------------- 8 | * Redistribution and use in source and binary forms, with or without 9 | * modification, are permitted only as authorized by the OpenLDAP 10 | * Public License. 11 | * 12 | * A copy of this license is available in the file LICENSE in the 13 | * top-level directory of the distribution or, alternatively, at 14 | * . 15 | * 16 | * Permission to use, copy, modify, and distribute this software for any 17 | * purpose with or without fee is hereby granted, provided that the above 18 | * copyright notice and this permission notice appear in all copies. 19 | * 20 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 21 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 22 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 23 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 24 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 25 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 26 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 27 | * -------------------------------------------------------------------------*/ 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include "common.h" 38 | #include "async_nif.h" 39 | #include "lmdb.h" 40 | 41 | static ErlNifResourceType *lmdb_RESOURCE; 42 | struct lmdb { 43 | MDB_env *env; 44 | MDB_txn *txn; 45 | MDB_cursor *cursor; 46 | MDB_dbi dbi; 47 | }; 48 | 49 | struct lmdb_priv_data { 50 | void *async_nif_priv; // Note: must be first element in struct 51 | }; 52 | 53 | /* Global init for async_nif. */ 54 | ASYNC_NIF_INIT(lmdb); 55 | 56 | /* Atoms (initialized in on_load) */ 57 | static ERL_NIF_TERM ATOM_ERROR; 58 | static ERL_NIF_TERM ATOM_OK; 59 | static ERL_NIF_TERM ATOM_NOT_FOUND; 60 | static ERL_NIF_TERM ATOM_EXISTS; 61 | static ERL_NIF_TERM ATOM_KEYEXIST; 62 | static ERL_NIF_TERM ATOM_NOTFOUND; 63 | static ERL_NIF_TERM ATOM_PAGE_NOTFOUND; 64 | static ERL_NIF_TERM ATOM_CORRUPTED; 65 | static ERL_NIF_TERM ATOM_PANIC; 66 | static ERL_NIF_TERM ATOM_VERSION_MISMATCH; 67 | static ERL_NIF_TERM ATOM_KEYEXIST; 68 | static ERL_NIF_TERM ATOM_MAP_FULL; 69 | static ERL_NIF_TERM ATOM_DBS_FULL; 70 | static ERL_NIF_TERM ATOM_READERS_FULL; 71 | static ERL_NIF_TERM ATOM_TLS_FULL; 72 | static ERL_NIF_TERM ATOM_TXN_FULL; 73 | static ERL_NIF_TERM ATOM_CURSOR_FULL; 74 | static ERL_NIF_TERM ATOM_PAGE_FULL; 75 | static ERL_NIF_TERM ATOM_MAP_RESIZED; 76 | static ERL_NIF_TERM ATOM_INCOMPATIBLE; 77 | static ERL_NIF_TERM ATOM_BAD_RSLOT; 78 | 79 | static ERL_NIF_TERM ATOM_TXN_STARTED; 80 | static ERL_NIF_TERM ATOM_TXN_NOT_STARTED; 81 | 82 | #define CHECK(expr, label) \ 83 | if (MDB_SUCCESS != (ret = (expr))) { \ 84 | DPRINTF("CHECK(\"%s\") failed \"%s\" at %s:%d in %s()\n", \ 85 | #expr, mdb_strerror(ret), __FILE__, __LINE__, __func__);\ 86 | err = __strerror_term(env, ret); \ 87 | goto label; \ 88 | } 89 | 90 | #define FAIL_ERR(e, label) \ 91 | do { \ 92 | err = __strerror_term(env, (e)); \ 93 | goto label; \ 94 | } while(0) 95 | 96 | /** 97 | * Convenience function to generate {error, {errno, Reason}} 98 | * 99 | * env NIF environment 100 | * err number of last error 101 | */ 102 | static ERL_NIF_TERM 103 | __strerror_term(ErlNifEnv* env, int err) 104 | { 105 | ERL_NIF_TERM term = 0; 106 | 107 | if (err < MDB_LAST_ERRCODE && err > MDB_KEYEXIST) { 108 | switch (err) { 109 | case MDB_KEYEXIST: /** key/data pair already exists */ 110 | term = ATOM_KEYEXIST; 111 | break; 112 | case MDB_NOTFOUND: /** key/data pair not found (EOF) */ 113 | term = ATOM_NOTFOUND; 114 | break; 115 | case MDB_PAGE_NOTFOUND: /** Requested page not found - this usually indicates corruption */ 116 | term = ATOM_PAGE_NOTFOUND; 117 | break; 118 | case MDB_CORRUPTED: /** Located page was wrong type */ 119 | term = ATOM_CORRUPTED; 120 | break; 121 | case MDB_PANIC : /** Update of meta page failed, probably I/O error */ 122 | term = ATOM_PANIC; 123 | break; 124 | case MDB_VERSION_MISMATCH: /** Environment version mismatch */ 125 | term = ATOM_VERSION_MISMATCH; 126 | break; 127 | case MDB_INVALID: /** File is not a valid MDB file */ 128 | term = ATOM_KEYEXIST; 129 | break; 130 | case MDB_MAP_FULL: /** Environment mapsize reached */ 131 | term = ATOM_MAP_FULL; 132 | break; 133 | case MDB_DBS_FULL: /** Environment maxdbs reached */ 134 | term = ATOM_DBS_FULL; 135 | break; 136 | case MDB_READERS_FULL: /** Environment maxreaders reached */ 137 | term = ATOM_READERS_FULL; 138 | break; 139 | case MDB_TLS_FULL: /** Too many TLS keys in use - Windows only */ 140 | term = ATOM_TLS_FULL; 141 | break; 142 | case MDB_TXN_FULL: /** Txn has too many dirty pages */ 143 | term = ATOM_TXN_FULL; 144 | break; 145 | case MDB_CURSOR_FULL: /** Cursor stack too deep - internal error */ 146 | term = ATOM_CURSOR_FULL; 147 | break; 148 | case MDB_PAGE_FULL: /** Page has not enough space - internal error */ 149 | term = ATOM_PAGE_FULL; 150 | break; 151 | case MDB_MAP_RESIZED: /** Database contents grew beyond environment mapsize */ 152 | term = ATOM_MAP_RESIZED; 153 | break; 154 | case MDB_INCOMPATIBLE: /** Database flags changed or would change */ 155 | term = ATOM_INCOMPATIBLE; 156 | break; 157 | case MDB_BAD_RSLOT: /** Invalid reuse of reader locktable slot */ 158 | term = ATOM_BAD_RSLOT; 159 | break; 160 | } 161 | } else { 162 | term = enif_make_atom(env, erl_errno_id(err)); 163 | } 164 | 165 | /* We return the errno value as well as the message here because the error 166 | message provided by strerror() for differ across platforms and/or may be 167 | localized to any given language (i18n). Use the errno atom rather than 168 | the message when matching in Erlang. You've been warned. */ 169 | return enif_make_tuple(env, 2, ATOM_ERROR, 170 | enif_make_tuple(env, 2, term, 171 | enif_make_string(env, mdb_strerror(err), ERL_NIF_LATIN1))); 172 | } 173 | 174 | /** 175 | * Opens a MDB database. 176 | * 177 | * argv[0] path to directory for the database files 178 | * argv[1] size of database 179 | * argv[2] flags 180 | */ 181 | ASYNC_NIF_DECL( 182 | lmdb_open, 183 | { // struct 184 | 185 | char dirname[MAXPATHLEN]; 186 | ErlNifUInt64 mapsize; 187 | ErlNifUInt64 envflags; 188 | }, 189 | { // pre 190 | if (!(argc == 3 && 191 | enif_is_list(env, argv[0]) && 192 | enif_is_number(env, argv[1]) && 193 | enif_is_number(env, argv[2]))) { 194 | ASYNC_NIF_RETURN_BADARG(); 195 | } 196 | if (enif_get_string(env, argv[0], args->dirname, 197 | MAXPATHLEN, ERL_NIF_LATIN1) <= 0) 198 | ASYNC_NIF_RETURN_BADARG(); 199 | enif_get_uint64(env, argv[1], &(args->mapsize)); 200 | enif_get_uint64(env, argv[2], &(args->envflags)); 201 | }, 202 | { // work 203 | 204 | ERL_NIF_TERM err; 205 | MDB_txn *txn; 206 | struct lmdb *handle; 207 | int ret; 208 | 209 | if ((handle = enif_alloc_resource(lmdb_RESOURCE, sizeof(struct lmdb))) == NULL) 210 | FAIL_ERR(ENOMEM, err3); 211 | 212 | CHECK(mdb_env_create(&(handle->env)), err2); 213 | 214 | if (mdb_env_set_mapsize(handle->env, args->mapsize)) { 215 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 216 | return; 217 | } 218 | 219 | CHECK(mdb_env_open(handle->env, args->dirname, args->envflags, 0664), err2); 220 | CHECK(mdb_txn_begin(handle->env, NULL, 0, &txn), err2); 221 | CHECK(mdb_open(txn, NULL, 0, &(handle->dbi)), err1); 222 | CHECK(mdb_txn_commit(txn), err1); 223 | 224 | handle->txn = NULL; 225 | handle->cursor = NULL; 226 | 227 | ERL_NIF_TERM term = enif_make_resource(env, handle); 228 | enif_release_resource(handle); 229 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_OK, term)); 230 | return; 231 | 232 | err1: 233 | mdb_txn_abort(txn); 234 | err2: 235 | mdb_env_close(handle->env); 236 | err3: 237 | ASYNC_NIF_REPLY(err); 238 | return; 239 | }, 240 | { // post 241 | 242 | }); 243 | 244 | 245 | /** 246 | * Closes a MDB database. 247 | * 248 | * argv[0] reference to the MDB handle resource 249 | */ 250 | ASYNC_NIF_DECL( 251 | lmdb_close, 252 | { // struct 253 | 254 | struct lmdb *handle; 255 | }, 256 | { // pre 257 | 258 | if (!(argc == 1 && 259 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle))) { 260 | ASYNC_NIF_RETURN_BADARG(); 261 | } 262 | if (!args->handle->env) 263 | ASYNC_NIF_RETURN_BADARG(); 264 | enif_keep_resource((void*)args->handle); 265 | }, 266 | { // work 267 | 268 | mdb_env_close(args->handle->env); 269 | args->handle->env = NULL; 270 | ASYNC_NIF_REPLY(ATOM_OK); 271 | return; 272 | }, 273 | { // post 274 | 275 | enif_release_resource((void*)args->handle); 276 | }); 277 | 278 | 279 | /** 280 | * Store a value indexed by key. 281 | * 282 | * argv[0] reference to the MDB handle resource 283 | * argv[1] key as an Erlang binary 284 | * argv[2] value as an Erlang binary 285 | */ 286 | ASYNC_NIF_DECL( 287 | lmdb_put, 288 | { // struct 289 | 290 | struct lmdb *handle; 291 | ERL_NIF_TERM key; 292 | ERL_NIF_TERM val; 293 | }, 294 | { // pre 295 | 296 | if (!(argc == 3 && 297 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle) && 298 | enif_is_binary(env, argv[1]) && 299 | enif_is_binary(env, argv[2]) )) { 300 | ASYNC_NIF_RETURN_BADARG(); 301 | } 302 | if (!args->handle->env) 303 | ASYNC_NIF_RETURN_BADARG(); 304 | enif_keep_resource((void*)args->handle); 305 | args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); 306 | args->val = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); 307 | }, 308 | { // work 309 | 310 | ERL_NIF_TERM err; 311 | ErlNifBinary key; 312 | ErlNifBinary val; 313 | MDB_val mkey; 314 | MDB_val mdata; 315 | MDB_txn * txn; 316 | int ret; 317 | 318 | if (!enif_inspect_iolist_as_binary(env, args->key, &key)) { 319 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 320 | return; 321 | } 322 | if (!enif_inspect_iolist_as_binary(env, args->val, &val)) { 323 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 324 | return; 325 | } 326 | 327 | mkey.mv_size = key.size; 328 | mkey.mv_data = key.data; 329 | mdata.mv_size = val.size; 330 | mdata.mv_data = val.data; 331 | if(args->handle->txn == NULL) { 332 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, & txn), err2); 333 | } else { 334 | txn = args->handle->txn; 335 | } 336 | 337 | ret = mdb_put(txn, args->handle->dbi, &mkey, &mdata, MDB_NOOVERWRITE); 338 | if (MDB_KEYEXIST == ret) { 339 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_ERROR, ATOM_EXISTS)); 340 | return; 341 | } 342 | if (ret != 0) 343 | FAIL_ERR(ret, err1); 344 | 345 | if(args->handle->txn == NULL) 346 | CHECK(mdb_txn_commit(txn), err1); 347 | ASYNC_NIF_REPLY(ATOM_OK); 348 | return; 349 | 350 | err1: 351 | mdb_txn_abort(txn); 352 | err2: 353 | ASYNC_NIF_REPLY(err); 354 | return; 355 | }, 356 | { // post 357 | 358 | enif_release_resource((void*)args->handle); 359 | }); 360 | 361 | /** 362 | * Update and existin value indexed by key. 363 | * 364 | * argv[0] reference to the MDB handle resource 365 | * argv[1] key as an Erlang binary 366 | * argv[2] value as an Erlang binary 367 | */ 368 | ASYNC_NIF_DECL( 369 | lmdb_update, 370 | { // struct 371 | 372 | struct lmdb *handle; 373 | ERL_NIF_TERM key; 374 | ERL_NIF_TERM val; 375 | }, 376 | { // pre 377 | 378 | if (!(argc == 3 && 379 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle) && 380 | enif_is_binary(env, argv[1]) && 381 | enif_is_binary(env, argv[2]) )) { 382 | ASYNC_NIF_RETURN_BADARG(); 383 | } 384 | if (!args->handle->env) 385 | ASYNC_NIF_RETURN_BADARG(); 386 | enif_keep_resource((void*)args->handle); 387 | args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); 388 | args->val = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); 389 | }, 390 | { // work 391 | 392 | ERL_NIF_TERM err; 393 | ErlNifBinary key; 394 | ErlNifBinary val; 395 | MDB_val mkey; 396 | MDB_val mdata; 397 | MDB_txn * txn; 398 | int ret; 399 | 400 | if (!enif_inspect_iolist_as_binary(env, args->key, &key)) { 401 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 402 | return; 403 | } 404 | if (!enif_inspect_iolist_as_binary(env, args->val, &val)) { 405 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 406 | return; 407 | } 408 | 409 | mkey.mv_size = key.size; 410 | mkey.mv_data = key.data; 411 | mdata.mv_size = val.size; 412 | mdata.mv_data = val.data; 413 | 414 | if(args->handle->txn == NULL) { 415 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, & txn), err2); 416 | } else { 417 | txn = args->handle->txn; 418 | } 419 | 420 | CHECK(mdb_put(txn, args->handle->dbi, &mkey, &mdata, 0), err1); 421 | 422 | if(args->handle->txn == NULL) 423 | CHECK(mdb_txn_commit(txn), err1); 424 | ASYNC_NIF_REPLY(ATOM_OK); 425 | return; 426 | 427 | err1: 428 | mdb_txn_abort(txn); 429 | err2: 430 | ASYNC_NIF_REPLY(err); 431 | return; 432 | }, 433 | { // post 434 | 435 | enif_release_resource((void*)args->handle); 436 | }); 437 | 438 | 439 | /** 440 | * Retrieve the value associated with the key. 441 | * 442 | * argv[0] reference to the MDB handle resource 443 | * argv[1] key as an Erlang binary 444 | */ 445 | ASYNC_NIF_DECL( 446 | lmdb_get, 447 | { // struct 448 | 449 | struct lmdb *handle; 450 | ERL_NIF_TERM key; 451 | }, 452 | { // pre 453 | 454 | if (!(argc == 2 && 455 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle) && 456 | enif_is_binary(env, argv[1]) )) { 457 | ASYNC_NIF_RETURN_BADARG(); 458 | } 459 | if (!args->handle->env) 460 | ASYNC_NIF_RETURN_BADARG(); 461 | enif_keep_resource((void*)args->handle); 462 | args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); 463 | }, 464 | { // work 465 | 466 | ERL_NIF_TERM err; 467 | ErlNifBinary key; 468 | ERL_NIF_TERM val; 469 | unsigned char *bin; 470 | MDB_val mkey; 471 | MDB_val mdata; 472 | MDB_txn * txn; 473 | int ret; 474 | 475 | if (!enif_inspect_iolist_as_binary(env, args->key, &key)) { 476 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 477 | return; 478 | } 479 | 480 | mkey.mv_size = key.size; 481 | mkey.mv_data = key.data; 482 | 483 | if(args->handle->txn == NULL) { 484 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, & txn), err); 485 | } else { 486 | txn = args->handle->txn; 487 | } 488 | 489 | ret = mdb_get(txn, args->handle->dbi, &mkey, &mdata); 490 | if(args->handle->txn == NULL) 491 | mdb_txn_abort(txn); 492 | if (MDB_NOTFOUND == ret) { 493 | ASYNC_NIF_REPLY(ATOM_NOT_FOUND); 494 | return; 495 | } 496 | 497 | if (ret != 0) 498 | FAIL_ERR(ret, err); 499 | 500 | bin = enif_make_new_binary(env, mdata.mv_size, &val); 501 | if (!bin) 502 | FAIL_ERR(ENOMEM, err); 503 | memcpy(bin, mdata.mv_data, mdata.mv_size); 504 | 505 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_OK, val)); 506 | return; 507 | 508 | err: 509 | ASYNC_NIF_REPLY(err); 510 | return; 511 | }, 512 | { // post 513 | 514 | enif_release_resource((void*)args->handle); 515 | }); 516 | 517 | 518 | /** 519 | * Delete the value associated with the key. 520 | * 521 | * argv[0] reference to the MDB handle resource 522 | * argv[1] key as an Erlang binary 523 | */ 524 | ASYNC_NIF_DECL( 525 | lmdb_del, 526 | { // struct 527 | 528 | struct lmdb *handle; 529 | ERL_NIF_TERM key; 530 | }, 531 | { // pre 532 | 533 | if (!(argc == 2 && 534 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle) && 535 | enif_is_binary(env, argv[1]) )) { 536 | ASYNC_NIF_RETURN_BADARG(); 537 | } 538 | if (!args->handle->env) 539 | ASYNC_NIF_RETURN_BADARG(); 540 | enif_keep_resource((void*)args->handle); 541 | args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); 542 | }, 543 | { // work 544 | 545 | ERL_NIF_TERM err; 546 | ErlNifBinary key; 547 | MDB_val mkey; 548 | MDB_txn * txn; 549 | int ret; 550 | 551 | if (!enif_inspect_iolist_as_binary(env, args->key, &key)) { 552 | ASYNC_NIF_REPLY(enif_make_badarg(env)); 553 | return; 554 | } 555 | 556 | mkey.mv_size = key.size; 557 | mkey.mv_data = key.data; 558 | 559 | if(args->handle->txn == NULL) { 560 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, & txn), err); 561 | } else { 562 | txn = args->handle->txn; 563 | } 564 | 565 | ret = mdb_del(txn, args->handle->dbi, &mkey, NULL); 566 | 567 | if(MDB_NOTFOUND == ret) { 568 | if(args->handle->txn == NULL) 569 | mdb_txn_abort(txn); 570 | ASYNC_NIF_REPLY(ATOM_NOT_FOUND); 571 | return; 572 | } 573 | if(args->handle->txn == NULL) 574 | CHECK(mdb_txn_commit(txn), err); 575 | ASYNC_NIF_REPLY(ATOM_OK); 576 | return; 577 | 578 | err: 579 | ASYNC_NIF_REPLY(err); 580 | return; 581 | }, 582 | { // post 583 | 584 | enif_release_resource((void*)args->handle); 585 | }); 586 | 587 | 588 | /** 589 | * Drop a MDB database. 590 | * 591 | * argv[0] reference to the MDB handle resource 592 | */ 593 | ASYNC_NIF_DECL( 594 | lmdb_drop, 595 | { // struct 596 | 597 | struct lmdb *handle; 598 | }, 599 | { // pre 600 | 601 | if (!(argc == 1 && 602 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle))) { 603 | ASYNC_NIF_RETURN_BADARG(); 604 | } 605 | if (!args->handle->env) 606 | ASYNC_NIF_RETURN_BADARG(); 607 | enif_keep_resource((void*)args->handle); 608 | }, 609 | { // work 610 | 611 | ERL_NIF_TERM err; 612 | MDB_txn * txn; 613 | int ret; 614 | 615 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, & txn), err2); 616 | CHECK(mdb_drop(txn, args->handle->dbi, 0), err1); 617 | CHECK(mdb_txn_commit(txn), err1); 618 | ASYNC_NIF_REPLY(ATOM_OK); 619 | return; 620 | 621 | err1: 622 | mdb_txn_abort(txn); 623 | 624 | err2: 625 | ASYNC_NIF_REPLY(err); 626 | return; 627 | }, 628 | { // post 629 | 630 | enif_release_resource((void*)args->handle); 631 | }); 632 | 633 | ASYNC_NIF_DECL( 634 | lmdb_txn_begin, 635 | { // struct 636 | 637 | struct lmdb *handle; 638 | }, 639 | { // pre 640 | 641 | if (!(argc == 1 && 642 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle))) { 643 | ASYNC_NIF_RETURN_BADARG(); 644 | } 645 | if (!args->handle->env) 646 | ASYNC_NIF_RETURN_BADARG(); 647 | enif_keep_resource((void*)args->handle); 648 | }, 649 | { // work 650 | 651 | ERL_NIF_TERM err; 652 | int ret; 653 | if(args->handle->txn == NULL) { 654 | CHECK(mdb_txn_begin(args->handle->env, NULL, 0, &(args->handle->txn)), err2); 655 | ASYNC_NIF_REPLY(ATOM_OK); 656 | } else 657 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_ERROR, ATOM_TXN_STARTED)); 658 | return; 659 | 660 | err2: 661 | ASYNC_NIF_REPLY(err); 662 | return; 663 | }, 664 | { // post 665 | 666 | enif_release_resource((void*)args->handle); 667 | }); 668 | 669 | ASYNC_NIF_DECL( 670 | lmdb_txn_commit, 671 | { // struct 672 | 673 | struct lmdb *handle; 674 | }, 675 | { // pre 676 | 677 | if (!(argc == 1 && 678 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle))) { 679 | ASYNC_NIF_RETURN_BADARG(); 680 | } 681 | if (!args->handle->env) 682 | ASYNC_NIF_RETURN_BADARG(); 683 | enif_keep_resource((void*)args->handle); 684 | }, 685 | { // work 686 | 687 | ERL_NIF_TERM err; 688 | int ret; 689 | if(args->handle->txn != NULL) { 690 | CHECK(mdb_txn_commit(args->handle->txn), err2); 691 | args->handle->txn = NULL; 692 | ASYNC_NIF_REPLY(ATOM_OK); 693 | } else 694 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_ERROR, ATOM_TXN_NOT_STARTED)); 695 | return; 696 | 697 | err2: 698 | ASYNC_NIF_REPLY(err); 699 | return; 700 | }, 701 | { // post 702 | 703 | enif_release_resource((void*)args->handle); 704 | }); 705 | 706 | ASYNC_NIF_DECL( 707 | lmdb_txn_abort, 708 | { // struct 709 | 710 | struct lmdb *handle; 711 | }, 712 | { // pre 713 | 714 | if (!(argc == 1 && 715 | enif_get_resource(env, argv[0], lmdb_RESOURCE, (void**)&args->handle))) { 716 | ASYNC_NIF_RETURN_BADARG(); 717 | } 718 | if (!args->handle->env) 719 | ASYNC_NIF_RETURN_BADARG(); 720 | enif_keep_resource((void*)args->handle); 721 | }, 722 | { // work 723 | 724 | if(args->handle->txn != NULL) { 725 | mdb_txn_abort(args->handle->txn); 726 | args->handle->txn = NULL; 727 | ASYNC_NIF_REPLY(ATOM_OK); 728 | } else 729 | ASYNC_NIF_REPLY(enif_make_tuple(env, 2, ATOM_ERROR, ATOM_TXN_NOT_STARTED)); 730 | return; 731 | }, 732 | { // post 733 | 734 | enif_release_resource((void*)args->handle); 735 | }); 736 | 737 | static int lmdb_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) 738 | { 739 | __UNUSED(load_info); 740 | 741 | ErlNifResourceFlags flags = ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER; 742 | 743 | struct lmdb_priv_data *priv = enif_alloc(sizeof(struct lmdb_priv_data)); 744 | if (!priv) 745 | return ENOMEM; 746 | memset(priv, 0, sizeof(struct lmdb_priv_data)); 747 | 748 | /* Note: !!! the first element of our priv_data struct *must* be the 749 | pointer to the async_nif's private data which we set here. */ 750 | ASYNC_NIF_LOAD(lmdb, env, priv->async_nif_priv); 751 | if (!priv) 752 | return ENOMEM; 753 | *priv_data = priv; 754 | 755 | ATOM_ERROR = enif_make_atom(env, "error"); 756 | ATOM_OK = enif_make_atom(env, "ok"); 757 | ATOM_NOT_FOUND = enif_make_atom(env, "not_found"); 758 | ATOM_EXISTS = enif_make_atom(env, "exists"); 759 | 760 | ATOM_KEYEXIST = enif_make_atom(env, "key_exist"); 761 | ATOM_NOTFOUND = enif_make_atom(env, "notfound"); 762 | ATOM_CORRUPTED = enif_make_atom(env, "corrupted"); 763 | ATOM_PANIC = enif_make_atom(env, "panic"); 764 | ATOM_VERSION_MISMATCH = enif_make_atom(env, "version_mismatch"); 765 | ATOM_MAP_FULL = enif_make_atom(env, "map_full"); 766 | ATOM_DBS_FULL = enif_make_atom(env, "dbs_full"); 767 | ATOM_READERS_FULL = enif_make_atom(env, "readers_full"); 768 | ATOM_TLS_FULL = enif_make_atom(env, "tls_full"); 769 | ATOM_TXN_FULL = enif_make_atom(env, "txn_full"); 770 | ATOM_CURSOR_FULL = enif_make_atom(env, "cursor_full"); 771 | ATOM_PAGE_FULL = enif_make_atom(env, "page_full"); 772 | ATOM_MAP_RESIZED = enif_make_atom(env, "map_resized"); 773 | ATOM_INCOMPATIBLE = enif_make_atom(env, "incompatible"); 774 | ATOM_BAD_RSLOT = enif_make_atom(env, "bad_rslot"); 775 | 776 | ATOM_TXN_STARTED = enif_make_atom(env, "txn_started"); 777 | ATOM_TXN_NOT_STARTED = enif_make_atom(env, "txn_not_started"); 778 | 779 | lmdb_RESOURCE = enif_open_resource_type(env, NULL, "lmdb_resource", 780 | NULL, flags, NULL); 781 | return (0); 782 | } 783 | 784 | static int lmdb_reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM info) 785 | { 786 | __UNUSED(env); 787 | __UNUSED(priv_data); 788 | __UNUSED(info); 789 | return (0); // TODO: 790 | } 791 | 792 | 793 | static int lmdb_upgrade(ErlNifEnv* env, void** priv_data, void** old_priv, ERL_NIF_TERM load_info) 794 | { 795 | __UNUSED(env); 796 | __UNUSED(priv_data); 797 | __UNUSED(old_priv); 798 | __UNUSED(load_info); 799 | ASYNC_NIF_UPGRADE(lmdb, env); 800 | return (0); // TODO: 801 | } 802 | 803 | 804 | static void lmdb_unload(ErlNifEnv* env, void* priv_data) 805 | { 806 | struct lmdb_priv_data *priv = (struct lmdb_priv_data *)priv_data; 807 | ASYNC_NIF_UNLOAD(lmdb, env, priv->async_nif_priv); 808 | enif_free(priv); 809 | return; 810 | } 811 | 812 | static ErlNifFunc nif_funcs [] = { 813 | {"open", 4, lmdb_open}, 814 | {"close", 2, lmdb_close}, 815 | {"put", 4, lmdb_put}, 816 | {"get", 3, lmdb_get}, 817 | {"del", 3, lmdb_del}, 818 | {"update", 4, lmdb_update}, 819 | {"drop", 2, lmdb_drop}, 820 | 821 | {"txn_begin", 2, lmdb_txn_begin}, 822 | {"txn_commit", 2, lmdb_txn_commit}, 823 | {"txn_abort", 2, lmdb_txn_abort}/*, 824 | 825 | {"cursor_open", 2, lmdb_cursor_open}, 826 | {"cursor_close", 2, lmdb_cursor_close} */ 827 | 828 | }; 829 | 830 | /* driver entry point */ 831 | ERL_NIF_INIT(lmdb, 832 | nif_funcs, 833 | & lmdb_load, 834 | & lmdb_reload, 835 | & lmdb_upgrade, 836 | & lmdb_unload) 837 | -------------------------------------------------------------------------------- /c_src/lmdb.h: -------------------------------------------------------------------------------- 1 | /** @file lmdb.h 2 | * @brief Lightning memory-mapped database library 3 | * 4 | * @mainpage Lightning Memory-Mapped Database Manager (LMDB) 5 | * 6 | * @section intro_sec Introduction 7 | * LMDB is a Btree-based database management library modeled loosely on the 8 | * BerkeleyDB API, but much simplified. The entire database is exposed 9 | * in a memory map, and all data fetches return data directly 10 | * from the mapped memory, so no malloc's or memcpy's occur during 11 | * data fetches. As such, the library is extremely simple because it 12 | * requires no page caching layer of its own, and it is extremely high 13 | * performance and memory-efficient. It is also fully transactional with 14 | * full ACID semantics, and when the memory map is read-only, the 15 | * database integrity cannot be corrupted by stray pointer writes from 16 | * application code. 17 | * 18 | * The library is fully thread-aware and supports concurrent read/write 19 | * access from multiple processes and threads. Data pages use a copy-on- 20 | * write strategy so no active data pages are ever overwritten, which 21 | * also provides resistance to corruption and eliminates the need of any 22 | * special recovery procedures after a system crash. Writes are fully 23 | * serialized; only one write transaction may be active at a time, which 24 | * guarantees that writers can never deadlock. The database structure is 25 | * multi-versioned so readers run with no locks; writers cannot block 26 | * readers, and readers don't block writers. 27 | * 28 | * Unlike other well-known database mechanisms which use either write-ahead 29 | * transaction logs or append-only data writes, LMDB requires no maintenance 30 | * during operation. Both write-ahead loggers and append-only databases 31 | * require periodic checkpointing and/or compaction of their log or database 32 | * files otherwise they grow without bound. LMDB tracks free pages within 33 | * the database and re-uses them for new write operations, so the database 34 | * size does not grow without bound in normal use. 35 | * 36 | * The memory map can be used as a read-only or read-write map. It is 37 | * read-only by default as this provides total immunity to corruption. 38 | * Using read-write mode offers much higher write performance, but adds 39 | * the possibility for stray application writes thru pointers to silently 40 | * corrupt the database. Of course if your application code is known to 41 | * be bug-free (...) then this is not an issue. 42 | * 43 | * If this is your first time using a transactional embedded key/value 44 | * store, you may find the \ref starting page to be helpful. 45 | * 46 | * @section caveats_sec Caveats 47 | * Troubleshooting the lock file, plus semaphores on BSD systems: 48 | * 49 | * - A broken lockfile can cause sync issues. 50 | * Stale reader transactions left behind by an aborted program 51 | * cause further writes to grow the database quickly, and 52 | * stale locks can block further operation. 53 | * 54 | * Fix: Check for stale readers periodically, using the 55 | * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. 56 | * Stale writers will be cleared automatically on most systems: 57 | * - Windows - automatic 58 | * - BSD, systems using SysV semaphores - automatic 59 | * - Linux, systems using POSIX mutexes with Robust option - automatic 60 | * Otherwise just make all programs using the database close it; 61 | * the lockfile is always reset on first open of the environment. 62 | * 63 | * - On BSD systems or others configured with MDB_USE_SYSV_SEM or 64 | * MDB_USE_POSIX_SEM, 65 | * startup can fail due to semaphores owned by another userid. 66 | * 67 | * Fix: Open and close the database as the user which owns the 68 | * semaphores (likely last user) or as root, while no other 69 | * process is using the database. 70 | * 71 | * Restrictions/caveats (in addition to those listed for some functions): 72 | * 73 | * - Only the database owner should normally use the database on 74 | * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. 75 | * Multiple users can cause startup to fail later, as noted above. 76 | * 77 | * - There is normally no pure read-only mode, since readers need write 78 | * access to locks and lock file. Exceptions: On read-only filesystems 79 | * or with the #MDB_NOLOCK flag described under #mdb_env_open(). 80 | * 81 | * - An LMDB configuration will often reserve considerable \b unused 82 | * memory address space and maybe file size for future growth. 83 | * This does not use actual memory or disk space, but users may need 84 | * to understand the difference so they won't be scared off. 85 | * 86 | * - By default, in versions before 0.9.10, unused portions of the data 87 | * file might receive garbage data from memory freed by other code. 88 | * (This does not happen when using the #MDB_WRITEMAP flag.) As of 89 | * 0.9.10 the default behavior is to initialize such memory before 90 | * writing to the data file. Since there may be a slight performance 91 | * cost due to this initialization, applications may disable it using 92 | * the #MDB_NOMEMINIT flag. Applications handling sensitive data 93 | * which must not be written should not use this flag. This flag is 94 | * irrelevant when using #MDB_WRITEMAP. 95 | * 96 | * - A thread can only use one transaction at a time, plus any child 97 | * transactions. Each transaction belongs to one thread. See below. 98 | * The #MDB_NOTLS flag changes this for read-only transactions. 99 | * 100 | * - Use an MDB_env* in the process which opened it, without fork()ing. 101 | * 102 | * - Do not have open an LMDB database twice in the same process at 103 | * the same time. Not even from a plain open() call - close()ing it 104 | * breaks flock() advisory locking. 105 | * 106 | * - Avoid long-lived transactions. Read transactions prevent 107 | * reuse of pages freed by newer write transactions, thus the 108 | * database can grow quickly. Write transactions prevent 109 | * other write transactions, since writes are serialized. 110 | * 111 | * - Avoid suspending a process with active transactions. These 112 | * would then be "long-lived" as above. Also read transactions 113 | * suspended when writers commit could sometimes see wrong data. 114 | * 115 | * ...when several processes can use a database concurrently: 116 | * 117 | * - Avoid aborting a process with an active transaction. 118 | * The transaction becomes "long-lived" as above until a check 119 | * for stale readers is performed or the lockfile is reset, 120 | * since the process may not remove it from the lockfile. 121 | * 122 | * This does not apply to write transactions if the system clears 123 | * stale writers, see above. 124 | * 125 | * - If you do that anyway, do a periodic check for stale readers. Or 126 | * close the environment once in a while, so the lockfile can get reset. 127 | * 128 | * - Do not use LMDB databases on remote filesystems, even between 129 | * processes on the same host. This breaks flock() on some OSes, 130 | * possibly memory map sync, and certainly sync between programs 131 | * on different hosts. 132 | * 133 | * - Opening a database can fail if another process is opening or 134 | * closing it at exactly the same time. 135 | * 136 | * @author Howard Chu, Symas Corporation. 137 | * 138 | * @copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved. 139 | * 140 | * Redistribution and use in source and binary forms, with or without 141 | * modification, are permitted only as authorized by the OpenLDAP 142 | * Public License. 143 | * 144 | * A copy of this license is available in the file LICENSE in the 145 | * top-level directory of the distribution or, alternatively, at 146 | * . 147 | * 148 | * @par Derived From: 149 | * This code is derived from btree.c written by Martin Hedenfalk. 150 | * 151 | * Copyright (c) 2009, 2010 Martin Hedenfalk 152 | * 153 | * Permission to use, copy, modify, and distribute this software for any 154 | * purpose with or without fee is hereby granted, provided that the above 155 | * copyright notice and this permission notice appear in all copies. 156 | * 157 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 158 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 159 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 160 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 161 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 162 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 163 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 164 | */ 165 | #ifndef _LMDB_H_ 166 | #define _LMDB_H_ 167 | 168 | #include 169 | #include 170 | 171 | #ifdef __cplusplus 172 | extern "C" { 173 | #endif 174 | 175 | /** Unix permissions for creating files, or dummy definition for Windows */ 176 | #ifdef _MSC_VER 177 | typedef int mdb_mode_t; 178 | #else 179 | typedef mode_t mdb_mode_t; 180 | #endif 181 | 182 | #ifdef MDB_VL32 183 | typedef uint64_t mdb_size_t; 184 | #define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ 185 | #else 186 | typedef size_t mdb_size_t; 187 | #endif 188 | 189 | /** An abstraction for a file handle. 190 | * On POSIX systems file handles are small integers. On Windows 191 | * they're opaque pointers. 192 | */ 193 | #ifdef _WIN32 194 | typedef void *mdb_filehandle_t; 195 | #else 196 | typedef int mdb_filehandle_t; 197 | #endif 198 | 199 | /** @defgroup mdb LMDB API 200 | * @{ 201 | * @brief OpenLDAP Lightning Memory-Mapped Database Manager 202 | */ 203 | /** @defgroup Version Version Macros 204 | * @{ 205 | */ 206 | /** Library major version */ 207 | #define MDB_VERSION_MAJOR 0 208 | /** Library minor version */ 209 | #define MDB_VERSION_MINOR 9 210 | /** Library patch version */ 211 | #define MDB_VERSION_PATCH 70 212 | 213 | /** Combine args a,b,c into a single integer for easy version comparisons */ 214 | #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) 215 | 216 | /** The full library version as a single integer */ 217 | #define MDB_VERSION_FULL \ 218 | MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) 219 | 220 | /** The release date of this library version */ 221 | #define MDB_VERSION_DATE "December 19, 2015" 222 | 223 | /** A stringifier for the version info */ 224 | #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" 225 | 226 | /** A helper for the stringifier macro */ 227 | #define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) 228 | 229 | /** The full library version as a C string */ 230 | #define MDB_VERSION_STRING \ 231 | MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) 232 | /** @} */ 233 | 234 | /** @brief Opaque structure for a database environment. 235 | * 236 | * A DB environment supports multiple databases, all residing in the same 237 | * shared-memory map. 238 | */ 239 | typedef struct MDB_env MDB_env; 240 | 241 | /** @brief Opaque structure for a transaction handle. 242 | * 243 | * All database operations require a transaction handle. Transactions may be 244 | * read-only or read-write. 245 | */ 246 | typedef struct MDB_txn MDB_txn; 247 | 248 | /** @brief A handle for an individual database in the DB environment. */ 249 | typedef unsigned int MDB_dbi; 250 | 251 | /** @brief Opaque structure for navigating through a database */ 252 | typedef struct MDB_cursor MDB_cursor; 253 | 254 | /** @brief Generic structure used for passing keys and data in and out 255 | * of the database. 256 | * 257 | * Values returned from the database are valid only until a subsequent 258 | * update operation, or the end of the transaction. Do not modify or 259 | * free them, they commonly point into the database itself. 260 | * 261 | * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. 262 | * The same applies to data sizes in databases with the #MDB_DUPSORT flag. 263 | * Other data items can in theory be from 0 to 0xffffffff bytes long. 264 | */ 265 | typedef struct MDB_val { 266 | size_t mv_size; /**< size of the data item */ 267 | void *mv_data; /**< address of the data item */ 268 | } MDB_val; 269 | 270 | /** @brief A callback function used to compare two keys in a database */ 271 | typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); 272 | 273 | /** @brief A callback function used to relocate a position-dependent data item 274 | * in a fixed-address database. 275 | * 276 | * The \b newptr gives the item's desired address in 277 | * the memory map, and \b oldptr gives its previous address. The item's actual 278 | * data resides at the address in \b item. This callback is expected to walk 279 | * through the fields of the record in \b item and modify any 280 | * values based at the \b oldptr address to be relative to the \b newptr address. 281 | * @param[in,out] item The item that is to be relocated. 282 | * @param[in] oldptr The previous address. 283 | * @param[in] newptr The new address to relocate to. 284 | * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). 285 | * @todo This feature is currently unimplemented. 286 | */ 287 | typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); 288 | 289 | /** @defgroup mdb_env Environment Flags 290 | * @{ 291 | */ 292 | /** mmap at a fixed address (experimental) */ 293 | #define MDB_FIXEDMAP 0x01 294 | /** no environment directory */ 295 | #define MDB_NOSUBDIR 0x4000 296 | /** don't fsync after commit */ 297 | #define MDB_NOSYNC 0x10000 298 | /** read only */ 299 | #define MDB_RDONLY 0x20000 300 | /** don't fsync metapage after commit */ 301 | #define MDB_NOMETASYNC 0x40000 302 | /** use writable mmap */ 303 | #define MDB_WRITEMAP 0x80000 304 | /** use asynchronous msync when #MDB_WRITEMAP is used */ 305 | #define MDB_MAPASYNC 0x100000 306 | /** tie reader locktable slots to #MDB_txn objects instead of to threads */ 307 | #define MDB_NOTLS 0x200000 308 | /** don't do any locking, caller must manage their own locks */ 309 | #define MDB_NOLOCK 0x400000 310 | /** don't do readahead (no effect on Windows) */ 311 | #define MDB_NORDAHEAD 0x800000 312 | /** don't initialize malloc'd memory before writing to datafile */ 313 | #define MDB_NOMEMINIT 0x1000000 314 | /** @} */ 315 | 316 | /** @defgroup mdb_dbi_open Database Flags 317 | * @{ 318 | */ 319 | /** use reverse string keys */ 320 | #define MDB_REVERSEKEY 0x02 321 | /** use sorted duplicates */ 322 | #define MDB_DUPSORT 0x04 323 | /** numeric keys in native byte order: either unsigned int or size_t. 324 | * The keys must all be of the same size. */ 325 | #define MDB_INTEGERKEY 0x08 326 | /** with #MDB_DUPSORT, sorted dup items have fixed size */ 327 | #define MDB_DUPFIXED 0x10 328 | /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */ 329 | #define MDB_INTEGERDUP 0x20 330 | /** with #MDB_DUPSORT, use reverse string dups */ 331 | #define MDB_REVERSEDUP 0x40 332 | /** create DB if not already existing */ 333 | #define MDB_CREATE 0x40000 334 | /** @} */ 335 | 336 | /** @defgroup mdb_put Write Flags 337 | * @{ 338 | */ 339 | /** For put: Don't write if the key already exists. */ 340 | #define MDB_NOOVERWRITE 0x10 341 | /** Only for #MDB_DUPSORT
342 | * For put: don't write if the key and data pair already exist.
343 | * For mdb_cursor_del: remove all duplicate data items. 344 | */ 345 | #define MDB_NODUPDATA 0x20 346 | /** For mdb_cursor_put: overwrite the current key/data pair */ 347 | #define MDB_CURRENT 0x40 348 | /** For put: Just reserve space for data, don't copy it. Return a 349 | * pointer to the reserved space. 350 | */ 351 | #define MDB_RESERVE 0x10000 352 | /** Data is being appended, don't split full pages. */ 353 | #define MDB_APPEND 0x20000 354 | /** Duplicate data is being appended, don't split full pages. */ 355 | #define MDB_APPENDDUP 0x40000 356 | /** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ 357 | #define MDB_MULTIPLE 0x80000 358 | /* @} */ 359 | 360 | /** @defgroup mdb_copy Copy Flags 361 | * @{ 362 | */ 363 | /** Compacting copy: Omit free space from copy, and renumber all 364 | * pages sequentially. 365 | */ 366 | #define MDB_CP_COMPACT 0x01 367 | /* @} */ 368 | 369 | /** @brief Cursor Get operations. 370 | * 371 | * This is the set of all operations for retrieving data 372 | * using a cursor. 373 | */ 374 | typedef enum MDB_cursor_op { 375 | MDB_FIRST, /**< Position at first key/data item */ 376 | MDB_FIRST_DUP, /**< Position at first data item of current key. 377 | Only for #MDB_DUPSORT */ 378 | MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ 379 | MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ 380 | MDB_GET_CURRENT, /**< Return key/data at current cursor position */ 381 | MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items 382 | from current cursor position. Move cursor to prepare 383 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ 384 | MDB_LAST, /**< Position at last key/data item */ 385 | MDB_LAST_DUP, /**< Position at last data item of current key. 386 | Only for #MDB_DUPSORT */ 387 | MDB_NEXT, /**< Position at next data item */ 388 | MDB_NEXT_DUP, /**< Position at next data item of current key. 389 | Only for #MDB_DUPSORT */ 390 | MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items 391 | from next cursor position. Move cursor to prepare 392 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ 393 | MDB_NEXT_NODUP, /**< Position at first data item of next key */ 394 | MDB_PREV, /**< Position at previous data item */ 395 | MDB_PREV_DUP, /**< Position at previous data item of current key. 396 | Only for #MDB_DUPSORT */ 397 | MDB_PREV_NODUP, /**< Position at last data item of previous key */ 398 | MDB_SET, /**< Position at specified key */ 399 | MDB_SET_KEY, /**< Position at specified key, return key + data */ 400 | MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ 401 | MDB_PREV_MULTIPLE /**< Position at previous page and return key and up to 402 | a page of duplicate data items. Only for #MDB_DUPFIXED */ 403 | } MDB_cursor_op; 404 | 405 | /** @defgroup errors Return Codes 406 | * 407 | * BerkeleyDB uses -30800 to -30999, we'll go under them 408 | * @{ 409 | */ 410 | /** Successful result */ 411 | #define MDB_SUCCESS 0 412 | /** key/data pair already exists */ 413 | #define MDB_KEYEXIST (-30799) 414 | /** key/data pair not found (EOF) */ 415 | #define MDB_NOTFOUND (-30798) 416 | /** Requested page not found - this usually indicates corruption */ 417 | #define MDB_PAGE_NOTFOUND (-30797) 418 | /** Located page was wrong type */ 419 | #define MDB_CORRUPTED (-30796) 420 | /** Update of meta page failed or environment had fatal error */ 421 | #define MDB_PANIC (-30795) 422 | /** Environment version mismatch */ 423 | #define MDB_VERSION_MISMATCH (-30794) 424 | /** File is not a valid LMDB file */ 425 | #define MDB_INVALID (-30793) 426 | /** Environment mapsize reached */ 427 | #define MDB_MAP_FULL (-30792) 428 | /** Environment maxdbs reached */ 429 | #define MDB_DBS_FULL (-30791) 430 | /** Environment maxreaders reached */ 431 | #define MDB_READERS_FULL (-30790) 432 | /** Too many TLS keys in use - Windows only */ 433 | #define MDB_TLS_FULL (-30789) 434 | /** Txn has too many dirty pages */ 435 | #define MDB_TXN_FULL (-30788) 436 | /** Cursor stack too deep - internal error */ 437 | #define MDB_CURSOR_FULL (-30787) 438 | /** Page has not enough space - internal error */ 439 | #define MDB_PAGE_FULL (-30786) 440 | /** Database contents grew beyond environment mapsize */ 441 | #define MDB_MAP_RESIZED (-30785) 442 | /** Operation and DB incompatible, or DB type changed. This can mean: 443 | *
    444 | *
  • The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database. 445 | *
  • Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY. 446 | *
  • Accessing a data record as a database, or vice versa. 447 | *
  • The database was dropped and recreated with different flags. 448 | *
449 | */ 450 | #define MDB_INCOMPATIBLE (-30784) 451 | /** Invalid reuse of reader locktable slot */ 452 | #define MDB_BAD_RSLOT (-30783) 453 | /** Transaction must abort, has a child, or is invalid */ 454 | #define MDB_BAD_TXN (-30782) 455 | /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ 456 | #define MDB_BAD_VALSIZE (-30781) 457 | /** The specified DBI was changed unexpectedly */ 458 | #define MDB_BAD_DBI (-30780) 459 | /** The last defined error code */ 460 | #define MDB_LAST_ERRCODE MDB_BAD_DBI 461 | /** @} */ 462 | 463 | /** @brief Statistics for a database in the environment */ 464 | typedef struct MDB_stat { 465 | unsigned int ms_psize; /**< Size of a database page. 466 | This is currently the same for all databases. */ 467 | unsigned int ms_depth; /**< Depth (height) of the B-tree */ 468 | mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ 469 | mdb_size_t ms_leaf_pages; /**< Number of leaf pages */ 470 | mdb_size_t ms_overflow_pages; /**< Number of overflow pages */ 471 | mdb_size_t ms_entries; /**< Number of data items */ 472 | } MDB_stat; 473 | 474 | /** @brief Information about the environment */ 475 | typedef struct MDB_envinfo { 476 | void *me_mapaddr; /**< Address of map, if fixed */ 477 | mdb_size_t me_mapsize; /**< Size of the data memory map */ 478 | mdb_size_t me_last_pgno; /**< ID of the last used page */ 479 | mdb_size_t me_last_txnid; /**< ID of the last committed transaction */ 480 | unsigned int me_maxreaders; /**< max reader slots in the environment */ 481 | unsigned int me_numreaders; /**< max reader slots used in the environment */ 482 | } MDB_envinfo; 483 | 484 | /** @brief Return the LMDB library version information. 485 | * 486 | * @param[out] major if non-NULL, the library major version number is copied here 487 | * @param[out] minor if non-NULL, the library minor version number is copied here 488 | * @param[out] patch if non-NULL, the library patch version number is copied here 489 | * @retval "version string" The library version as a string 490 | */ 491 | char *mdb_version(int *major, int *minor, int *patch); 492 | 493 | /** @brief Return a string describing a given error code. 494 | * 495 | * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) 496 | * function. If the error code is greater than or equal to 0, then the string 497 | * returned by the system function strerror(3) is returned. If the error code 498 | * is less than 0, an error string corresponding to the LMDB library error is 499 | * returned. See @ref errors for a list of LMDB-specific error codes. 500 | * @param[in] err The error code 501 | * @retval "error message" The description of the error 502 | */ 503 | char *mdb_strerror(int err); 504 | 505 | /** @brief Create an LMDB environment handle. 506 | * 507 | * This function allocates memory for a #MDB_env structure. To release 508 | * the allocated memory and discard the handle, call #mdb_env_close(). 509 | * Before the handle may be used, it must be opened using #mdb_env_open(). 510 | * Various other options may also need to be set before opening the handle, 511 | * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), 512 | * depending on usage requirements. 513 | * @param[out] env The address where the new handle will be stored 514 | * @return A non-zero error value on failure and 0 on success. 515 | */ 516 | int mdb_env_create(MDB_env **env); 517 | 518 | /** @brief Open an environment handle. 519 | * 520 | * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. 521 | * @param[in] env An environment handle returned by #mdb_env_create() 522 | * @param[in] path The directory in which the database files reside. This 523 | * directory must already exist and be writable. 524 | * @param[in] flags Special options for this environment. This parameter 525 | * must be set to 0 or by bitwise OR'ing together one or more of the 526 | * values described here. 527 | * Flags set by mdb_env_set_flags() are also used. 528 | *
    529 | *
  • #MDB_FIXEDMAP 530 | * use a fixed address for the mmap region. This flag must be specified 531 | * when creating the environment, and is stored persistently in the environment. 532 | * If successful, the memory map will always reside at the same virtual address 533 | * and pointers used to reference data items in the database will be constant 534 | * across multiple invocations. This option may not always work, depending on 535 | * how the operating system has allocated memory to shared libraries and other uses. 536 | * The feature is highly experimental. 537 | *
  • #MDB_NOSUBDIR 538 | * By default, LMDB creates its environment in a directory whose 539 | * pathname is given in \b path, and creates its data and lock files 540 | * under that directory. With this option, \b path is used as-is for 541 | * the database main data file. The database lock file is the \b path 542 | * with "-lock" appended. 543 | *
  • #MDB_RDONLY 544 | * Open the environment in read-only mode. No write operations will be 545 | * allowed. LMDB will still modify the lock file - except on read-only 546 | * filesystems, where LMDB does not use locks. 547 | *
  • #MDB_WRITEMAP 548 | * Use a writeable memory map unless MDB_RDONLY is set. This uses 549 | * fewer mallocs but loses protection from application bugs 550 | * like wild pointer writes and other bad updates into the database. 551 | * This may be slightly faster for DBs that fit entirely in RAM, but 552 | * is slower for DBs larger than RAM. 553 | * Incompatible with nested transactions. 554 | * Do not mix processes with and without MDB_WRITEMAP on the same 555 | * environment. This can defeat durability (#mdb_env_sync etc). 556 | *
  • #MDB_NOMETASYNC 557 | * Flush system buffers to disk only once per transaction, omit the 558 | * metadata flush. Defer that until the system flushes files to disk, 559 | * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization 560 | * maintains database integrity, but a system crash may undo the last 561 | * committed transaction. I.e. it preserves the ACI (atomicity, 562 | * consistency, isolation) but not D (durability) database property. 563 | * This flag may be changed at any time using #mdb_env_set_flags(). 564 | *
  • #MDB_NOSYNC 565 | * Don't flush system buffers to disk when committing a transaction. 566 | * This optimization means a system crash can corrupt the database or 567 | * lose the last transactions if buffers are not yet flushed to disk. 568 | * The risk is governed by how often the system flushes dirty buffers 569 | * to disk and how often #mdb_env_sync() is called. However, if the 570 | * filesystem preserves write order and the #MDB_WRITEMAP flag is not 571 | * used, transactions exhibit ACI (atomicity, consistency, isolation) 572 | * properties and only lose D (durability). I.e. database integrity 573 | * is maintained, but a system crash may undo the final transactions. 574 | * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no 575 | * hint for when to write transactions to disk, unless #mdb_env_sync() 576 | * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. 577 | * This flag may be changed at any time using #mdb_env_set_flags(). 578 | *
  • #MDB_MAPASYNC 579 | * When using #MDB_WRITEMAP, use asynchronous flushes to disk. 580 | * As with #MDB_NOSYNC, a system crash can then corrupt the 581 | * database or lose the last transactions. Calling #mdb_env_sync() 582 | * ensures on-disk database integrity until next commit. 583 | * This flag may be changed at any time using #mdb_env_set_flags(). 584 | *
  • #MDB_NOTLS 585 | * Don't use Thread-Local Storage. Tie reader locktable slots to 586 | * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps 587 | * the slot reseved for the #MDB_txn object. A thread may use parallel 588 | * read-only transactions. A read-only transaction may span threads if 589 | * the user synchronizes its use. Applications that multiplex many 590 | * user threads over individual OS threads need this option. Such an 591 | * application must also serialize the write transactions in an OS 592 | * thread, since LMDB's write locking is unaware of the user threads. 593 | *
  • #MDB_NOLOCK 594 | * Don't do any locking. If concurrent access is anticipated, the 595 | * caller must manage all concurrency itself. For proper operation 596 | * the caller must enforce single-writer semantics, and must ensure 597 | * that no readers are using old transactions while a writer is 598 | * active. The simplest approach is to use an exclusive lock so that 599 | * no readers may be active at all when a writer begins. 600 | *
  • #MDB_NORDAHEAD 601 | * Turn off readahead. Most operating systems perform readahead on 602 | * read requests by default. This option turns it off if the OS 603 | * supports it. Turning it off may help random read performance 604 | * when the DB is larger than RAM and system RAM is full. 605 | * The option is not implemented on Windows. 606 | *
  • #MDB_NOMEMINIT 607 | * Don't initialize malloc'd memory before writing to unused spaces 608 | * in the data file. By default, memory for pages written to the data 609 | * file is obtained using malloc. While these pages may be reused in 610 | * subsequent transactions, freshly malloc'd pages will be initialized 611 | * to zeroes before use. This avoids persisting leftover data from other 612 | * code (that used the heap and subsequently freed the memory) into the 613 | * data file. Note that many other system libraries may allocate 614 | * and free memory from the heap for arbitrary uses. E.g., stdio may 615 | * use the heap for file I/O buffers. This initialization step has a 616 | * modest performance cost so some applications may want to disable 617 | * it using this flag. This option can be a problem for applications 618 | * which handle sensitive data like passwords, and it makes memory 619 | * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, 620 | * which writes directly to the mmap instead of using malloc for pages. The 621 | * initialization is also skipped if #MDB_RESERVE is used; the 622 | * caller is expected to overwrite all of the memory that was 623 | * reserved in that case. 624 | * This flag may be changed at any time using #mdb_env_set_flags(). 625 | *
626 | * @param[in] mode The UNIX permissions to set on created files and semaphores. 627 | * This parameter is ignored on Windows. 628 | * @return A non-zero error value on failure and 0 on success. Some possible 629 | * errors are: 630 | *
    631 | *
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the 632 | * version that created the database environment. 633 | *
  • #MDB_INVALID - the environment file headers are corrupted. 634 | *
  • ENOENT - the directory specified by the path parameter doesn't exist. 635 | *
  • EACCES - the user didn't have permission to access the environment files. 636 | *
  • EAGAIN - the environment was locked by another process. 637 | *
638 | */ 639 | int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); 640 | 641 | /** @brief Copy an LMDB environment to the specified path. 642 | * 643 | * This function may be used to make a backup of an existing environment. 644 | * No lockfile is created, since it gets recreated at need. 645 | * @note This call can trigger significant file size growth if run in 646 | * parallel with write transactions, because it employs a read-only 647 | * transaction. See long-lived transactions under @ref caveats_sec. 648 | * @param[in] env An environment handle returned by #mdb_env_create(). It 649 | * must have already been opened successfully. 650 | * @param[in] path The directory in which the copy will reside. This 651 | * directory must already exist and be writable but must otherwise be 652 | * empty. 653 | * @return A non-zero error value on failure and 0 on success. 654 | */ 655 | int mdb_env_copy(MDB_env *env, const char *path); 656 | 657 | /** @brief Copy an LMDB environment to the specified file descriptor. 658 | * 659 | * This function may be used to make a backup of an existing environment. 660 | * No lockfile is created, since it gets recreated at need. 661 | * @note This call can trigger significant file size growth if run in 662 | * parallel with write transactions, because it employs a read-only 663 | * transaction. See long-lived transactions under @ref caveats_sec. 664 | * @param[in] env An environment handle returned by #mdb_env_create(). It 665 | * must have already been opened successfully. 666 | * @param[in] fd The filedescriptor to write the copy to. It must 667 | * have already been opened for Write access. 668 | * @return A non-zero error value on failure and 0 on success. 669 | */ 670 | int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); 671 | 672 | /** @brief Copy an LMDB environment to the specified path, with options. 673 | * 674 | * This function may be used to make a backup of an existing environment. 675 | * No lockfile is created, since it gets recreated at need. 676 | * @note This call can trigger significant file size growth if run in 677 | * parallel with write transactions, because it employs a read-only 678 | * transaction. See long-lived transactions under @ref caveats_sec. 679 | * @param[in] env An environment handle returned by #mdb_env_create(). It 680 | * must have already been opened successfully. 681 | * @param[in] path The directory in which the copy will reside. This 682 | * directory must already exist and be writable but must otherwise be 683 | * empty. 684 | * @param[in] flags Special options for this operation. This parameter 685 | * must be set to 0 or by bitwise OR'ing together one or more of the 686 | * values described here. 687 | *
    688 | *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free 689 | * pages and sequentially renumber all pages in output. This option 690 | * consumes more CPU and runs more slowly than the default. 691 | *
692 | * @return A non-zero error value on failure and 0 on success. 693 | */ 694 | int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); 695 | 696 | /** @brief Copy an LMDB environment to the specified file descriptor, 697 | * with options. 698 | * 699 | * This function may be used to make a backup of an existing environment. 700 | * No lockfile is created, since it gets recreated at need. See 701 | * #mdb_env_copy2() for further details. 702 | * @note This call can trigger significant file size growth if run in 703 | * parallel with write transactions, because it employs a read-only 704 | * transaction. See long-lived transactions under @ref caveats_sec. 705 | * @param[in] env An environment handle returned by #mdb_env_create(). It 706 | * must have already been opened successfully. 707 | * @param[in] fd The filedescriptor to write the copy to. It must 708 | * have already been opened for Write access. 709 | * @param[in] flags Special options for this operation. 710 | * See #mdb_env_copy2() for options. 711 | * @return A non-zero error value on failure and 0 on success. 712 | */ 713 | int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); 714 | 715 | /** @brief Return statistics about the LMDB environment. 716 | * 717 | * @param[in] env An environment handle returned by #mdb_env_create() 718 | * @param[out] stat The address of an #MDB_stat structure 719 | * where the statistics will be copied 720 | */ 721 | int mdb_env_stat(MDB_env *env, MDB_stat *stat); 722 | 723 | /** @brief Return information about the LMDB environment. 724 | * 725 | * @param[in] env An environment handle returned by #mdb_env_create() 726 | * @param[out] stat The address of an #MDB_envinfo structure 727 | * where the information will be copied 728 | */ 729 | int mdb_env_info(MDB_env *env, MDB_envinfo *stat); 730 | 731 | /** @brief Flush the data buffers to disk. 732 | * 733 | * Data is always written to disk when #mdb_txn_commit() is called, 734 | * but the operating system may keep it buffered. LMDB always flushes 735 | * the OS buffers upon commit as well, unless the environment was 736 | * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is 737 | * not valid if the environment was opened with #MDB_RDONLY. 738 | * @param[in] env An environment handle returned by #mdb_env_create() 739 | * @param[in] force If non-zero, force a synchronous flush. Otherwise 740 | * if the environment has the #MDB_NOSYNC flag set the flushes 741 | * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. 742 | * @return A non-zero error value on failure and 0 on success. Some possible 743 | * errors are: 744 | *
    745 | *
  • EACCES - the environment is read-only. 746 | *
  • EINVAL - an invalid parameter was specified. 747 | *
  • EIO - an error occurred during synchronization. 748 | *
749 | */ 750 | int mdb_env_sync(MDB_env *env, int force); 751 | 752 | /** @brief Close the environment and release the memory map. 753 | * 754 | * Only a single thread may call this function. All transactions, databases, 755 | * and cursors must already be closed before calling this function. Attempts to 756 | * use any such handles after calling this function will cause a SIGSEGV. 757 | * The environment handle will be freed and must not be used again after this call. 758 | * @param[in] env An environment handle returned by #mdb_env_create() 759 | */ 760 | void mdb_env_close(MDB_env *env); 761 | 762 | /** @brief Set environment flags. 763 | * 764 | * This may be used to set some flags in addition to those from 765 | * #mdb_env_open(), or to unset these flags. If several threads 766 | * change the flags at the same time, the result is undefined. 767 | * @param[in] env An environment handle returned by #mdb_env_create() 768 | * @param[in] flags The flags to change, bitwise OR'ed together 769 | * @param[in] onoff A non-zero value sets the flags, zero clears them. 770 | * @return A non-zero error value on failure and 0 on success. Some possible 771 | * errors are: 772 | *
    773 | *
  • EINVAL - an invalid parameter was specified. 774 | *
775 | */ 776 | int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); 777 | 778 | /** @brief Get environment flags. 779 | * 780 | * @param[in] env An environment handle returned by #mdb_env_create() 781 | * @param[out] flags The address of an integer to store the flags 782 | * @return A non-zero error value on failure and 0 on success. Some possible 783 | * errors are: 784 | *
    785 | *
  • EINVAL - an invalid parameter was specified. 786 | *
787 | */ 788 | int mdb_env_get_flags(MDB_env *env, unsigned int *flags); 789 | 790 | /** @brief Return the path that was used in #mdb_env_open(). 791 | * 792 | * @param[in] env An environment handle returned by #mdb_env_create() 793 | * @param[out] path Address of a string pointer to contain the path. This 794 | * is the actual string in the environment, not a copy. It should not be 795 | * altered in any way. 796 | * @return A non-zero error value on failure and 0 on success. Some possible 797 | * errors are: 798 | *
    799 | *
  • EINVAL - an invalid parameter was specified. 800 | *
801 | */ 802 | int mdb_env_get_path(MDB_env *env, const char **path); 803 | 804 | /** @brief Return the filedescriptor for the given environment. 805 | * 806 | * @param[in] env An environment handle returned by #mdb_env_create() 807 | * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. 808 | * @return A non-zero error value on failure and 0 on success. Some possible 809 | * errors are: 810 | *
    811 | *
  • EINVAL - an invalid parameter was specified. 812 | *
813 | */ 814 | int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); 815 | 816 | /** @brief Set the size of the memory map to use for this environment. 817 | * 818 | * The size should be a multiple of the OS page size. The default is 819 | * 10485760 bytes. The size of the memory map is also the maximum size 820 | * of the database. The value should be chosen as large as possible, 821 | * to accommodate future growth of the database. 822 | * This function should be called after #mdb_env_create() and before #mdb_env_open(). 823 | * It may be called at later times if no transactions are active in 824 | * this process. Note that the library does not check for this condition, 825 | * the caller must ensure it explicitly. 826 | * 827 | * The new size takes effect immediately for the current process but 828 | * will not be persisted to any others until a write transaction has been 829 | * committed by the current process. Also, only mapsize increases are 830 | * persisted into the environment. 831 | * 832 | * If the mapsize is increased by another process, and data has grown 833 | * beyond the range of the current mapsize, #mdb_txn_begin() will 834 | * return #MDB_MAP_RESIZED. This function may be called with a size 835 | * of zero to adopt the new size. 836 | * 837 | * Any attempt to set a size smaller than the space already consumed 838 | * by the environment will be silently changed to the current size of the used space. 839 | * @param[in] env An environment handle returned by #mdb_env_create() 840 | * @param[in] size The size in bytes 841 | * @return A non-zero error value on failure and 0 on success. Some possible 842 | * errors are: 843 | *
    844 | *
  • EINVAL - an invalid parameter was specified, or the environment has 845 | * an active write transaction. 846 | *
847 | */ 848 | int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size); 849 | 850 | /** @brief Set the maximum number of threads/reader slots for the environment. 851 | * 852 | * This defines the number of slots in the lock table that is used to track readers in the 853 | * the environment. The default is 126. 854 | * Starting a read-only transaction normally ties a lock table slot to the 855 | * current thread until the environment closes or the thread exits. If 856 | * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the 857 | * MDB_txn object until it or the #MDB_env object is destroyed. 858 | * This function may only be called after #mdb_env_create() and before #mdb_env_open(). 859 | * @param[in] env An environment handle returned by #mdb_env_create() 860 | * @param[in] readers The maximum number of reader lock table slots 861 | * @return A non-zero error value on failure and 0 on success. Some possible 862 | * errors are: 863 | *
    864 | *
  • EINVAL - an invalid parameter was specified, or the environment is already open. 865 | *
866 | */ 867 | int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); 868 | 869 | /** @brief Get the maximum number of threads/reader slots for the environment. 870 | * 871 | * @param[in] env An environment handle returned by #mdb_env_create() 872 | * @param[out] readers Address of an integer to store the number of readers 873 | * @return A non-zero error value on failure and 0 on success. Some possible 874 | * errors are: 875 | *
    876 | *
  • EINVAL - an invalid parameter was specified. 877 | *
878 | */ 879 | int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); 880 | 881 | /** @brief Set the maximum number of named databases for the environment. 882 | * 883 | * This function is only needed if multiple databases will be used in the 884 | * environment. Simpler applications that use the environment as a single 885 | * unnamed database can ignore this option. 886 | * This function may only be called after #mdb_env_create() and before #mdb_env_open(). 887 | * 888 | * Currently a moderate number of slots are cheap but a huge number gets 889 | * expensive: 7-120 words per transaction, and every #mdb_dbi_open() 890 | * does a linear search of the opened slots. 891 | * @param[in] env An environment handle returned by #mdb_env_create() 892 | * @param[in] dbs The maximum number of databases 893 | * @return A non-zero error value on failure and 0 on success. Some possible 894 | * errors are: 895 | *
    896 | *
  • EINVAL - an invalid parameter was specified, or the environment is already open. 897 | *
898 | */ 899 | int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); 900 | 901 | /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. 902 | * 903 | * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. 904 | * See @ref MDB_val. 905 | * @param[in] env An environment handle returned by #mdb_env_create() 906 | * @return The maximum size of a key we can write 907 | */ 908 | int mdb_env_get_maxkeysize(MDB_env *env); 909 | 910 | /** @brief Set application information associated with the #MDB_env. 911 | * 912 | * @param[in] env An environment handle returned by #mdb_env_create() 913 | * @param[in] ctx An arbitrary pointer for whatever the application needs. 914 | * @return A non-zero error value on failure and 0 on success. 915 | */ 916 | int mdb_env_set_userctx(MDB_env *env, void *ctx); 917 | 918 | /** @brief Get the application information associated with the #MDB_env. 919 | * 920 | * @param[in] env An environment handle returned by #mdb_env_create() 921 | * @return The pointer set by #mdb_env_set_userctx(). 922 | */ 923 | void *mdb_env_get_userctx(MDB_env *env); 924 | 925 | /** @brief A callback function for most LMDB assert() failures, 926 | * called before printing the message and aborting. 927 | * 928 | * @param[in] env An environment handle returned by #mdb_env_create(). 929 | * @param[in] msg The assertion message, not including newline. 930 | */ 931 | typedef void MDB_assert_func(MDB_env *env, const char *msg); 932 | 933 | /** Set or reset the assert() callback of the environment. 934 | * Disabled if liblmdb is buillt with NDEBUG. 935 | * @note This hack should become obsolete as lmdb's error handling matures. 936 | * @param[in] env An environment handle returned by #mdb_env_create(). 937 | * @param[in] func An #MDB_assert_func function, or 0. 938 | * @return A non-zero error value on failure and 0 on success. 939 | */ 940 | int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); 941 | 942 | /** @brief Create a transaction for use with the environment. 943 | * 944 | * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). 945 | * @note A transaction and its cursors must only be used by a single 946 | * thread, and a thread may only have a single transaction at a time. 947 | * If #MDB_NOTLS is in use, this does not apply to read-only transactions. 948 | * @note Cursors may not span transactions. 949 | * @param[in] env An environment handle returned by #mdb_env_create() 950 | * @param[in] parent If this parameter is non-NULL, the new transaction 951 | * will be a nested transaction, with the transaction indicated by \b parent 952 | * as its parent. Transactions may be nested to any level. A parent 953 | * transaction and its cursors may not issue any other operations than 954 | * mdb_txn_commit and mdb_txn_abort while it has active child transactions. 955 | * @param[in] flags Special options for this transaction. This parameter 956 | * must be set to 0 or by bitwise OR'ing together one or more of the 957 | * values described here. 958 | *
    959 | *
  • #MDB_RDONLY 960 | * This transaction will not perform any write operations. 961 | *
  • #MDB_NOSYNC 962 | * Don't flush system buffers to disk when committing this transaction. 963 | *
  • #MDB_NOMETASYNC 964 | * Flush system buffers but omit metadata flush when committing this transaction. 965 | *
966 | * @param[out] txn Address where the new #MDB_txn handle will be stored 967 | * @return A non-zero error value on failure and 0 on success. Some possible 968 | * errors are: 969 | *
    970 | *
  • #MDB_PANIC - a fatal error occurred earlier and the environment 971 | * must be shut down. 972 | *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's 973 | * mapsize and this environment's map must be resized as well. 974 | * See #mdb_env_set_mapsize(). 975 | *
  • #MDB_READERS_FULL - a read-only transaction was requested and 976 | * the reader lock table is full. See #mdb_env_set_maxreaders(). 977 | *
  • ENOMEM - out of memory. 978 | *
979 | */ 980 | int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); 981 | 982 | /** @brief Returns the transaction's #MDB_env 983 | * 984 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 985 | */ 986 | MDB_env *mdb_txn_env(MDB_txn *txn); 987 | 988 | /** @brief Return the transaction's ID. 989 | * 990 | * This returns the identifier associated with this transaction. For a 991 | * read-only transaction, this corresponds to the snapshot being read; 992 | * concurrent readers will frequently have the same transaction ID. 993 | * 994 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 995 | * @return A transaction ID, valid if input is an active transaction. 996 | */ 997 | mdb_size_t mdb_txn_id(MDB_txn *txn); 998 | 999 | /** @brief Commit all the operations of a transaction into the database. 1000 | * 1001 | * The transaction handle is freed. It and its cursors must not be used 1002 | * again after this call, except with #mdb_cursor_renew(). 1003 | * @note Earlier documentation incorrectly said all cursors would be freed. 1004 | * Only write-transactions free cursors. 1005 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1006 | * @return A non-zero error value on failure and 0 on success. Some possible 1007 | * errors are: 1008 | *
    1009 | *
  • EINVAL - an invalid parameter was specified. 1010 | *
  • ENOSPC - no more disk space. 1011 | *
  • EIO - a low-level I/O error occurred while writing. 1012 | *
  • ENOMEM - out of memory. 1013 | *
1014 | */ 1015 | int mdb_txn_commit(MDB_txn *txn); 1016 | 1017 | /** @brief Abandon all the operations of the transaction instead of saving them. 1018 | * 1019 | * The transaction handle is freed. It and its cursors must not be used 1020 | * again after this call, except with #mdb_cursor_renew(). 1021 | * @note Earlier documentation incorrectly said all cursors would be freed. 1022 | * Only write-transactions free cursors. 1023 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1024 | */ 1025 | void mdb_txn_abort(MDB_txn *txn); 1026 | 1027 | /** @brief Reset a read-only transaction. 1028 | * 1029 | * Abort the transaction like #mdb_txn_abort(), but keep the transaction 1030 | * handle. #mdb_txn_renew() may reuse the handle. This saves allocation 1031 | * overhead if the process will start a new read-only transaction soon, 1032 | * and also locking overhead if #MDB_NOTLS is in use. The reader table 1033 | * lock is released, but the table slot stays tied to its thread or 1034 | * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free 1035 | * its lock table slot if MDB_NOTLS is in use. 1036 | * Cursors opened within the transaction must not be used 1037 | * again after this call, except with #mdb_cursor_renew(). 1038 | * Reader locks generally don't interfere with writers, but they keep old 1039 | * versions of database pages allocated. Thus they prevent the old pages 1040 | * from being reused when writers commit new data, and so under heavy load 1041 | * the database size may grow much more rapidly than otherwise. 1042 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1043 | */ 1044 | void mdb_txn_reset(MDB_txn *txn); 1045 | 1046 | /** @brief Renew a read-only transaction. 1047 | * 1048 | * This acquires a new reader lock for a transaction handle that had been 1049 | * released by #mdb_txn_reset(). It must be called before a reset transaction 1050 | * may be used again. 1051 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1052 | * @return A non-zero error value on failure and 0 on success. Some possible 1053 | * errors are: 1054 | *
    1055 | *
  • #MDB_PANIC - a fatal error occurred earlier and the environment 1056 | * must be shut down. 1057 | *
  • EINVAL - an invalid parameter was specified. 1058 | *
1059 | */ 1060 | int mdb_txn_renew(MDB_txn *txn); 1061 | 1062 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ 1063 | #define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) 1064 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ 1065 | #define mdb_close(env,dbi) mdb_dbi_close(env,dbi) 1066 | 1067 | /** @brief Open a database in the environment. 1068 | * 1069 | * A database handle denotes the name and parameters of a database, 1070 | * independently of whether such a database exists. 1071 | * The database handle may be discarded by calling #mdb_dbi_close(). 1072 | * The old database handle is returned if the database was already open. 1073 | * The handle may only be closed once. 1074 | * 1075 | * The database handle will be private to the current transaction until 1076 | * the transaction is successfully committed. If the transaction is 1077 | * aborted the handle will be closed automatically. 1078 | * After a successful commit the handle will reside in the shared 1079 | * environment, and may be used by other transactions. 1080 | * 1081 | * This function must not be called from multiple concurrent 1082 | * transactions in the same process. A transaction that uses 1083 | * this function must finish (either commit or abort) before 1084 | * any other transaction in the process may use this function. 1085 | * 1086 | * To use named databases (with name != NULL), #mdb_env_set_maxdbs() 1087 | * must be called before opening the environment. Database names are 1088 | * keys in the unnamed database, and may be read but not written. 1089 | * 1090 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1091 | * @param[in] name The name of the database to open. If only a single 1092 | * database is needed in the environment, this value may be NULL. 1093 | * @param[in] flags Special options for this database. This parameter 1094 | * must be set to 0 or by bitwise OR'ing together one or more of the 1095 | * values described here. 1096 | *
    1097 | *
  • #MDB_REVERSEKEY 1098 | * Keys are strings to be compared in reverse order, from the end 1099 | * of the strings to the beginning. By default, Keys are treated as strings and 1100 | * compared from beginning to end. 1101 | *
  • #MDB_DUPSORT 1102 | * Duplicate keys may be used in the database. (Or, from another perspective, 1103 | * keys may have multiple data items, stored in sorted order.) By default 1104 | * keys must be unique and may have only a single data item. 1105 | *
  • #MDB_INTEGERKEY 1106 | * Keys are binary integers in native byte order, either unsigned int 1107 | * or size_t, and will be sorted as such. 1108 | * The keys must all be of the same size. 1109 | *
  • #MDB_DUPFIXED 1110 | * This flag may only be used in combination with #MDB_DUPSORT. This option 1111 | * tells the library that the data items for this database are all the same 1112 | * size, which allows further optimizations in storage and retrieval. When 1113 | * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE 1114 | * cursor operations may be used to retrieve multiple items at once. 1115 | *
  • #MDB_INTEGERDUP 1116 | * This option specifies that duplicate data items are binary integers, 1117 | * similar to #MDB_INTEGERKEY keys. 1118 | *
  • #MDB_REVERSEDUP 1119 | * This option specifies that duplicate data items should be compared as 1120 | * strings in reverse order. 1121 | *
  • #MDB_CREATE 1122 | * Create the named database if it doesn't exist. This option is not 1123 | * allowed in a read-only transaction or a read-only environment. 1124 | *
1125 | * @param[out] dbi Address where the new #MDB_dbi handle will be stored 1126 | * @return A non-zero error value on failure and 0 on success. Some possible 1127 | * errors are: 1128 | *
    1129 | *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment 1130 | * and #MDB_CREATE was not specified. 1131 | *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). 1132 | *
1133 | */ 1134 | int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); 1135 | 1136 | /** @brief Retrieve statistics for a database. 1137 | * 1138 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1139 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1140 | * @param[out] stat The address of an #MDB_stat structure 1141 | * where the statistics will be copied 1142 | * @return A non-zero error value on failure and 0 on success. Some possible 1143 | * errors are: 1144 | *
    1145 | *
  • EINVAL - an invalid parameter was specified. 1146 | *
1147 | */ 1148 | int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); 1149 | 1150 | /** @brief Retrieve the DB flags for a database handle. 1151 | * 1152 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1153 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1154 | * @param[out] flags Address where the flags will be returned. 1155 | * @return A non-zero error value on failure and 0 on success. 1156 | */ 1157 | int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); 1158 | 1159 | /** @brief Close a database handle. Normally unnecessary. Use with care: 1160 | * 1161 | * This call is not mutex protected. Handles should only be closed by 1162 | * a single thread, and only if no other threads are going to reference 1163 | * the database handle or one of its cursors any further. Do not close 1164 | * a handle if an existing transaction has modified its database. 1165 | * Doing so can cause misbehavior from database corruption to errors 1166 | * like MDB_BAD_VALSIZE (since the DB name is gone). 1167 | * 1168 | * Closing a database handle is not necessary, but lets #mdb_dbi_open() 1169 | * reuse the handle value. Usually it's better to set a bigger 1170 | * #mdb_env_set_maxdbs(), unless that value would be large. 1171 | * 1172 | * @param[in] env An environment handle returned by #mdb_env_create() 1173 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1174 | */ 1175 | void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); 1176 | 1177 | /** @brief Empty or delete+close a database. 1178 | * 1179 | * See #mdb_dbi_close() for restrictions about closing the DB handle. 1180 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1181 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1182 | * @param[in] del 0 to empty the DB, 1 to delete it from the 1183 | * environment and close the DB handle. 1184 | * @return A non-zero error value on failure and 0 on success. 1185 | */ 1186 | int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); 1187 | 1188 | /** @brief Set a custom key comparison function for a database. 1189 | * 1190 | * The comparison function is called whenever it is necessary to compare a 1191 | * key specified by the application with a key currently stored in the database. 1192 | * If no comparison function is specified, and no special key flags were specified 1193 | * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating 1194 | * before longer keys. 1195 | * @warning This function must be called before any data access functions are used, 1196 | * otherwise data corruption may occur. The same comparison function must be used by every 1197 | * program accessing the database, every time the database is used. 1198 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1199 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1200 | * @param[in] cmp A #MDB_cmp_func function 1201 | * @return A non-zero error value on failure and 0 on success. Some possible 1202 | * errors are: 1203 | *
    1204 | *
  • EINVAL - an invalid parameter was specified. 1205 | *
1206 | */ 1207 | int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); 1208 | 1209 | /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. 1210 | * 1211 | * This comparison function is called whenever it is necessary to compare a data 1212 | * item specified by the application with a data item currently stored in the database. 1213 | * This function only takes effect if the database was opened with the #MDB_DUPSORT 1214 | * flag. 1215 | * If no comparison function is specified, and no special key flags were specified 1216 | * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating 1217 | * before longer items. 1218 | * @warning This function must be called before any data access functions are used, 1219 | * otherwise data corruption may occur. The same comparison function must be used by every 1220 | * program accessing the database, every time the database is used. 1221 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1222 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1223 | * @param[in] cmp A #MDB_cmp_func function 1224 | * @return A non-zero error value on failure and 0 on success. Some possible 1225 | * errors are: 1226 | *
    1227 | *
  • EINVAL - an invalid parameter was specified. 1228 | *
1229 | */ 1230 | int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); 1231 | 1232 | /** @brief Set a relocation function for a #MDB_FIXEDMAP database. 1233 | * 1234 | * @todo The relocation function is called whenever it is necessary to move the data 1235 | * of an item to a different position in the database (e.g. through tree 1236 | * balancing operations, shifts as a result of adds or deletes, etc.). It is 1237 | * intended to allow address/position-dependent data items to be stored in 1238 | * a database in an environment opened with the #MDB_FIXEDMAP option. 1239 | * Currently the relocation feature is unimplemented and setting 1240 | * this function has no effect. 1241 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1242 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1243 | * @param[in] rel A #MDB_rel_func function 1244 | * @return A non-zero error value on failure and 0 on success. Some possible 1245 | * errors are: 1246 | *
    1247 | *
  • EINVAL - an invalid parameter was specified. 1248 | *
1249 | */ 1250 | int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); 1251 | 1252 | /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. 1253 | * 1254 | * See #mdb_set_relfunc and #MDB_rel_func for more details. 1255 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1256 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1257 | * @param[in] ctx An arbitrary pointer for whatever the application needs. 1258 | * It will be passed to the callback function set by #mdb_set_relfunc 1259 | * as its \b relctx parameter whenever the callback is invoked. 1260 | * @return A non-zero error value on failure and 0 on success. Some possible 1261 | * errors are: 1262 | *
    1263 | *
  • EINVAL - an invalid parameter was specified. 1264 | *
1265 | */ 1266 | int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); 1267 | 1268 | /** @brief Get items from a database. 1269 | * 1270 | * This function retrieves key/data pairs from the database. The address 1271 | * and length of the data associated with the specified \b key are returned 1272 | * in the structure to which \b data refers. 1273 | * If the database supports duplicate keys (#MDB_DUPSORT) then the 1274 | * first data item for the key will be returned. Retrieval of other 1275 | * items requires the use of #mdb_cursor_get(). 1276 | * 1277 | * @note The memory pointed to by the returned values is owned by the 1278 | * database. The caller need not dispose of the memory, and may not 1279 | * modify it in any way. For values returned in a read-only transaction 1280 | * any modification attempts will cause a SIGSEGV. 1281 | * @note Values returned from the database are valid only until a 1282 | * subsequent update operation, or the end of the transaction. 1283 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1284 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1285 | * @param[in] key The key to search for in the database 1286 | * @param[out] data The data corresponding to the key 1287 | * @return A non-zero error value on failure and 0 on success. Some possible 1288 | * errors are: 1289 | *
    1290 | *
  • #MDB_NOTFOUND - the key was not in the database. 1291 | *
  • EINVAL - an invalid parameter was specified. 1292 | *
1293 | */ 1294 | int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); 1295 | 1296 | /** @brief Store items into a database. 1297 | * 1298 | * This function stores key/data pairs in the database. The default behavior 1299 | * is to enter the new key/data pair, replacing any previously existing key 1300 | * if duplicates are disallowed, or adding a duplicate data item if 1301 | * duplicates are allowed (#MDB_DUPSORT). 1302 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1303 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1304 | * @param[in] key The key to store in the database 1305 | * @param[in,out] data The data to store 1306 | * @param[in] flags Special options for this operation. This parameter 1307 | * must be set to 0 or by bitwise OR'ing together one or more of the 1308 | * values described here. 1309 | *
    1310 | *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not 1311 | * already appear in the database. This flag may only be specified 1312 | * if the database was opened with #MDB_DUPSORT. The function will 1313 | * return #MDB_KEYEXIST if the key/data pair already appears in the 1314 | * database. 1315 | *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key 1316 | * does not already appear in the database. The function will return 1317 | * #MDB_KEYEXIST if the key already appears in the database, even if 1318 | * the database supports duplicates (#MDB_DUPSORT). The \b data 1319 | * parameter will be set to point to the existing item. 1320 | *
  • #MDB_RESERVE - reserve space for data of the given size, but 1321 | * don't copy the given data. Instead, return a pointer to the 1322 | * reserved space, which the caller can fill in later - before 1323 | * the next update operation or the transaction ends. This saves 1324 | * an extra memcpy if the data is being generated later. 1325 | * LMDB does nothing else with this memory, the caller is expected 1326 | * to modify all of the space requested. This flag must not be 1327 | * specified if the database was opened with #MDB_DUPSORT. 1328 | *
  • #MDB_APPEND - append the given key/data pair to the end of the 1329 | * database. This option allows fast bulk loading when keys are 1330 | * already known to be in the correct order. Loading unsorted keys 1331 | * with this flag will cause a #MDB_KEYEXIST error. 1332 | *
  • #MDB_APPENDDUP - as above, but for sorted dup data. 1333 | *
1334 | * @return A non-zero error value on failure and 0 on success. Some possible 1335 | * errors are: 1336 | *
    1337 | *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). 1338 | *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. 1339 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1340 | *
  • EINVAL - an invalid parameter was specified. 1341 | *
1342 | */ 1343 | int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, 1344 | unsigned int flags); 1345 | 1346 | /** @brief Delete items from a database. 1347 | * 1348 | * This function removes key/data pairs from the database. 1349 | * If the database does not support sorted duplicate data items 1350 | * (#MDB_DUPSORT) the data parameter is ignored. 1351 | * If the database supports sorted duplicates and the data parameter 1352 | * is NULL, all of the duplicate data items for the key will be 1353 | * deleted. Otherwise, if the data parameter is non-NULL 1354 | * only the matching data item will be deleted. 1355 | * This function will return #MDB_NOTFOUND if the specified key/data 1356 | * pair is not in the database. 1357 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1358 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1359 | * @param[in] key The key to delete from the database 1360 | * @param[in] data The data to delete 1361 | * @return A non-zero error value on failure and 0 on success. Some possible 1362 | * errors are: 1363 | *
    1364 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1365 | *
  • EINVAL - an invalid parameter was specified. 1366 | *
1367 | */ 1368 | int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); 1369 | 1370 | /** @brief Create a cursor handle. 1371 | * 1372 | * A cursor is associated with a specific transaction and database. 1373 | * A cursor cannot be used when its database handle is closed. Nor 1374 | * when its transaction has ended, except with #mdb_cursor_renew(). 1375 | * It can be discarded with #mdb_cursor_close(). 1376 | * A cursor in a write-transaction can be closed before its transaction 1377 | * ends, and will otherwise be closed when its transaction ends. 1378 | * A cursor in a read-only transaction must be closed explicitly, before 1379 | * or after its transaction ends. It can be reused with 1380 | * #mdb_cursor_renew() before finally closing it. 1381 | * @note Earlier documentation said that cursors in every transaction 1382 | * were closed when the transaction committed or aborted. 1383 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1384 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1385 | * @param[out] cursor Address where the new #MDB_cursor handle will be stored 1386 | * @return A non-zero error value on failure and 0 on success. Some possible 1387 | * errors are: 1388 | *
    1389 | *
  • EINVAL - an invalid parameter was specified. 1390 | *
1391 | */ 1392 | int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); 1393 | 1394 | /** @brief Close a cursor handle. 1395 | * 1396 | * The cursor handle will be freed and must not be used again after this call. 1397 | * Its transaction must still be live if it is a write-transaction. 1398 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1399 | */ 1400 | void mdb_cursor_close(MDB_cursor *cursor); 1401 | 1402 | /** @brief Renew a cursor handle. 1403 | * 1404 | * A cursor is associated with a specific transaction and database. 1405 | * Cursors that are only used in read-only 1406 | * transactions may be re-used, to avoid unnecessary malloc/free overhead. 1407 | * The cursor may be associated with a new read-only transaction, and 1408 | * referencing the same database handle as it was created with. 1409 | * This may be done whether the previous transaction is live or dead. 1410 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1411 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1412 | * @return A non-zero error value on failure and 0 on success. Some possible 1413 | * errors are: 1414 | *
    1415 | *
  • EINVAL - an invalid parameter was specified. 1416 | *
1417 | */ 1418 | int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); 1419 | 1420 | /** @brief Return the cursor's transaction handle. 1421 | * 1422 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1423 | */ 1424 | MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); 1425 | 1426 | /** @brief Return the cursor's database handle. 1427 | * 1428 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1429 | */ 1430 | MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); 1431 | 1432 | /** @brief Retrieve by cursor. 1433 | * 1434 | * This function retrieves key/data pairs from the database. The address and length 1435 | * of the key are returned in the object to which \b key refers (except for the 1436 | * case of the #MDB_SET option, in which the \b key object is unchanged), and 1437 | * the address and length of the data are returned in the object to which \b data 1438 | * refers. 1439 | * See #mdb_get() for restrictions on using the output values. 1440 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1441 | * @param[in,out] key The key for a retrieved item 1442 | * @param[in,out] data The data of a retrieved item 1443 | * @param[in] op A cursor operation #MDB_cursor_op 1444 | * @return A non-zero error value on failure and 0 on success. Some possible 1445 | * errors are: 1446 | *
    1447 | *
  • #MDB_NOTFOUND - no matching key found. 1448 | *
  • EINVAL - an invalid parameter was specified. 1449 | *
1450 | */ 1451 | int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, 1452 | MDB_cursor_op op); 1453 | 1454 | /** @brief Store by cursor. 1455 | * 1456 | * This function stores key/data pairs into the database. 1457 | * The cursor is positioned at the new item, or on failure usually near it. 1458 | * @note Earlier documentation incorrectly said errors would leave the 1459 | * state of the cursor unchanged. 1460 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1461 | * @param[in] key The key operated on. 1462 | * @param[in] data The data operated on. 1463 | * @param[in] flags Options for this operation. This parameter 1464 | * must be set to 0 or one of the values described here. 1465 | *
    1466 | *
  • #MDB_CURRENT - replace the item at the current cursor position. 1467 | * The \b key parameter must still be provided, and must match it. 1468 | * If using sorted duplicates (#MDB_DUPSORT) the data item must still 1469 | * sort into the same place. This is intended to be used when the 1470 | * new data is the same size as the old. Otherwise it will simply 1471 | * perform a delete of the old record followed by an insert. 1472 | *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not 1473 | * already appear in the database. This flag may only be specified 1474 | * if the database was opened with #MDB_DUPSORT. The function will 1475 | * return #MDB_KEYEXIST if the key/data pair already appears in the 1476 | * database. 1477 | *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key 1478 | * does not already appear in the database. The function will return 1479 | * #MDB_KEYEXIST if the key already appears in the database, even if 1480 | * the database supports duplicates (#MDB_DUPSORT). 1481 | *
  • #MDB_RESERVE - reserve space for data of the given size, but 1482 | * don't copy the given data. Instead, return a pointer to the 1483 | * reserved space, which the caller can fill in later - before 1484 | * the next update operation or the transaction ends. This saves 1485 | * an extra memcpy if the data is being generated later. This flag 1486 | * must not be specified if the database was opened with #MDB_DUPSORT. 1487 | *
  • #MDB_APPEND - append the given key/data pair to the end of the 1488 | * database. No key comparisons are performed. This option allows 1489 | * fast bulk loading when keys are already known to be in the 1490 | * correct order. Loading unsorted keys with this flag will cause 1491 | * a #MDB_KEYEXIST error. 1492 | *
  • #MDB_APPENDDUP - as above, but for sorted dup data. 1493 | *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a 1494 | * single request. This flag may only be specified if the database 1495 | * was opened with #MDB_DUPFIXED. The \b data argument must be an 1496 | * array of two MDB_vals. The mv_size of the first MDB_val must be 1497 | * the size of a single data element. The mv_data of the first MDB_val 1498 | * must point to the beginning of the array of contiguous data elements. 1499 | * The mv_size of the second MDB_val must be the count of the number 1500 | * of data elements to store. On return this field will be set to 1501 | * the count of the number of elements actually written. The mv_data 1502 | * of the second MDB_val is unused. 1503 | *
1504 | * @return A non-zero error value on failure and 0 on success. Some possible 1505 | * errors are: 1506 | *
    1507 | *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). 1508 | *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. 1509 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1510 | *
  • EINVAL - an invalid parameter was specified. 1511 | *
1512 | */ 1513 | int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, 1514 | unsigned int flags); 1515 | 1516 | /** @brief Delete current key/data pair 1517 | * 1518 | * This function deletes the key/data pair to which the cursor refers. 1519 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1520 | * @param[in] flags Options for this operation. This parameter 1521 | * must be set to 0 or one of the values described here. 1522 | *
    1523 | *
  • #MDB_NODUPDATA - delete all of the data items for the current key. 1524 | * This flag may only be specified if the database was opened with #MDB_DUPSORT. 1525 | *
1526 | * @return A non-zero error value on failure and 0 on success. Some possible 1527 | * errors are: 1528 | *
    1529 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1530 | *
  • EINVAL - an invalid parameter was specified. 1531 | *
1532 | */ 1533 | int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); 1534 | 1535 | /** @brief Return count of duplicates for current key. 1536 | * 1537 | * This call is only valid on databases that support sorted duplicate 1538 | * data items #MDB_DUPSORT. 1539 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1540 | * @param[out] countp Address where the count will be stored 1541 | * @return A non-zero error value on failure and 0 on success. Some possible 1542 | * errors are: 1543 | *
    1544 | *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. 1545 | *
1546 | */ 1547 | int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp); 1548 | 1549 | /** @brief Compare two data items according to a particular database. 1550 | * 1551 | * This returns a comparison as if the two data items were keys in the 1552 | * specified database. 1553 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1554 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1555 | * @param[in] a The first item to compare 1556 | * @param[in] b The second item to compare 1557 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b 1558 | */ 1559 | int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); 1560 | 1561 | /** @brief Compare two data items according to a particular database. 1562 | * 1563 | * This returns a comparison as if the two items were data items of 1564 | * the specified database. The database must have the #MDB_DUPSORT flag. 1565 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1566 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1567 | * @param[in] a The first item to compare 1568 | * @param[in] b The second item to compare 1569 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b 1570 | */ 1571 | int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); 1572 | 1573 | /** @brief A callback function used to print a message from the library. 1574 | * 1575 | * @param[in] msg The string to be printed. 1576 | * @param[in] ctx An arbitrary context pointer for the callback. 1577 | * @return < 0 on failure, >= 0 on success. 1578 | */ 1579 | typedef int (MDB_msg_func)(const char *msg, void *ctx); 1580 | 1581 | /** @brief Dump the entries in the reader lock table. 1582 | * 1583 | * @param[in] env An environment handle returned by #mdb_env_create() 1584 | * @param[in] func A #MDB_msg_func function 1585 | * @param[in] ctx Anything the message function needs 1586 | * @return < 0 on failure, >= 0 on success. 1587 | */ 1588 | int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); 1589 | 1590 | /** @brief Check for stale entries in the reader lock table. 1591 | * 1592 | * @param[in] env An environment handle returned by #mdb_env_create() 1593 | * @param[out] dead Number of stale slots that were cleared 1594 | * @return 0 on success, non-zero on failure. 1595 | */ 1596 | int mdb_reader_check(MDB_env *env, int *dead); 1597 | /** @} */ 1598 | 1599 | #ifdef __cplusplus 1600 | } 1601 | #endif 1602 | /** @page tools LMDB Command Line Tools 1603 | The following describes the command line tools that are available for LMDB. 1604 | \li \ref mdb_copy_1 1605 | \li \ref mdb_dump_1 1606 | \li \ref mdb_load_1 1607 | \li \ref mdb_stat_1 1608 | */ 1609 | 1610 | #endif /* _LMDB_H_ */ 1611 | --------------------------------------------------------------------------------