├── .gitignore
├── .npmignore
├── LICENSE
├── README.md
├── binding.gyp
├── deps
├── lmdb.gyp
└── lmdb
│ ├── CHANGES
│ ├── COPYRIGHT
│ ├── LICENSE
│ ├── lmdb.h
│ ├── mdb.c
│ ├── midl.c
│ └── midl.h
├── examples
├── consumer.js
├── producer.js
└── status.js
├── lib
├── consumer-stream.js
├── consumer.js
├── index.js
└── producer.js
├── package.json
└── src
├── consumer.cc
├── consumer.h
├── env.cc
├── env.h
├── module.cc
├── producer.cc
├── producer.h
├── topic.cc
├── topic.h
└── wrapper.h
/.gitignore:
--------------------------------------------------------------------------------
1 | # IDE File
2 | .idea
3 | .vscode
4 |
5 | # Test Data
6 | test-data
7 |
8 | # Logs
9 | logs
10 | *.log
11 |
12 | # Runtime data
13 | pids
14 | *.pid
15 | *.seed
16 |
17 | # Directory for instrumented libs generated by jscoverage/JSCover
18 | lib-cov
19 |
20 | # Coverage directory used by tools like istanbul
21 | coverage
22 |
23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
24 | .grunt
25 |
26 | # Compiled binary addons (http://nodejs.org/api/addons.html)
27 | build
28 | Release
29 | *.sln
30 | *.vcxproj
31 | *.filters
32 |
33 | # Dependency directory
34 | # Commenting this out is preferred by some people, see
35 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git-
36 | node_modules
37 |
38 | # Users Environment Variables
39 | .lock-wscript
40 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | *.tar.gz
2 | *.mdb
3 | *.o
4 | *.obj
5 | *.sln
6 | *.vcxproj
7 |
8 | build
9 | test-data
10 | Release
11 | Debug
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2015 Tal Rasha
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # lmdb-queue
2 | A high performance & embeded queue that can process millions of messages per second. The queue store can be accessed by multi-process.
3 | (Now, it is node.js only, but its core is implemented by C++, it'll be easy to make it work for other languages.)
4 |
5 | ## Usage
6 | - Producer
7 | ```js
8 | var LmdbQueue = require('../'),
9 | Producer = LmdbQueue.Producer;
10 |
11 | var producer = new Producer({ path: __dirname + '/test-data', topic: 'test', dataType: LmdbQueue.STRING_TYPE, chunkSize: 64 * 1024 * 1024, chunksToKeep: 8 });
12 |
13 | /*
14 | options:
15 | path: The path where to put queue files, remember to create it before open it.
16 | topic: Topic name.
17 | dataType:
18 | LmdbQueue.STRING_TYPE: message is treated as string.
19 | LmdbQueue.BUFFER_TYPE: message is treated as buffer.
20 | chunkSize: The size of data chunk(in bytes).
21 | chunksToKeep: How many chunks to keep.
22 | */
23 |
24 | producer.push(['abcdeaef', 'deffdaf']);
25 | ```
26 |
27 | - Consumer
28 | ```js
29 | var LmdbQueue = require('../'),
30 | Consumer = LmdbQueue.Consumer;
31 |
32 | var consumer = new Consumer({ path: __dirname + '/test-data', topic: 'test', name: 'test', dataType: LmdbQueue.STRING_TYPE, chunkSize: 64 * 1024 * 1024, batchSize: 1024 * 16 });
33 | /*
34 | options:
35 | path: The path where to put queue files, remember to create it before open it.
36 | topic: Topic name.
37 | dataType:
38 | LmdbQueue.STRING_TYPE: message is treated as string.
39 | LmdbQueue.BUFFER_TYPE: message is treated as buffer.
40 | chunkSize: The size of data chunk(in bytes).
41 | batchSize: The read batch size.
42 | */
43 |
44 | var msg;
45 | while (msg = consumer.pop()) {
46 | // ...do sth with msg...
47 | }
48 | ```
49 |
--------------------------------------------------------------------------------
/binding.gyp:
--------------------------------------------------------------------------------
1 | {
2 | "targets": [
3 | {
4 | "target_name": "lmdb-queue",
5 | "include_dirs" : [
6 | ".
11 |
12 | OpenLDAP is a registered trademark of the OpenLDAP Foundation.
13 |
14 | Individual files and/or contributed packages may be copyright by
15 | other parties and/or subject to additional restrictions.
16 |
17 | This work also contains materials derived from public sources.
18 |
19 | Additional information about OpenLDAP can be obtained at
20 | .
21 |
--------------------------------------------------------------------------------
/deps/lmdb/LICENSE:
--------------------------------------------------------------------------------
1 | The OpenLDAP Public License
2 | Version 2.8, 17 August 2003
3 |
4 | Redistribution and use of this software and associated documentation
5 | ("Software"), with or without modification, are permitted provided
6 | that the following conditions are met:
7 |
8 | 1. Redistributions in source form must retain copyright statements
9 | and notices,
10 |
11 | 2. Redistributions in binary form must reproduce applicable copyright
12 | statements and notices, this list of conditions, and the following
13 | disclaimer in the documentation and/or other materials provided
14 | with the distribution, and
15 |
16 | 3. Redistributions must contain a verbatim copy of this document.
17 |
18 | The OpenLDAP Foundation may revise this license from time to time.
19 | Each revision is distinguished by a version number. You may use
20 | this Software under terms of this license revision or under the
21 | terms of any subsequent revision of the license.
22 |
23 | THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS
24 | CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
25 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
26 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
27 | SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S)
28 | OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
29 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
32 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
34 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 | POSSIBILITY OF SUCH DAMAGE.
36 |
37 | The names of the authors and copyright holders must not be used in
38 | advertising or otherwise to promote the sale, use or other dealing
39 | in this Software without specific, written prior permission. Title
40 | to copyright in this Software shall at all times remain with copyright
41 | holders.
42 |
43 | OpenLDAP is a registered trademark of the OpenLDAP Foundation.
44 |
45 | Copyright 1999-2003 The OpenLDAP Foundation, Redwood City,
46 | California, USA. All Rights Reserved. Permission to copy and
47 | distribute verbatim copies of this document is granted.
48 |
--------------------------------------------------------------------------------
/deps/lmdb/lmdb.h:
--------------------------------------------------------------------------------
1 | /** @file lmdb.h
2 | * @brief Lightning memory-mapped database library
3 | *
4 | * @mainpage Lightning Memory-Mapped Database Manager (LMDB)
5 | *
6 | * @section intro_sec Introduction
7 | * LMDB is a Btree-based database management library modeled loosely on the
8 | * BerkeleyDB API, but much simplified. The entire database is exposed
9 | * in a memory map, and all data fetches return data directly
10 | * from the mapped memory, so no malloc's or memcpy's occur during
11 | * data fetches. As such, the library is extremely simple because it
12 | * requires no page caching layer of its own, and it is extremely high
13 | * performance and memory-efficient. It is also fully transactional with
14 | * full ACID semantics, and when the memory map is read-only, the
15 | * database integrity cannot be corrupted by stray pointer writes from
16 | * application code.
17 | *
18 | * The library is fully thread-aware and supports concurrent read/write
19 | * access from multiple processes and threads. Data pages use a copy-on-
20 | * write strategy so no active data pages are ever overwritten, which
21 | * also provides resistance to corruption and eliminates the need of any
22 | * special recovery procedures after a system crash. Writes are fully
23 | * serialized; only one write transaction may be active at a time, which
24 | * guarantees that writers can never deadlock. The database structure is
25 | * multi-versioned so readers run with no locks; writers cannot block
26 | * readers, and readers don't block writers.
27 | *
28 | * Unlike other well-known database mechanisms which use either write-ahead
29 | * transaction logs or append-only data writes, LMDB requires no maintenance
30 | * during operation. Both write-ahead loggers and append-only databases
31 | * require periodic checkpointing and/or compaction of their log or database
32 | * files otherwise they grow without bound. LMDB tracks free pages within
33 | * the database and re-uses them for new write operations, so the database
34 | * size does not grow without bound in normal use.
35 | *
36 | * The memory map can be used as a read-only or read-write map. It is
37 | * read-only by default as this provides total immunity to corruption.
38 | * Using read-write mode offers much higher write performance, but adds
39 | * the possibility for stray application writes thru pointers to silently
40 | * corrupt the database. Of course if your application code is known to
41 | * be bug-free (...) then this is not an issue.
42 | *
43 | * @section caveats_sec Caveats
44 | * Troubleshooting the lock file, plus semaphores on BSD systems:
45 | *
46 | * - A broken lockfile can cause sync issues.
47 | * Stale reader transactions left behind by an aborted program
48 | * cause further writes to grow the database quickly, and
49 | * stale locks can block further operation.
50 | *
51 | * Fix: Check for stale readers periodically, using the
52 | * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just
53 | * make all programs using the database close it; the lockfile
54 | * is always reset on first open of the environment.
55 | *
56 | * - On BSD systems or others configured with MDB_USE_POSIX_SEM,
57 | * startup can fail due to semaphores owned by another userid.
58 | *
59 | * Fix: Open and close the database as the user which owns the
60 | * semaphores (likely last user) or as root, while no other
61 | * process is using the database.
62 | *
63 | * Restrictions/caveats (in addition to those listed for some functions):
64 | *
65 | * - Only the database owner should normally use the database on
66 | * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM.
67 | * Multiple users can cause startup to fail later, as noted above.
68 | *
69 | * - There is normally no pure read-only mode, since readers need write
70 | * access to locks and lock file. Exceptions: On read-only filesystems
71 | * or with the #MDB_NOLOCK flag described under #mdb_env_open().
72 | *
73 | * - By default, in versions before 0.9.10, unused portions of the data
74 | * file might receive garbage data from memory freed by other code.
75 | * (This does not happen when using the #MDB_WRITEMAP flag.) As of
76 | * 0.9.10 the default behavior is to initialize such memory before
77 | * writing to the data file. Since there may be a slight performance
78 | * cost due to this initialization, applications may disable it using
79 | * the #MDB_NOMEMINIT flag. Applications handling sensitive data
80 | * which must not be written should not use this flag. This flag is
81 | * irrelevant when using #MDB_WRITEMAP.
82 | *
83 | * - A thread can only use one transaction at a time, plus any child
84 | * transactions. Each transaction belongs to one thread. See below.
85 | * The #MDB_NOTLS flag changes this for read-only transactions.
86 | *
87 | * - Use an MDB_env* in the process which opened it, without fork()ing.
88 | *
89 | * - Do not have open an LMDB database twice in the same process at
90 | * the same time. Not even from a plain open() call - close()ing it
91 | * breaks flock() advisory locking.
92 | *
93 | * - Avoid long-lived transactions. Read transactions prevent
94 | * reuse of pages freed by newer write transactions, thus the
95 | * database can grow quickly. Write transactions prevent
96 | * other write transactions, since writes are serialized.
97 | *
98 | * - Avoid suspending a process with active transactions. These
99 | * would then be "long-lived" as above. Also read transactions
100 | * suspended when writers commit could sometimes see wrong data.
101 | *
102 | * ...when several processes can use a database concurrently:
103 | *
104 | * - Avoid aborting a process with an active transaction.
105 | * The transaction becomes "long-lived" as above until a check
106 | * for stale readers is performed or the lockfile is reset,
107 | * since the process may not remove it from the lockfile.
108 | *
109 | * - If you do that anyway, do a periodic check for stale readers. Or
110 | * close the environment once in a while, so the lockfile can get reset.
111 | *
112 | * - Do not use LMDB databases on remote filesystems, even between
113 | * processes on the same host. This breaks flock() on some OSes,
114 | * possibly memory map sync, and certainly sync between programs
115 | * on different hosts.
116 | *
117 | * - Opening a database can fail if another process is opening or
118 | * closing it at exactly the same time.
119 | *
120 | * @author Howard Chu, Symas Corporation.
121 | *
122 | * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
123 | *
124 | * Redistribution and use in source and binary forms, with or without
125 | * modification, are permitted only as authorized by the OpenLDAP
126 | * Public License.
127 | *
128 | * A copy of this license is available in the file LICENSE in the
129 | * top-level directory of the distribution or, alternatively, at
130 | * .
131 | *
132 | * @par Derived From:
133 | * This code is derived from btree.c written by Martin Hedenfalk.
134 | *
135 | * Copyright (c) 2009, 2010 Martin Hedenfalk
136 | *
137 | * Permission to use, copy, modify, and distribute this software for any
138 | * purpose with or without fee is hereby granted, provided that the above
139 | * copyright notice and this permission notice appear in all copies.
140 | *
141 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
142 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
143 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
144 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
145 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
146 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
147 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
148 | */
149 | #ifndef _LMDB_H_
150 | #define _LMDB_H_
151 |
152 | #include
153 |
154 | #ifdef __cplusplus
155 | extern "C" {
156 | #endif
157 |
158 | /** Unix permissions for creating files, or dummy definition for Windows */
159 | #ifdef _MSC_VER
160 | typedef int mdb_mode_t;
161 | #else
162 | typedef mode_t mdb_mode_t;
163 | #endif
164 |
165 | /** An abstraction for a file handle.
166 | * On POSIX systems file handles are small integers. On Windows
167 | * they're opaque pointers.
168 | */
169 | #ifdef _WIN32
170 | typedef void *mdb_filehandle_t;
171 | #else
172 | typedef int mdb_filehandle_t;
173 | #endif
174 |
175 | /** @defgroup mdb LMDB API
176 | * @{
177 | * @brief OpenLDAP Lightning Memory-Mapped Database Manager
178 | */
179 | /** @defgroup Version Version Macros
180 | * @{
181 | */
182 | /** Library major version */
183 | #define MDB_VERSION_MAJOR 0
184 | /** Library minor version */
185 | #define MDB_VERSION_MINOR 9
186 | /** Library patch version */
187 | #define MDB_VERSION_PATCH 16
188 |
189 | /** Combine args a,b,c into a single integer for easy version comparisons */
190 | #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
191 |
192 | /** The full library version as a single integer */
193 | #define MDB_VERSION_FULL \
194 | MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
195 |
196 | /** The release date of this library version */
197 | #define MDB_VERSION_DATE "August 14, 2015"
198 |
199 | /** A stringifier for the version info */
200 | #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"
201 |
202 | /** A helper for the stringifier macro */
203 | #define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d)
204 |
205 | /** The full library version as a C string */
206 | #define MDB_VERSION_STRING \
207 | MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE)
208 | /** @} */
209 |
210 | /** @brief Opaque structure for a database environment.
211 | *
212 | * A DB environment supports multiple databases, all residing in the same
213 | * shared-memory map.
214 | */
215 | typedef struct MDB_env MDB_env;
216 |
217 | /** @brief Opaque structure for a transaction handle.
218 | *
219 | * All database operations require a transaction handle. Transactions may be
220 | * read-only or read-write.
221 | */
222 | typedef struct MDB_txn MDB_txn;
223 |
224 | /** @brief A handle for an individual database in the DB environment. */
225 | typedef unsigned int MDB_dbi;
226 |
227 | /** @brief Opaque structure for navigating through a database */
228 | typedef struct MDB_cursor MDB_cursor;
229 |
230 | /** @brief Generic structure used for passing keys and data in and out
231 | * of the database.
232 | *
233 | * Values returned from the database are valid only until a subsequent
234 | * update operation, or the end of the transaction. Do not modify or
235 | * free them, they commonly point into the database itself.
236 | *
237 | * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive.
238 | * The same applies to data sizes in databases with the #MDB_DUPSORT flag.
239 | * Other data items can in theory be from 0 to 0xffffffff bytes long.
240 | */
241 | typedef struct MDB_val {
242 | size_t mv_size; /**< size of the data item */
243 | void *mv_data; /**< address of the data item */
244 | } MDB_val;
245 |
246 | /** @brief A callback function used to compare two keys in a database */
247 | typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b);
248 |
249 | /** @brief A callback function used to relocate a position-dependent data item
250 | * in a fixed-address database.
251 | *
252 | * The \b newptr gives the item's desired address in
253 | * the memory map, and \b oldptr gives its previous address. The item's actual
254 | * data resides at the address in \b item. This callback is expected to walk
255 | * through the fields of the record in \b item and modify any
256 | * values based at the \b oldptr address to be relative to the \b newptr address.
257 | * @param[in,out] item The item that is to be relocated.
258 | * @param[in] oldptr The previous address.
259 | * @param[in] newptr The new address to relocate to.
260 | * @param[in] relctx An application-provided context, set by #mdb_set_relctx().
261 | * @todo This feature is currently unimplemented.
262 | */
263 | typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx);
264 |
265 | /** @defgroup mdb_env Environment Flags
266 | * @{
267 | */
268 | /** mmap at a fixed address (experimental) */
269 | #define MDB_FIXEDMAP 0x01
270 | /** no environment directory */
271 | #define MDB_NOSUBDIR 0x4000
272 | /** don't fsync after commit */
273 | #define MDB_NOSYNC 0x10000
274 | /** read only */
275 | #define MDB_RDONLY 0x20000
276 | /** don't fsync metapage after commit */
277 | #define MDB_NOMETASYNC 0x40000
278 | /** use writable mmap */
279 | #define MDB_WRITEMAP 0x80000
280 | /** use asynchronous msync when #MDB_WRITEMAP is used */
281 | #define MDB_MAPASYNC 0x100000
282 | /** tie reader locktable slots to #MDB_txn objects instead of to threads */
283 | #define MDB_NOTLS 0x200000
284 | /** don't do any locking, caller must manage their own locks */
285 | #define MDB_NOLOCK 0x400000
286 | /** don't do readahead (no effect on Windows) */
287 | #define MDB_NORDAHEAD 0x800000
288 | /** don't initialize malloc'd memory before writing to datafile */
289 | #define MDB_NOMEMINIT 0x1000000
290 | /** @} */
291 |
292 | /** @defgroup mdb_dbi_open Database Flags
293 | * @{
294 | */
295 | /** use reverse string keys */
296 | #define MDB_REVERSEKEY 0x02
297 | /** use sorted duplicates */
298 | #define MDB_DUPSORT 0x04
299 | /** numeric keys in native byte order: either unsigned int or size_t.
300 | * The keys must all be of the same size. */
301 | #define MDB_INTEGERKEY 0x08
302 | /** with #MDB_DUPSORT, sorted dup items have fixed size */
303 | #define MDB_DUPFIXED 0x10
304 | /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
305 | #define MDB_INTEGERDUP 0x20
306 | /** with #MDB_DUPSORT, use reverse string dups */
307 | #define MDB_REVERSEDUP 0x40
308 | /** create DB if not already existing */
309 | #define MDB_CREATE 0x40000
310 | /** @} */
311 |
312 | /** @defgroup mdb_put Write Flags
313 | * @{
314 | */
315 | /** For put: Don't write if the key already exists. */
316 | #define MDB_NOOVERWRITE 0x10
317 | /** Only for #MDB_DUPSORT
318 | * For put: don't write if the key and data pair already exist.
319 | * For mdb_cursor_del: remove all duplicate data items.
320 | */
321 | #define MDB_NODUPDATA 0x20
322 | /** For mdb_cursor_put: overwrite the current key/data pair */
323 | #define MDB_CURRENT 0x40
324 | /** For put: Just reserve space for data, don't copy it. Return a
325 | * pointer to the reserved space.
326 | */
327 | #define MDB_RESERVE 0x10000
328 | /** Data is being appended, don't split full pages. */
329 | #define MDB_APPEND 0x20000
330 | /** Duplicate data is being appended, don't split full pages. */
331 | #define MDB_APPENDDUP 0x40000
332 | /** Store multiple data items in one call. Only for #MDB_DUPFIXED. */
333 | #define MDB_MULTIPLE 0x80000
334 | /* @} */
335 |
336 | /** @defgroup mdb_copy Copy Flags
337 | * @{
338 | */
339 | /** Compacting copy: Omit free space from copy, and renumber all
340 | * pages sequentially.
341 | */
342 | #define MDB_CP_COMPACT 0x01
343 | /* @} */
344 |
345 | /** @brief Cursor Get operations.
346 | *
347 | * This is the set of all operations for retrieving data
348 | * using a cursor.
349 | */
350 | typedef enum MDB_cursor_op {
351 | MDB_FIRST, /**< Position at first key/data item */
352 | MDB_FIRST_DUP, /**< Position at first data item of current key.
353 | Only for #MDB_DUPSORT */
354 | MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */
355 | MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */
356 | MDB_GET_CURRENT, /**< Return key/data at current cursor position */
357 | MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items
358 | from current cursor position. Move cursor to prepare
359 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
360 | MDB_LAST, /**< Position at last key/data item */
361 | MDB_LAST_DUP, /**< Position at last data item of current key.
362 | Only for #MDB_DUPSORT */
363 | MDB_NEXT, /**< Position at next data item */
364 | MDB_NEXT_DUP, /**< Position at next data item of current key.
365 | Only for #MDB_DUPSORT */
366 | MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items
367 | from next cursor position. Move cursor to prepare
368 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
369 | MDB_NEXT_NODUP, /**< Position at first data item of next key */
370 | MDB_PREV, /**< Position at previous data item */
371 | MDB_PREV_DUP, /**< Position at previous data item of current key.
372 | Only for #MDB_DUPSORT */
373 | MDB_PREV_NODUP, /**< Position at last data item of previous key */
374 | MDB_SET, /**< Position at specified key */
375 | MDB_SET_KEY, /**< Position at specified key, return key + data */
376 | MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */
377 | } MDB_cursor_op;
378 |
379 | /** @defgroup errors Return Codes
380 | *
381 | * BerkeleyDB uses -30800 to -30999, we'll go under them
382 | * @{
383 | */
384 | /** Successful result */
385 | #define MDB_SUCCESS 0
386 | /** key/data pair already exists */
387 | #define MDB_KEYEXIST (-30799)
388 | /** key/data pair not found (EOF) */
389 | #define MDB_NOTFOUND (-30798)
390 | /** Requested page not found - this usually indicates corruption */
391 | #define MDB_PAGE_NOTFOUND (-30797)
392 | /** Located page was wrong type */
393 | #define MDB_CORRUPTED (-30796)
394 | /** Update of meta page failed, probably I/O error */
395 | #define MDB_PANIC (-30795)
396 | /** Environment version mismatch */
397 | #define MDB_VERSION_MISMATCH (-30794)
398 | /** File is not a valid LMDB file */
399 | #define MDB_INVALID (-30793)
400 | /** Environment mapsize reached */
401 | #define MDB_MAP_FULL (-30792)
402 | /** Environment maxdbs reached */
403 | #define MDB_DBS_FULL (-30791)
404 | /** Environment maxreaders reached */
405 | #define MDB_READERS_FULL (-30790)
406 | /** Too many TLS keys in use - Windows only */
407 | #define MDB_TLS_FULL (-30789)
408 | /** Txn has too many dirty pages */
409 | #define MDB_TXN_FULL (-30788)
410 | /** Cursor stack too deep - internal error */
411 | #define MDB_CURSOR_FULL (-30787)
412 | /** Page has not enough space - internal error */
413 | #define MDB_PAGE_FULL (-30786)
414 | /** Database contents grew beyond environment mapsize */
415 | #define MDB_MAP_RESIZED (-30785)
416 | /** Operation and DB incompatible, or DB type changed. This can mean:
417 | *
418 | *
The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database.
419 | *
Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY.
420 | *
Accessing a data record as a database, or vice versa.
421 | *
The database was dropped and recreated with different flags.
422 | *
423 | */
424 | #define MDB_INCOMPATIBLE (-30784)
425 | /** Invalid reuse of reader locktable slot */
426 | #define MDB_BAD_RSLOT (-30783)
427 | /** Transaction cannot recover - it must be aborted */
428 | #define MDB_BAD_TXN (-30782)
429 | /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */
430 | #define MDB_BAD_VALSIZE (-30781)
431 | /** The specified DBI was changed unexpectedly */
432 | #define MDB_BAD_DBI (-30780)
433 | /** The last defined error code */
434 | #define MDB_LAST_ERRCODE MDB_BAD_DBI
435 | /** @} */
436 |
437 | /** @brief Statistics for a database in the environment */
438 | typedef struct MDB_stat {
439 | unsigned int ms_psize; /**< Size of a database page.
440 | This is currently the same for all databases. */
441 | unsigned int ms_depth; /**< Depth (height) of the B-tree */
442 | size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */
443 | size_t ms_leaf_pages; /**< Number of leaf pages */
444 | size_t ms_overflow_pages; /**< Number of overflow pages */
445 | size_t ms_entries; /**< Number of data items */
446 | } MDB_stat;
447 |
448 | /** @brief Information about the environment */
449 | typedef struct MDB_envinfo {
450 | void *me_mapaddr; /**< Address of map, if fixed */
451 | size_t me_mapsize; /**< Size of the data memory map */
452 | size_t me_last_pgno; /**< ID of the last used page */
453 | size_t me_last_txnid; /**< ID of the last committed transaction */
454 | unsigned int me_maxreaders; /**< max reader slots in the environment */
455 | unsigned int me_numreaders; /**< max reader slots used in the environment */
456 | } MDB_envinfo;
457 |
458 | /** @brief Return the LMDB library version information.
459 | *
460 | * @param[out] major if non-NULL, the library major version number is copied here
461 | * @param[out] minor if non-NULL, the library minor version number is copied here
462 | * @param[out] patch if non-NULL, the library patch version number is copied here
463 | * @retval "version string" The library version as a string
464 | */
465 | char *mdb_version(int *major, int *minor, int *patch);
466 |
467 | /** @brief Return a string describing a given error code.
468 | *
469 | * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3)
470 | * function. If the error code is greater than or equal to 0, then the string
471 | * returned by the system function strerror(3) is returned. If the error code
472 | * is less than 0, an error string corresponding to the LMDB library error is
473 | * returned. See @ref errors for a list of LMDB-specific error codes.
474 | * @param[in] err The error code
475 | * @retval "error message" The description of the error
476 | */
477 | char *mdb_strerror(int err);
478 |
479 | /** @brief Create an LMDB environment handle.
480 | *
481 | * This function allocates memory for a #MDB_env structure. To release
482 | * the allocated memory and discard the handle, call #mdb_env_close().
483 | * Before the handle may be used, it must be opened using #mdb_env_open().
484 | * Various other options may also need to be set before opening the handle,
485 | * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(),
486 | * depending on usage requirements.
487 | * @param[out] env The address where the new handle will be stored
488 | * @return A non-zero error value on failure and 0 on success.
489 | */
490 | int mdb_env_create(MDB_env **env);
491 |
492 | /** @brief Open an environment handle.
493 | *
494 | * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle.
495 | * @param[in] env An environment handle returned by #mdb_env_create()
496 | * @param[in] path The directory in which the database files reside. This
497 | * directory must already exist and be writable.
498 | * @param[in] flags Special options for this environment. This parameter
499 | * must be set to 0 or by bitwise OR'ing together one or more of the
500 | * values described here.
501 | * Flags set by mdb_env_set_flags() are also used.
502 | *
503 | *
#MDB_FIXEDMAP
504 | * use a fixed address for the mmap region. This flag must be specified
505 | * when creating the environment, and is stored persistently in the environment.
506 | * If successful, the memory map will always reside at the same virtual address
507 | * and pointers used to reference data items in the database will be constant
508 | * across multiple invocations. This option may not always work, depending on
509 | * how the operating system has allocated memory to shared libraries and other uses.
510 | * The feature is highly experimental.
511 | *
#MDB_NOSUBDIR
512 | * By default, LMDB creates its environment in a directory whose
513 | * pathname is given in \b path, and creates its data and lock files
514 | * under that directory. With this option, \b path is used as-is for
515 | * the database main data file. The database lock file is the \b path
516 | * with "-lock" appended.
517 | *
#MDB_RDONLY
518 | * Open the environment in read-only mode. No write operations will be
519 | * allowed. LMDB will still modify the lock file - except on read-only
520 | * filesystems, where LMDB does not use locks.
521 | *
#MDB_WRITEMAP
522 | * Use a writeable memory map unless MDB_RDONLY is set. This is faster
523 | * and uses fewer mallocs, but loses protection from application bugs
524 | * like wild pointer writes and other bad updates into the database.
525 | * Incompatible with nested transactions.
526 | * Do not mix processes with and without MDB_WRITEMAP on the same
527 | * environment. This can defeat durability (#mdb_env_sync etc).
528 | *
#MDB_NOMETASYNC
529 | * Flush system buffers to disk only once per transaction, omit the
530 | * metadata flush. Defer that until the system flushes files to disk,
531 | * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization
532 | * maintains database integrity, but a system crash may undo the last
533 | * committed transaction. I.e. it preserves the ACI (atomicity,
534 | * consistency, isolation) but not D (durability) database property.
535 | * This flag may be changed at any time using #mdb_env_set_flags().
536 | *
#MDB_NOSYNC
537 | * Don't flush system buffers to disk when committing a transaction.
538 | * This optimization means a system crash can corrupt the database or
539 | * lose the last transactions if buffers are not yet flushed to disk.
540 | * The risk is governed by how often the system flushes dirty buffers
541 | * to disk and how often #mdb_env_sync() is called. However, if the
542 | * filesystem preserves write order and the #MDB_WRITEMAP flag is not
543 | * used, transactions exhibit ACI (atomicity, consistency, isolation)
544 | * properties and only lose D (durability). I.e. database integrity
545 | * is maintained, but a system crash may undo the final transactions.
546 | * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no
547 | * hint for when to write transactions to disk, unless #mdb_env_sync()
548 | * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable.
549 | * This flag may be changed at any time using #mdb_env_set_flags().
550 | *
#MDB_MAPASYNC
551 | * When using #MDB_WRITEMAP, use asynchronous flushes to disk.
552 | * As with #MDB_NOSYNC, a system crash can then corrupt the
553 | * database or lose the last transactions. Calling #mdb_env_sync()
554 | * ensures on-disk database integrity until next commit.
555 | * This flag may be changed at any time using #mdb_env_set_flags().
556 | *
#MDB_NOTLS
557 | * Don't use Thread-Local Storage. Tie reader locktable slots to
558 | * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps
559 | * the slot reseved for the #MDB_txn object. A thread may use parallel
560 | * read-only transactions. A read-only transaction may span threads if
561 | * the user synchronizes its use. Applications that multiplex many
562 | * user threads over individual OS threads need this option. Such an
563 | * application must also serialize the write transactions in an OS
564 | * thread, since LMDB's write locking is unaware of the user threads.
565 | *
#MDB_NOLOCK
566 | * Don't do any locking. If concurrent access is anticipated, the
567 | * caller must manage all concurrency itself. For proper operation
568 | * the caller must enforce single-writer semantics, and must ensure
569 | * that no readers are using old transactions while a writer is
570 | * active. The simplest approach is to use an exclusive lock so that
571 | * no readers may be active at all when a writer begins.
572 | *
#MDB_NORDAHEAD
573 | * Turn off readahead. Most operating systems perform readahead on
574 | * read requests by default. This option turns it off if the OS
575 | * supports it. Turning it off may help random read performance
576 | * when the DB is larger than RAM and system RAM is full.
577 | * The option is not implemented on Windows.
578 | *
#MDB_NOMEMINIT
579 | * Don't initialize malloc'd memory before writing to unused spaces
580 | * in the data file. By default, memory for pages written to the data
581 | * file is obtained using malloc. While these pages may be reused in
582 | * subsequent transactions, freshly malloc'd pages will be initialized
583 | * to zeroes before use. This avoids persisting leftover data from other
584 | * code (that used the heap and subsequently freed the memory) into the
585 | * data file. Note that many other system libraries may allocate
586 | * and free memory from the heap for arbitrary uses. E.g., stdio may
587 | * use the heap for file I/O buffers. This initialization step has a
588 | * modest performance cost so some applications may want to disable
589 | * it using this flag. This option can be a problem for applications
590 | * which handle sensitive data like passwords, and it makes memory
591 | * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP,
592 | * which writes directly to the mmap instead of using malloc for pages. The
593 | * initialization is also skipped if #MDB_RESERVE is used; the
594 | * caller is expected to overwrite all of the memory that was
595 | * reserved in that case.
596 | * This flag may be changed at any time using #mdb_env_set_flags().
597 | *
598 | * @param[in] mode The UNIX permissions to set on created files and semaphores.
599 | * This parameter is ignored on Windows.
600 | * @return A non-zero error value on failure and 0 on success. Some possible
601 | * errors are:
602 | *
603 | *
#MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the
604 | * version that created the database environment.
605 | *
#MDB_INVALID - the environment file headers are corrupted.
606 | *
ENOENT - the directory specified by the path parameter doesn't exist.
607 | *
EACCES - the user didn't have permission to access the environment files.
608 | *
EAGAIN - the environment was locked by another process.
609 | *
610 | */
611 | int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode);
612 |
613 | /** @brief Copy an LMDB environment to the specified path.
614 | *
615 | * This function may be used to make a backup of an existing environment.
616 | * No lockfile is created, since it gets recreated at need.
617 | * @note This call can trigger significant file size growth if run in
618 | * parallel with write transactions, because it employs a read-only
619 | * transaction. See long-lived transactions under @ref caveats_sec.
620 | * @param[in] env An environment handle returned by #mdb_env_create(). It
621 | * must have already been opened successfully.
622 | * @param[in] path The directory in which the copy will reside. This
623 | * directory must already exist and be writable but must otherwise be
624 | * empty.
625 | * @return A non-zero error value on failure and 0 on success.
626 | */
627 | int mdb_env_copy(MDB_env *env, const char *path);
628 |
629 | /** @brief Copy an LMDB environment to the specified file descriptor.
630 | *
631 | * This function may be used to make a backup of an existing environment.
632 | * No lockfile is created, since it gets recreated at need.
633 | * @note This call can trigger significant file size growth if run in
634 | * parallel with write transactions, because it employs a read-only
635 | * transaction. See long-lived transactions under @ref caveats_sec.
636 | * @param[in] env An environment handle returned by #mdb_env_create(). It
637 | * must have already been opened successfully.
638 | * @param[in] fd The filedescriptor to write the copy to. It must
639 | * have already been opened for Write access.
640 | * @return A non-zero error value on failure and 0 on success.
641 | */
642 | int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
643 |
644 | /** @brief Copy an LMDB environment to the specified path, with options.
645 | *
646 | * This function may be used to make a backup of an existing environment.
647 | * No lockfile is created, since it gets recreated at need.
648 | * @note This call can trigger significant file size growth if run in
649 | * parallel with write transactions, because it employs a read-only
650 | * transaction. See long-lived transactions under @ref caveats_sec.
651 | * @param[in] env An environment handle returned by #mdb_env_create(). It
652 | * must have already been opened successfully.
653 | * @param[in] path The directory in which the copy will reside. This
654 | * directory must already exist and be writable but must otherwise be
655 | * empty.
656 | * @param[in] flags Special options for this operation. This parameter
657 | * must be set to 0 or by bitwise OR'ing together one or more of the
658 | * values described here.
659 | *
660 | *
#MDB_CP_COMPACT - Perform compaction while copying: omit free
661 | * pages and sequentially renumber all pages in output. This option
662 | * consumes more CPU and runs more slowly than the default.
663 | *
664 | * @return A non-zero error value on failure and 0 on success.
665 | */
666 | int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
667 |
668 | /** @brief Copy an LMDB environment to the specified file descriptor,
669 | * with options.
670 | *
671 | * This function may be used to make a backup of an existing environment.
672 | * No lockfile is created, since it gets recreated at need. See
673 | * #mdb_env_copy2() for further details.
674 | * @note This call can trigger significant file size growth if run in
675 | * parallel with write transactions, because it employs a read-only
676 | * transaction. See long-lived transactions under @ref caveats_sec.
677 | * @param[in] env An environment handle returned by #mdb_env_create(). It
678 | * must have already been opened successfully.
679 | * @param[in] fd The filedescriptor to write the copy to. It must
680 | * have already been opened for Write access.
681 | * @param[in] flags Special options for this operation.
682 | * See #mdb_env_copy2() for options.
683 | * @return A non-zero error value on failure and 0 on success.
684 | */
685 | int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
686 |
687 | /** @brief Return statistics about the LMDB environment.
688 | *
689 | * @param[in] env An environment handle returned by #mdb_env_create()
690 | * @param[out] stat The address of an #MDB_stat structure
691 | * where the statistics will be copied
692 | */
693 | int mdb_env_stat(MDB_env *env, MDB_stat *stat);
694 |
695 | /** @brief Return information about the LMDB environment.
696 | *
697 | * @param[in] env An environment handle returned by #mdb_env_create()
698 | * @param[out] stat The address of an #MDB_envinfo structure
699 | * where the information will be copied
700 | */
701 | int mdb_env_info(MDB_env *env, MDB_envinfo *stat);
702 |
703 | /** @brief Flush the data buffers to disk.
704 | *
705 | * Data is always written to disk when #mdb_txn_commit() is called,
706 | * but the operating system may keep it buffered. LMDB always flushes
707 | * the OS buffers upon commit as well, unless the environment was
708 | * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is
709 | * not valid if the environment was opened with #MDB_RDONLY.
710 | * @param[in] env An environment handle returned by #mdb_env_create()
711 | * @param[in] force If non-zero, force a synchronous flush. Otherwise
712 | * if the environment has the #MDB_NOSYNC flag set the flushes
713 | * will be omitted, and with #MDB_MAPASYNC they will be asynchronous.
714 | * @return A non-zero error value on failure and 0 on success. Some possible
715 | * errors are:
716 | *
717 | *
EACCES - the environment is read-only.
718 | *
EINVAL - an invalid parameter was specified.
719 | *
EIO - an error occurred during synchronization.
720 | *
721 | */
722 | int mdb_env_sync(MDB_env *env, int force);
723 |
724 | /** @brief Close the environment and release the memory map.
725 | *
726 | * Only a single thread may call this function. All transactions, databases,
727 | * and cursors must already be closed before calling this function. Attempts to
728 | * use any such handles after calling this function will cause a SIGSEGV.
729 | * The environment handle will be freed and must not be used again after this call.
730 | * @param[in] env An environment handle returned by #mdb_env_create()
731 | */
732 | void mdb_env_close(MDB_env *env);
733 |
734 | /** @brief Set environment flags.
735 | *
736 | * This may be used to set some flags in addition to those from
737 | * #mdb_env_open(), or to unset these flags. If several threads
738 | * change the flags at the same time, the result is undefined.
739 | * @param[in] env An environment handle returned by #mdb_env_create()
740 | * @param[in] flags The flags to change, bitwise OR'ed together
741 | * @param[in] onoff A non-zero value sets the flags, zero clears them.
742 | * @return A non-zero error value on failure and 0 on success. Some possible
743 | * errors are:
744 | *
745 | *
EINVAL - an invalid parameter was specified.
746 | *
747 | */
748 | int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff);
749 |
750 | /** @brief Get environment flags.
751 | *
752 | * @param[in] env An environment handle returned by #mdb_env_create()
753 | * @param[out] flags The address of an integer to store the flags
754 | * @return A non-zero error value on failure and 0 on success. Some possible
755 | * errors are:
756 | *
757 | *
EINVAL - an invalid parameter was specified.
758 | *
759 | */
760 | int mdb_env_get_flags(MDB_env *env, unsigned int *flags);
761 |
762 | /** @brief Return the path that was used in #mdb_env_open().
763 | *
764 | * @param[in] env An environment handle returned by #mdb_env_create()
765 | * @param[out] path Address of a string pointer to contain the path. This
766 | * is the actual string in the environment, not a copy. It should not be
767 | * altered in any way.
768 | * @return A non-zero error value on failure and 0 on success. Some possible
769 | * errors are:
770 | *
771 | *
EINVAL - an invalid parameter was specified.
772 | *
773 | */
774 | int mdb_env_get_path(MDB_env *env, const char **path);
775 |
776 | /** @brief Return the filedescriptor for the given environment.
777 | *
778 | * @param[in] env An environment handle returned by #mdb_env_create()
779 | * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor.
780 | * @return A non-zero error value on failure and 0 on success. Some possible
781 | * errors are:
782 | *
783 | *
EINVAL - an invalid parameter was specified.
784 | *
785 | */
786 | int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd);
787 |
788 | /** @brief Set the size of the memory map to use for this environment.
789 | *
790 | * The size should be a multiple of the OS page size. The default is
791 | * 10485760 bytes. The size of the memory map is also the maximum size
792 | * of the database. The value should be chosen as large as possible,
793 | * to accommodate future growth of the database.
794 | * This function should be called after #mdb_env_create() and before #mdb_env_open().
795 | * It may be called at later times if no transactions are active in
796 | * this process. Note that the library does not check for this condition,
797 | * the caller must ensure it explicitly.
798 | *
799 | * The new size takes effect immediately for the current process but
800 | * will not be persisted to any others until a write transaction has been
801 | * committed by the current process. Also, only mapsize increases are
802 | * persisted into the environment.
803 | *
804 | * If the mapsize is increased by another process, and data has grown
805 | * beyond the range of the current mapsize, #mdb_txn_begin() will
806 | * return #MDB_MAP_RESIZED. This function may be called with a size
807 | * of zero to adopt the new size.
808 | *
809 | * Any attempt to set a size smaller than the space already consumed
810 | * by the environment will be silently changed to the current size of the used space.
811 | * @param[in] env An environment handle returned by #mdb_env_create()
812 | * @param[in] size The size in bytes
813 | * @return A non-zero error value on failure and 0 on success. Some possible
814 | * errors are:
815 | *
816 | *
EINVAL - an invalid parameter was specified, or the environment has
817 | * an active write transaction.
818 | *
819 | */
820 | int mdb_env_set_mapsize(MDB_env *env, size_t size);
821 |
822 | /** @brief Set the maximum number of threads/reader slots for the environment.
823 | *
824 | * This defines the number of slots in the lock table that is used to track readers in the
825 | * the environment. The default is 126.
826 | * Starting a read-only transaction normally ties a lock table slot to the
827 | * current thread until the environment closes or the thread exits. If
828 | * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the
829 | * MDB_txn object until it or the #MDB_env object is destroyed.
830 | * This function may only be called after #mdb_env_create() and before #mdb_env_open().
831 | * @param[in] env An environment handle returned by #mdb_env_create()
832 | * @param[in] readers The maximum number of reader lock table slots
833 | * @return A non-zero error value on failure and 0 on success. Some possible
834 | * errors are:
835 | *
836 | *
EINVAL - an invalid parameter was specified, or the environment is already open.
837 | *
838 | */
839 | int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers);
840 |
841 | /** @brief Get the maximum number of threads/reader slots for the environment.
842 | *
843 | * @param[in] env An environment handle returned by #mdb_env_create()
844 | * @param[out] readers Address of an integer to store the number of readers
845 | * @return A non-zero error value on failure and 0 on success. Some possible
846 | * errors are:
847 | *
848 | *
EINVAL - an invalid parameter was specified.
849 | *
850 | */
851 | int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers);
852 |
853 | /** @brief Set the maximum number of named databases for the environment.
854 | *
855 | * This function is only needed if multiple databases will be used in the
856 | * environment. Simpler applications that use the environment as a single
857 | * unnamed database can ignore this option.
858 | * This function may only be called after #mdb_env_create() and before #mdb_env_open().
859 | *
860 | * Currently a moderate number of slots are cheap but a huge number gets
861 | * expensive: 7-120 words per transaction, and every #mdb_dbi_open()
862 | * does a linear search of the opened slots.
863 | * @param[in] env An environment handle returned by #mdb_env_create()
864 | * @param[in] dbs The maximum number of databases
865 | * @return A non-zero error value on failure and 0 on success. Some possible
866 | * errors are:
867 | *
868 | *
EINVAL - an invalid parameter was specified, or the environment is already open.
869 | *
870 | */
871 | int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs);
872 |
873 | /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write.
874 | *
875 | * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511.
876 | * See @ref MDB_val.
877 | * @param[in] env An environment handle returned by #mdb_env_create()
878 | * @return The maximum size of a key we can write
879 | */
880 | int mdb_env_get_maxkeysize(MDB_env *env);
881 |
882 | /** @brief Set application information associated with the #MDB_env.
883 | *
884 | * @param[in] env An environment handle returned by #mdb_env_create()
885 | * @param[in] ctx An arbitrary pointer for whatever the application needs.
886 | * @return A non-zero error value on failure and 0 on success.
887 | */
888 | int mdb_env_set_userctx(MDB_env *env, void *ctx);
889 |
890 | /** @brief Get the application information associated with the #MDB_env.
891 | *
892 | * @param[in] env An environment handle returned by #mdb_env_create()
893 | * @return The pointer set by #mdb_env_set_userctx().
894 | */
895 | void *mdb_env_get_userctx(MDB_env *env);
896 |
897 | /** @brief A callback function for most LMDB assert() failures,
898 | * called before printing the message and aborting.
899 | *
900 | * @param[in] env An environment handle returned by #mdb_env_create().
901 | * @param[in] msg The assertion message, not including newline.
902 | */
903 | typedef void MDB_assert_func(MDB_env *env, const char *msg);
904 |
905 | /** Set or reset the assert() callback of the environment.
906 | * Disabled if liblmdb is buillt with NDEBUG.
907 | * @note This hack should become obsolete as lmdb's error handling matures.
908 | * @param[in] env An environment handle returned by #mdb_env_create().
909 | * @param[in] func An #MDB_assert_func function, or 0.
910 | * @return A non-zero error value on failure and 0 on success.
911 | */
912 | int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
913 |
914 | /** @brief Create a transaction for use with the environment.
915 | *
916 | * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit().
917 | * @note A transaction and its cursors must only be used by a single
918 | * thread, and a thread may only have a single transaction at a time.
919 | * If #MDB_NOTLS is in use, this does not apply to read-only transactions.
920 | * @note Cursors may not span transactions.
921 | * @param[in] env An environment handle returned by #mdb_env_create()
922 | * @param[in] parent If this parameter is non-NULL, the new transaction
923 | * will be a nested transaction, with the transaction indicated by \b parent
924 | * as its parent. Transactions may be nested to any level. A parent
925 | * transaction and its cursors may not issue any other operations than
926 | * mdb_txn_commit and mdb_txn_abort while it has active child transactions.
927 | * @param[in] flags Special options for this transaction. This parameter
928 | * must be set to 0 or by bitwise OR'ing together one or more of the
929 | * values described here.
930 | *
931 | *
#MDB_RDONLY
932 | * This transaction will not perform any write operations.
933 | *
934 | * @param[out] txn Address where the new #MDB_txn handle will be stored
935 | * @return A non-zero error value on failure and 0 on success. Some possible
936 | * errors are:
937 | *
938 | *
#MDB_PANIC - a fatal error occurred earlier and the environment
939 | * must be shut down.
940 | *
#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's
941 | * mapsize and this environment's map must be resized as well.
942 | * See #mdb_env_set_mapsize().
943 | *
#MDB_READERS_FULL - a read-only transaction was requested and
944 | * the reader lock table is full. See #mdb_env_set_maxreaders().
945 | *
ENOMEM - out of memory.
946 | *
947 | */
948 | int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn);
949 |
950 | /** @brief Returns the transaction's #MDB_env
951 | *
952 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
953 | */
954 | MDB_env *mdb_txn_env(MDB_txn *txn);
955 |
956 | /** @brief Commit all the operations of a transaction into the database.
957 | *
958 | * The transaction handle is freed. It and its cursors must not be used
959 | * again after this call, except with #mdb_cursor_renew().
960 | * @note Earlier documentation incorrectly said all cursors would be freed.
961 | * Only write-transactions free cursors.
962 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
963 | * @return A non-zero error value on failure and 0 on success. Some possible
964 | * errors are:
965 | *
966 | *
EINVAL - an invalid parameter was specified.
967 | *
ENOSPC - no more disk space.
968 | *
EIO - a low-level I/O error occurred while writing.
969 | *
ENOMEM - out of memory.
970 | *
971 | */
972 | int mdb_txn_commit(MDB_txn *txn);
973 |
974 | /** @brief Abandon all the operations of the transaction instead of saving them.
975 | *
976 | * The transaction handle is freed. It and its cursors must not be used
977 | * again after this call, except with #mdb_cursor_renew().
978 | * @note Earlier documentation incorrectly said all cursors would be freed.
979 | * Only write-transactions free cursors.
980 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
981 | */
982 | void mdb_txn_abort(MDB_txn *txn);
983 |
984 | /** @brief Reset a read-only transaction.
985 | *
986 | * Abort the transaction like #mdb_txn_abort(), but keep the transaction
987 | * handle. #mdb_txn_renew() may reuse the handle. This saves allocation
988 | * overhead if the process will start a new read-only transaction soon,
989 | * and also locking overhead if #MDB_NOTLS is in use. The reader table
990 | * lock is released, but the table slot stays tied to its thread or
991 | * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free
992 | * its lock table slot if MDB_NOTLS is in use.
993 | * Cursors opened within the transaction must not be used
994 | * again after this call, except with #mdb_cursor_renew().
995 | * Reader locks generally don't interfere with writers, but they keep old
996 | * versions of database pages allocated. Thus they prevent the old pages
997 | * from being reused when writers commit new data, and so under heavy load
998 | * the database size may grow much more rapidly than otherwise.
999 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1000 | */
1001 | void mdb_txn_reset(MDB_txn *txn);
1002 |
1003 | /** @brief Renew a read-only transaction.
1004 | *
1005 | * This acquires a new reader lock for a transaction handle that had been
1006 | * released by #mdb_txn_reset(). It must be called before a reset transaction
1007 | * may be used again.
1008 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1009 | * @return A non-zero error value on failure and 0 on success. Some possible
1010 | * errors are:
1011 | *
1012 | *
#MDB_PANIC - a fatal error occurred earlier and the environment
1013 | * must be shut down.
1014 | *
EINVAL - an invalid parameter was specified.
1015 | *
1016 | */
1017 | int mdb_txn_renew(MDB_txn *txn);
1018 |
1019 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */
1020 | #define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi)
1021 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */
1022 | #define mdb_close(env,dbi) mdb_dbi_close(env,dbi)
1023 |
1024 | /** @brief Open a database in the environment.
1025 | *
1026 | * A database handle denotes the name and parameters of a database,
1027 | * independently of whether such a database exists.
1028 | * The database handle may be discarded by calling #mdb_dbi_close().
1029 | * The old database handle is returned if the database was already open.
1030 | * The handle may only be closed once.
1031 | *
1032 | * The database handle will be private to the current transaction until
1033 | * the transaction is successfully committed. If the transaction is
1034 | * aborted the handle will be closed automatically.
1035 | * After a successful commit the handle will reside in the shared
1036 | * environment, and may be used by other transactions.
1037 | *
1038 | * This function must not be called from multiple concurrent
1039 | * transactions in the same process. A transaction that uses
1040 | * this function must finish (either commit or abort) before
1041 | * any other transaction in the process may use this function.
1042 | *
1043 | * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
1044 | * must be called before opening the environment. Database names are
1045 | * keys in the unnamed database, and may be read but not written.
1046 | *
1047 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1048 | * @param[in] name The name of the database to open. If only a single
1049 | * database is needed in the environment, this value may be NULL.
1050 | * @param[in] flags Special options for this database. This parameter
1051 | * must be set to 0 or by bitwise OR'ing together one or more of the
1052 | * values described here.
1053 | *
1054 | *
#MDB_REVERSEKEY
1055 | * Keys are strings to be compared in reverse order, from the end
1056 | * of the strings to the beginning. By default, Keys are treated as strings and
1057 | * compared from beginning to end.
1058 | *
#MDB_DUPSORT
1059 | * Duplicate keys may be used in the database. (Or, from another perspective,
1060 | * keys may have multiple data items, stored in sorted order.) By default
1061 | * keys must be unique and may have only a single data item.
1062 | *
#MDB_INTEGERKEY
1063 | * Keys are binary integers in native byte order, either unsigned int
1064 | * or size_t, and will be sorted as such.
1065 | * The keys must all be of the same size.
1066 | *
#MDB_DUPFIXED
1067 | * This flag may only be used in combination with #MDB_DUPSORT. This option
1068 | * tells the library that the data items for this database are all the same
1069 | * size, which allows further optimizations in storage and retrieval. When
1070 | * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
1071 | * cursor operations may be used to retrieve multiple items at once.
1072 | *
#MDB_INTEGERDUP
1073 | * This option specifies that duplicate data items are binary integers,
1074 | * similar to #MDB_INTEGERKEY keys.
1075 | *
#MDB_REVERSEDUP
1076 | * This option specifies that duplicate data items should be compared as
1077 | * strings in reverse order.
1078 | *
#MDB_CREATE
1079 | * Create the named database if it doesn't exist. This option is not
1080 | * allowed in a read-only transaction or a read-only environment.
1081 | *
1082 | * @param[out] dbi Address where the new #MDB_dbi handle will be stored
1083 | * @return A non-zero error value on failure and 0 on success. Some possible
1084 | * errors are:
1085 | *
1086 | *
#MDB_NOTFOUND - the specified database doesn't exist in the environment
1087 | * and #MDB_CREATE was not specified.
1088 | *
#MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs().
1089 | *
1090 | */
1091 | int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi);
1092 |
1093 | /** @brief Retrieve statistics for a database.
1094 | *
1095 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1096 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1097 | * @param[out] stat The address of an #MDB_stat structure
1098 | * where the statistics will be copied
1099 | * @return A non-zero error value on failure and 0 on success. Some possible
1100 | * errors are:
1101 | *
1102 | *
EINVAL - an invalid parameter was specified.
1103 | *
1104 | */
1105 | int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat);
1106 |
1107 | /** @brief Retrieve the DB flags for a database handle.
1108 | *
1109 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1110 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1111 | * @param[out] flags Address where the flags will be returned.
1112 | * @return A non-zero error value on failure and 0 on success.
1113 | */
1114 | int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags);
1115 |
1116 | /** @brief Close a database handle. Normally unnecessary. Use with care:
1117 | *
1118 | * This call is not mutex protected. Handles should only be closed by
1119 | * a single thread, and only if no other threads are going to reference
1120 | * the database handle or one of its cursors any further. Do not close
1121 | * a handle if an existing transaction has modified its database.
1122 | * Doing so can cause misbehavior from database corruption to errors
1123 | * like MDB_BAD_VALSIZE (since the DB name is gone).
1124 | *
1125 | * Closing a database handle is not necessary, but lets #mdb_dbi_open()
1126 | * reuse the handle value. Usually it's better to set a bigger
1127 | * #mdb_env_set_maxdbs(), unless that value would be large.
1128 | *
1129 | * @param[in] env An environment handle returned by #mdb_env_create()
1130 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1131 | */
1132 | void mdb_dbi_close(MDB_env *env, MDB_dbi dbi);
1133 |
1134 | /** @brief Empty or delete+close a database.
1135 | *
1136 | * See #mdb_dbi_close() for restrictions about closing the DB handle.
1137 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1138 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1139 | * @param[in] del 0 to empty the DB, 1 to delete it from the
1140 | * environment and close the DB handle.
1141 | * @return A non-zero error value on failure and 0 on success.
1142 | */
1143 | int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del);
1144 |
1145 | /** @brief Set a custom key comparison function for a database.
1146 | *
1147 | * The comparison function is called whenever it is necessary to compare a
1148 | * key specified by the application with a key currently stored in the database.
1149 | * If no comparison function is specified, and no special key flags were specified
1150 | * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating
1151 | * before longer keys.
1152 | * @warning This function must be called before any data access functions are used,
1153 | * otherwise data corruption may occur. The same comparison function must be used by every
1154 | * program accessing the database, every time the database is used.
1155 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1156 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1157 | * @param[in] cmp A #MDB_cmp_func function
1158 | * @return A non-zero error value on failure and 0 on success. Some possible
1159 | * errors are:
1160 | *
1161 | *
EINVAL - an invalid parameter was specified.
1162 | *
1163 | */
1164 | int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp);
1165 |
1166 | /** @brief Set a custom data comparison function for a #MDB_DUPSORT database.
1167 | *
1168 | * This comparison function is called whenever it is necessary to compare a data
1169 | * item specified by the application with a data item currently stored in the database.
1170 | * This function only takes effect if the database was opened with the #MDB_DUPSORT
1171 | * flag.
1172 | * If no comparison function is specified, and no special key flags were specified
1173 | * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating
1174 | * before longer items.
1175 | * @warning This function must be called before any data access functions are used,
1176 | * otherwise data corruption may occur. The same comparison function must be used by every
1177 | * program accessing the database, every time the database is used.
1178 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1179 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1180 | * @param[in] cmp A #MDB_cmp_func function
1181 | * @return A non-zero error value on failure and 0 on success. Some possible
1182 | * errors are:
1183 | *
1184 | *
EINVAL - an invalid parameter was specified.
1185 | *
1186 | */
1187 | int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp);
1188 |
1189 | /** @brief Set a relocation function for a #MDB_FIXEDMAP database.
1190 | *
1191 | * @todo The relocation function is called whenever it is necessary to move the data
1192 | * of an item to a different position in the database (e.g. through tree
1193 | * balancing operations, shifts as a result of adds or deletes, etc.). It is
1194 | * intended to allow address/position-dependent data items to be stored in
1195 | * a database in an environment opened with the #MDB_FIXEDMAP option.
1196 | * Currently the relocation feature is unimplemented and setting
1197 | * this function has no effect.
1198 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1199 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1200 | * @param[in] rel A #MDB_rel_func function
1201 | * @return A non-zero error value on failure and 0 on success. Some possible
1202 | * errors are:
1203 | *
1204 | *
EINVAL - an invalid parameter was specified.
1205 | *
1206 | */
1207 | int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel);
1208 |
1209 | /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function.
1210 | *
1211 | * See #mdb_set_relfunc and #MDB_rel_func for more details.
1212 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1213 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1214 | * @param[in] ctx An arbitrary pointer for whatever the application needs.
1215 | * It will be passed to the callback function set by #mdb_set_relfunc
1216 | * as its \b relctx parameter whenever the callback is invoked.
1217 | * @return A non-zero error value on failure and 0 on success. Some possible
1218 | * errors are:
1219 | *
1220 | *
EINVAL - an invalid parameter was specified.
1221 | *
1222 | */
1223 | int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx);
1224 |
1225 | /** @brief Get items from a database.
1226 | *
1227 | * This function retrieves key/data pairs from the database. The address
1228 | * and length of the data associated with the specified \b key are returned
1229 | * in the structure to which \b data refers.
1230 | * If the database supports duplicate keys (#MDB_DUPSORT) then the
1231 | * first data item for the key will be returned. Retrieval of other
1232 | * items requires the use of #mdb_cursor_get().
1233 | *
1234 | * @note The memory pointed to by the returned values is owned by the
1235 | * database. The caller need not dispose of the memory, and may not
1236 | * modify it in any way. For values returned in a read-only transaction
1237 | * any modification attempts will cause a SIGSEGV.
1238 | * @note Values returned from the database are valid only until a
1239 | * subsequent update operation, or the end of the transaction.
1240 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1241 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1242 | * @param[in] key The key to search for in the database
1243 | * @param[out] data The data corresponding to the key
1244 | * @return A non-zero error value on failure and 0 on success. Some possible
1245 | * errors are:
1246 | *
1247 | *
#MDB_NOTFOUND - the key was not in the database.
1248 | *
EINVAL - an invalid parameter was specified.
1249 | *
1250 | */
1251 | int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
1252 |
1253 | /** @brief Store items into a database.
1254 | *
1255 | * This function stores key/data pairs in the database. The default behavior
1256 | * is to enter the new key/data pair, replacing any previously existing key
1257 | * if duplicates are disallowed, or adding a duplicate data item if
1258 | * duplicates are allowed (#MDB_DUPSORT).
1259 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1260 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1261 | * @param[in] key The key to store in the database
1262 | * @param[in,out] data The data to store
1263 | * @param[in] flags Special options for this operation. This parameter
1264 | * must be set to 0 or by bitwise OR'ing together one or more of the
1265 | * values described here.
1266 | *
1267 | *
#MDB_NODUPDATA - enter the new key/data pair only if it does not
1268 | * already appear in the database. This flag may only be specified
1269 | * if the database was opened with #MDB_DUPSORT. The function will
1270 | * return #MDB_KEYEXIST if the key/data pair already appears in the
1271 | * database.
1272 | *
#MDB_NOOVERWRITE - enter the new key/data pair only if the key
1273 | * does not already appear in the database. The function will return
1274 | * #MDB_KEYEXIST if the key already appears in the database, even if
1275 | * the database supports duplicates (#MDB_DUPSORT). The \b data
1276 | * parameter will be set to point to the existing item.
1277 | *
#MDB_RESERVE - reserve space for data of the given size, but
1278 | * don't copy the given data. Instead, return a pointer to the
1279 | * reserved space, which the caller can fill in later - before
1280 | * the next update operation or the transaction ends. This saves
1281 | * an extra memcpy if the data is being generated later.
1282 | * LMDB does nothing else with this memory, the caller is expected
1283 | * to modify all of the space requested.
1284 | *
#MDB_APPEND - append the given key/data pair to the end of the
1285 | * database. This option allows fast bulk loading when keys are
1286 | * already known to be in the correct order. Loading unsorted keys
1287 | * with this flag will cause a #MDB_KEYEXIST error.
1288 | *
#MDB_APPENDDUP - as above, but for sorted dup data.
1289 | *
1290 | * @return A non-zero error value on failure and 0 on success. Some possible
1291 | * errors are:
1292 | *
1293 | *
#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
1294 | *
#MDB_TXN_FULL - the transaction has too many dirty pages.
1295 | *
EACCES - an attempt was made to write in a read-only transaction.
1296 | *
EINVAL - an invalid parameter was specified.
1297 | *
1298 | */
1299 | int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data,
1300 | unsigned int flags);
1301 |
1302 | /** @brief Delete items from a database.
1303 | *
1304 | * This function removes key/data pairs from the database.
1305 | * If the database does not support sorted duplicate data items
1306 | * (#MDB_DUPSORT) the data parameter is ignored.
1307 | * If the database supports sorted duplicates and the data parameter
1308 | * is NULL, all of the duplicate data items for the key will be
1309 | * deleted. Otherwise, if the data parameter is non-NULL
1310 | * only the matching data item will be deleted.
1311 | * This function will return #MDB_NOTFOUND if the specified key/data
1312 | * pair is not in the database.
1313 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1314 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1315 | * @param[in] key The key to delete from the database
1316 | * @param[in] data The data to delete
1317 | * @return A non-zero error value on failure and 0 on success. Some possible
1318 | * errors are:
1319 | *
1320 | *
EACCES - an attempt was made to write in a read-only transaction.
1321 | *
EINVAL - an invalid parameter was specified.
1322 | *
1323 | */
1324 | int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
1325 |
1326 | /** @brief Create a cursor handle.
1327 | *
1328 | * A cursor is associated with a specific transaction and database.
1329 | * A cursor cannot be used when its database handle is closed. Nor
1330 | * when its transaction has ended, except with #mdb_cursor_renew().
1331 | * It can be discarded with #mdb_cursor_close().
1332 | * A cursor in a write-transaction can be closed before its transaction
1333 | * ends, and will otherwise be closed when its transaction ends.
1334 | * A cursor in a read-only transaction must be closed explicitly, before
1335 | * or after its transaction ends. It can be reused with
1336 | * #mdb_cursor_renew() before finally closing it.
1337 | * @note Earlier documentation said that cursors in every transaction
1338 | * were closed when the transaction committed or aborted.
1339 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1340 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1341 | * @param[out] cursor Address where the new #MDB_cursor handle will be stored
1342 | * @return A non-zero error value on failure and 0 on success. Some possible
1343 | * errors are:
1344 | *
1345 | *
EINVAL - an invalid parameter was specified.
1346 | *
1347 | */
1348 | int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor);
1349 |
1350 | /** @brief Close a cursor handle.
1351 | *
1352 | * The cursor handle will be freed and must not be used again after this call.
1353 | * Its transaction must still be live if it is a write-transaction.
1354 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1355 | */
1356 | void mdb_cursor_close(MDB_cursor *cursor);
1357 |
1358 | /** @brief Renew a cursor handle.
1359 | *
1360 | * A cursor is associated with a specific transaction and database.
1361 | * Cursors that are only used in read-only
1362 | * transactions may be re-used, to avoid unnecessary malloc/free overhead.
1363 | * The cursor may be associated with a new read-only transaction, and
1364 | * referencing the same database handle as it was created with.
1365 | * This may be done whether the previous transaction is live or dead.
1366 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1367 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1368 | * @return A non-zero error value on failure and 0 on success. Some possible
1369 | * errors are:
1370 | *
1371 | *
EINVAL - an invalid parameter was specified.
1372 | *
1373 | */
1374 | int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor);
1375 |
1376 | /** @brief Return the cursor's transaction handle.
1377 | *
1378 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1379 | */
1380 | MDB_txn *mdb_cursor_txn(MDB_cursor *cursor);
1381 |
1382 | /** @brief Return the cursor's database handle.
1383 | *
1384 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1385 | */
1386 | MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor);
1387 |
1388 | /** @brief Retrieve by cursor.
1389 | *
1390 | * This function retrieves key/data pairs from the database. The address and length
1391 | * of the key are returned in the object to which \b key refers (except for the
1392 | * case of the #MDB_SET option, in which the \b key object is unchanged), and
1393 | * the address and length of the data are returned in the object to which \b data
1394 | * refers.
1395 | * See #mdb_get() for restrictions on using the output values.
1396 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1397 | * @param[in,out] key The key for a retrieved item
1398 | * @param[in,out] data The data of a retrieved item
1399 | * @param[in] op A cursor operation #MDB_cursor_op
1400 | * @return A non-zero error value on failure and 0 on success. Some possible
1401 | * errors are:
1402 | *
1403 | *
#MDB_NOTFOUND - no matching key found.
1404 | *
EINVAL - an invalid parameter was specified.
1405 | *
1406 | */
1407 | int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
1408 | MDB_cursor_op op);
1409 |
1410 | /** @brief Store by cursor.
1411 | *
1412 | * This function stores key/data pairs into the database.
1413 | * The cursor is positioned at the new item, or on failure usually near it.
1414 | * @note Earlier documentation incorrectly said errors would leave the
1415 | * state of the cursor unchanged.
1416 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1417 | * @param[in] key The key operated on.
1418 | * @param[in] data The data operated on.
1419 | * @param[in] flags Options for this operation. This parameter
1420 | * must be set to 0 or one of the values described here.
1421 | *
1422 | *
#MDB_CURRENT - replace the item at the current cursor position.
1423 | * The \b key parameter must still be provided, and must match it.
1424 | * If using sorted duplicates (#MDB_DUPSORT) the data item must still
1425 | * sort into the same place. This is intended to be used when the
1426 | * new data is the same size as the old. Otherwise it will simply
1427 | * perform a delete of the old record followed by an insert.
1428 | *
#MDB_NODUPDATA - enter the new key/data pair only if it does not
1429 | * already appear in the database. This flag may only be specified
1430 | * if the database was opened with #MDB_DUPSORT. The function will
1431 | * return #MDB_KEYEXIST if the key/data pair already appears in the
1432 | * database.
1433 | *
#MDB_NOOVERWRITE - enter the new key/data pair only if the key
1434 | * does not already appear in the database. The function will return
1435 | * #MDB_KEYEXIST if the key already appears in the database, even if
1436 | * the database supports duplicates (#MDB_DUPSORT).
1437 | *
#MDB_RESERVE - reserve space for data of the given size, but
1438 | * don't copy the given data. Instead, return a pointer to the
1439 | * reserved space, which the caller can fill in later. This saves
1440 | * an extra memcpy if the data is being generated later.
1441 | *
#MDB_APPEND - append the given key/data pair to the end of the
1442 | * database. No key comparisons are performed. This option allows
1443 | * fast bulk loading when keys are already known to be in the
1444 | * correct order. Loading unsorted keys with this flag will cause
1445 | * data corruption.
1446 | *
#MDB_APPENDDUP - as above, but for sorted dup data.
1447 | *
#MDB_MULTIPLE - store multiple contiguous data elements in a
1448 | * single request. This flag may only be specified if the database
1449 | * was opened with #MDB_DUPFIXED. The \b data argument must be an
1450 | * array of two MDB_vals. The mv_size of the first MDB_val must be
1451 | * the size of a single data element. The mv_data of the first MDB_val
1452 | * must point to the beginning of the array of contiguous data elements.
1453 | * The mv_size of the second MDB_val must be the count of the number
1454 | * of data elements to store. On return this field will be set to
1455 | * the count of the number of elements actually written. The mv_data
1456 | * of the second MDB_val is unused.
1457 | *
1458 | * @return A non-zero error value on failure and 0 on success. Some possible
1459 | * errors are:
1460 | *
1461 | *
#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
1462 | *
#MDB_TXN_FULL - the transaction has too many dirty pages.
1463 | *
EACCES - an attempt was made to write in a read-only transaction.
1464 | *
EINVAL - an invalid parameter was specified.
1465 | *
1466 | */
1467 | int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
1468 | unsigned int flags);
1469 |
1470 | /** @brief Delete current key/data pair
1471 | *
1472 | * This function deletes the key/data pair to which the cursor refers.
1473 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1474 | * @param[in] flags Options for this operation. This parameter
1475 | * must be set to 0 or one of the values described here.
1476 | *
1477 | *
#MDB_NODUPDATA - delete all of the data items for the current key.
1478 | * This flag may only be specified if the database was opened with #MDB_DUPSORT.
1479 | *
1480 | * @return A non-zero error value on failure and 0 on success. Some possible
1481 | * errors are:
1482 | *
1483 | *
EACCES - an attempt was made to write in a read-only transaction.
1484 | *
EINVAL - an invalid parameter was specified.
1485 | *
1486 | */
1487 | int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
1488 |
1489 | /** @brief Return count of duplicates for current key.
1490 | *
1491 | * This call is only valid on databases that support sorted duplicate
1492 | * data items #MDB_DUPSORT.
1493 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
1494 | * @param[out] countp Address where the count will be stored
1495 | * @return A non-zero error value on failure and 0 on success. Some possible
1496 | * errors are:
1497 | *
1498 | *
EINVAL - cursor is not initialized, or an invalid parameter was specified.
1499 | *
1500 | */
1501 | int mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
1502 |
1503 | /** @brief Compare two data items according to a particular database.
1504 | *
1505 | * This returns a comparison as if the two data items were keys in the
1506 | * specified database.
1507 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1508 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1509 | * @param[in] a The first item to compare
1510 | * @param[in] b The second item to compare
1511 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b
1512 | */
1513 | int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b);
1514 |
1515 | /** @brief Compare two data items according to a particular database.
1516 | *
1517 | * This returns a comparison as if the two items were data items of
1518 | * the specified database. The database must have the #MDB_DUPSORT flag.
1519 | * @param[in] txn A transaction handle returned by #mdb_txn_begin()
1520 | * @param[in] dbi A database handle returned by #mdb_dbi_open()
1521 | * @param[in] a The first item to compare
1522 | * @param[in] b The second item to compare
1523 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b
1524 | */
1525 | int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b);
1526 |
1527 | /** @brief A callback function used to print a message from the library.
1528 | *
1529 | * @param[in] msg The string to be printed.
1530 | * @param[in] ctx An arbitrary context pointer for the callback.
1531 | * @return < 0 on failure, >= 0 on success.
1532 | */
1533 | typedef int (MDB_msg_func)(const char *msg, void *ctx);
1534 |
1535 | /** @brief Dump the entries in the reader lock table.
1536 | *
1537 | * @param[in] env An environment handle returned by #mdb_env_create()
1538 | * @param[in] func A #MDB_msg_func function
1539 | * @param[in] ctx Anything the message function needs
1540 | * @return < 0 on failure, >= 0 on success.
1541 | */
1542 | int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx);
1543 |
1544 | /** @brief Check for stale entries in the reader lock table.
1545 | *
1546 | * @param[in] env An environment handle returned by #mdb_env_create()
1547 | * @param[out] dead Number of stale slots that were cleared
1548 | * @return 0 on success, non-zero on failure.
1549 | */
1550 | int mdb_reader_check(MDB_env *env, int *dead);
1551 | /** @} */
1552 |
1553 | #ifdef __cplusplus
1554 | }
1555 | #endif
1556 | /** @page tools LMDB Command Line Tools
1557 | The following describes the command line tools that are available for LMDB.
1558 | \li \ref mdb_copy_1
1559 | \li \ref mdb_dump_1
1560 | \li \ref mdb_load_1
1561 | \li \ref mdb_stat_1
1562 | */
1563 |
1564 | #endif /* _LMDB_H_ */
1565 |
--------------------------------------------------------------------------------
/deps/lmdb/midl.c:
--------------------------------------------------------------------------------
1 | /** @file midl.c
2 | * @brief ldap bdb back-end ID List functions */
3 | /* $OpenLDAP$ */
4 | /* This work is part of OpenLDAP Software .
5 | *
6 | * Copyright 2000-2015 The OpenLDAP Foundation.
7 | * All rights reserved.
8 | *
9 | * Redistribution and use in source and binary forms, with or without
10 | * modification, are permitted only as authorized by the OpenLDAP
11 | * Public License.
12 | *
13 | * A copy of this license is available in the file LICENSE in the
14 | * top-level directory of the distribution or, alternatively, at
15 | * .
16 | */
17 |
18 | #include
19 | #include
20 | #include
21 | #include
22 | #include
23 | #include "midl.h"
24 |
25 | /** @defgroup internal LMDB Internals
26 | * @{
27 | */
28 | /** @defgroup idls ID List Management
29 | * @{
30 | */
31 | #define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) )
32 |
33 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
34 | {
35 | /*
36 | * binary search of id in ids
37 | * if found, returns position of id
38 | * if not found, returns first position greater than id
39 | */
40 | unsigned base = 0;
41 | unsigned cursor = 1;
42 | int val = 0;
43 | unsigned n = ids[0];
44 |
45 | while( 0 < n ) {
46 | unsigned pivot = n >> 1;
47 | cursor = base + pivot + 1;
48 | val = CMP( ids[cursor], id );
49 |
50 | if( val < 0 ) {
51 | n = pivot;
52 |
53 | } else if ( val > 0 ) {
54 | base = cursor;
55 | n -= pivot + 1;
56 |
57 | } else {
58 | return cursor;
59 | }
60 | }
61 |
62 | if( val > 0 ) {
63 | ++cursor;
64 | }
65 | return cursor;
66 | }
67 |
68 | #if 0 /* superseded by append/sort */
69 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id )
70 | {
71 | unsigned x, i;
72 |
73 | x = mdb_midl_search( ids, id );
74 | assert( x > 0 );
75 |
76 | if( x < 1 ) {
77 | /* internal error */
78 | return -2;
79 | }
80 |
81 | if ( x <= ids[0] && ids[x] == id ) {
82 | /* duplicate */
83 | assert(0);
84 | return -1;
85 | }
86 |
87 | if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
88 | /* no room */
89 | --ids[0];
90 | return -2;
91 |
92 | } else {
93 | /* insert id */
94 | for (i=ids[0]; i>x; i--)
95 | ids[i] = ids[i-1];
96 | ids[x] = id;
97 | }
98 |
99 | return 0;
100 | }
101 | #endif
102 |
103 | MDB_IDL mdb_midl_alloc(int num)
104 | {
105 | MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
106 | if (ids) {
107 | *ids++ = num;
108 | *ids = 0;
109 | }
110 | return ids;
111 | }
112 |
113 | void mdb_midl_free(MDB_IDL ids)
114 | {
115 | if (ids)
116 | free(ids-1);
117 | }
118 |
119 | void mdb_midl_shrink( MDB_IDL *idp )
120 | {
121 | MDB_IDL ids = *idp;
122 | if (*(--ids) > MDB_IDL_UM_MAX &&
123 | (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID))))
124 | {
125 | *ids++ = MDB_IDL_UM_MAX;
126 | *idp = ids;
127 | }
128 | }
129 |
130 | static int mdb_midl_grow( MDB_IDL *idp, int num )
131 | {
132 | MDB_IDL idn = *idp-1;
133 | /* grow it */
134 | idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID));
135 | if (!idn)
136 | return ENOMEM;
137 | *idn++ += num;
138 | *idp = idn;
139 | return 0;
140 | }
141 |
142 | int mdb_midl_need( MDB_IDL *idp, unsigned num )
143 | {
144 | MDB_IDL ids = *idp;
145 | num += ids[0];
146 | if (num > ids[-1]) {
147 | num = (num + num/4 + (256 + 2)) & -256;
148 | if (!(ids = realloc(ids-1, num * sizeof(MDB_ID))))
149 | return ENOMEM;
150 | *ids++ = num - 2;
151 | *idp = ids;
152 | }
153 | return 0;
154 | }
155 |
156 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id )
157 | {
158 | MDB_IDL ids = *idp;
159 | /* Too big? */
160 | if (ids[0] >= ids[-1]) {
161 | if (mdb_midl_grow(idp, MDB_IDL_UM_MAX))
162 | return ENOMEM;
163 | ids = *idp;
164 | }
165 | ids[0]++;
166 | ids[ids[0]] = id;
167 | return 0;
168 | }
169 |
170 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app )
171 | {
172 | MDB_IDL ids = *idp;
173 | /* Too big? */
174 | if (ids[0] + app[0] >= ids[-1]) {
175 | if (mdb_midl_grow(idp, app[0]))
176 | return ENOMEM;
177 | ids = *idp;
178 | }
179 | memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID));
180 | ids[0] += app[0];
181 | return 0;
182 | }
183 |
184 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n )
185 | {
186 | MDB_ID *ids = *idp, len = ids[0];
187 | /* Too big? */
188 | if (len + n > ids[-1]) {
189 | if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX))
190 | return ENOMEM;
191 | ids = *idp;
192 | }
193 | ids[0] = len + n;
194 | ids += len;
195 | while (n)
196 | ids[n--] = id++;
197 | return 0;
198 | }
199 |
200 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge )
201 | {
202 | MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k;
203 | idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */
204 | old_id = idl[j];
205 | while (i) {
206 | merge_id = merge[i--];
207 | for (; old_id < merge_id; old_id = idl[--j])
208 | idl[k--] = old_id;
209 | idl[k--] = merge_id;
210 | }
211 | idl[0] = total;
212 | }
213 |
214 | /* Quicksort + Insertion sort for small arrays */
215 |
216 | #define SMALL 8
217 | #define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; }
218 |
219 | void
220 | mdb_midl_sort( MDB_IDL ids )
221 | {
222 | /* Max possible depth of int-indexed tree * 2 items/level */
223 | int istack[sizeof(int)*CHAR_BIT * 2];
224 | int i,j,k,l,ir,jstack;
225 | MDB_ID a, itmp;
226 |
227 | ir = (int)ids[0];
228 | l = 1;
229 | jstack = 0;
230 | for(;;) {
231 | if (ir - l < SMALL) { /* Insertion sort */
232 | for (j=l+1;j<=ir;j++) {
233 | a = ids[j];
234 | for (i=j-1;i>=1;i--) {
235 | if (ids[i] >= a) break;
236 | ids[i+1] = ids[i];
237 | }
238 | ids[i+1] = a;
239 | }
240 | if (jstack == 0) break;
241 | ir = istack[jstack--];
242 | l = istack[jstack--];
243 | } else {
244 | k = (l + ir) >> 1; /* Choose median of left, center, right */
245 | MIDL_SWAP(ids[k], ids[l+1]);
246 | if (ids[l] < ids[ir]) {
247 | MIDL_SWAP(ids[l], ids[ir]);
248 | }
249 | if (ids[l+1] < ids[ir]) {
250 | MIDL_SWAP(ids[l+1], ids[ir]);
251 | }
252 | if (ids[l] < ids[l+1]) {
253 | MIDL_SWAP(ids[l], ids[l+1]);
254 | }
255 | i = l+1;
256 | j = ir;
257 | a = ids[l+1];
258 | for(;;) {
259 | do i++; while(ids[i] > a);
260 | do j--; while(ids[j] < a);
261 | if (j < i) break;
262 | MIDL_SWAP(ids[i],ids[j]);
263 | }
264 | ids[l+1] = ids[j];
265 | ids[j] = a;
266 | jstack += 2;
267 | if (ir-i+1 >= j-l) {
268 | istack[jstack] = ir;
269 | istack[jstack-1] = i;
270 | ir = j-1;
271 | } else {
272 | istack[jstack] = j-1;
273 | istack[jstack-1] = l;
274 | l = i;
275 | }
276 | }
277 | }
278 | }
279 |
280 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id )
281 | {
282 | /*
283 | * binary search of id in ids
284 | * if found, returns position of id
285 | * if not found, returns first position greater than id
286 | */
287 | unsigned base = 0;
288 | unsigned cursor = 1;
289 | int val = 0;
290 | unsigned n = (unsigned)ids[0].mid;
291 |
292 | while( 0 < n ) {
293 | unsigned pivot = n >> 1;
294 | cursor = base + pivot + 1;
295 | val = CMP( id, ids[cursor].mid );
296 |
297 | if( val < 0 ) {
298 | n = pivot;
299 |
300 | } else if ( val > 0 ) {
301 | base = cursor;
302 | n -= pivot + 1;
303 |
304 | } else {
305 | return cursor;
306 | }
307 | }
308 |
309 | if( val > 0 ) {
310 | ++cursor;
311 | }
312 | return cursor;
313 | }
314 |
315 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id )
316 | {
317 | unsigned x, i;
318 |
319 | x = mdb_mid2l_search( ids, id->mid );
320 |
321 | if( x < 1 ) {
322 | /* internal error */
323 | return -2;
324 | }
325 |
326 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
327 | /* duplicate */
328 | return -1;
329 | }
330 |
331 | if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
332 | /* too big */
333 | return -2;
334 |
335 | } else {
336 | /* insert id */
337 | ids[0].mid++;
338 | for (i=(unsigned)ids[0].mid; i>x; i--)
339 | ids[i] = ids[i-1];
340 | ids[x] = *id;
341 | }
342 |
343 | return 0;
344 | }
345 |
346 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
347 | {
348 | /* Too big? */
349 | if (ids[0].mid >= MDB_IDL_UM_MAX) {
350 | return -2;
351 | }
352 | ids[0].mid++;
353 | ids[ids[0].mid] = *id;
354 | return 0;
355 | }
356 |
357 | /** @} */
358 | /** @} */
359 |
--------------------------------------------------------------------------------
/deps/lmdb/midl.h:
--------------------------------------------------------------------------------
1 | /** @file midl.h
2 | * @brief LMDB ID List header file.
3 | *
4 | * This file was originally part of back-bdb but has been
5 | * modified for use in libmdb. Most of the macros defined
6 | * in this file are unused, just left over from the original.
7 | *
8 | * This file is only used internally in libmdb and its definitions
9 | * are not exposed publicly.
10 | */
11 | /* $OpenLDAP$ */
12 | /* This work is part of OpenLDAP Software .
13 | *
14 | * Copyright 2000-2015 The OpenLDAP Foundation.
15 | * All rights reserved.
16 | *
17 | * Redistribution and use in source and binary forms, with or without
18 | * modification, are permitted only as authorized by the OpenLDAP
19 | * Public License.
20 | *
21 | * A copy of this license is available in the file LICENSE in the
22 | * top-level directory of the distribution or, alternatively, at
23 | * .
24 | */
25 |
26 | #ifndef _MDB_MIDL_H_
27 | #define _MDB_MIDL_H_
28 |
29 | #include
30 |
31 | #ifdef __cplusplus
32 | extern "C" {
33 | #endif
34 |
35 | /** @defgroup internal LMDB Internals
36 | * @{
37 | */
38 |
39 | /** @defgroup idls ID List Management
40 | * @{
41 | */
42 | /** A generic unsigned ID number. These were entryIDs in back-bdb.
43 | * Preferably it should have the same size as a pointer.
44 | */
45 | typedef size_t MDB_ID;
46 |
47 | /** An IDL is an ID List, a sorted array of IDs. The first
48 | * element of the array is a counter for how many actual
49 | * IDs are in the list. In the original back-bdb code, IDLs are
50 | * sorted in ascending order. For libmdb IDLs are sorted in
51 | * descending order.
52 | */
53 | typedef MDB_ID *MDB_IDL;
54 |
55 | /* IDL sizes - likely should be even bigger
56 | * limiting factors: sizeof(ID), thread stack size
57 | */
58 | #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
59 | #define MDB_IDL_DB_SIZE (1<getTopic(topic)), _name(name), _current(0), _lastOffset(0), _env(nullptr), _db(0), _rtxn(nullptr), _cursor(nullptr) {
7 | if (opt) {
8 | _opt = *opt;
9 | } else {
10 | /* Default opt */
11 | _opt.chunkSize = 1024 * 1024 * 1024;
12 | _opt.chunksToKeep = 8;
13 | }
14 |
15 | Txn txn(_topic->getEnv(), NULL);
16 | openHead(&txn);
17 | txn.commit();
18 | }
19 |
20 | Consumer::~Consumer() {
21 | closeCurrent();
22 | }
23 |
24 | void Consumer::pop(BatchType& result, size_t cnt) {
25 | result.reserve(cnt);
26 | bool shouldRotate = false;
27 |
28 | {
29 | Txn txn(_topic->getEnv(), NULL);
30 | mdb_txn_renew(_rtxn);
31 |
32 | uint64_t head = _topic->getConsumerHead(txn, _name);
33 | int rc = _cursor->gte(head);
34 |
35 | if (rc == 0) {
36 | uint64_t offset = 0;
37 | for (; rc == 0 && cnt > 0; --cnt) {
38 | offset = _cursor->key();
39 | const char* data = (const char*)_cursor->val().mv_data;
40 | size_t len = _cursor->val().mv_size;
41 | result.push_back(ItemType(offset, data, len));
42 | rc = _cursor->next();
43 | }
44 |
45 | if (offset > 0) {
46 | _topic->setConsumerHead(txn, _name, offset + 1);
47 | txn.commit();
48 | }
49 | } else {
50 | if (rc != MDB_NOTFOUND) cout << "Consumer seek error: " << mdb_strerror(rc) << endl;
51 |
52 | if (head < _topic->getProducerHead(txn)) {
53 | // shouldRotate = true;
54 | shouldRotate = _current != _topic->getProducerHeadFile(txn);
55 | }
56 | }
57 | }
58 |
59 | if (shouldRotate) {
60 | rotate();
61 | pop(result, cnt);
62 | }
63 | }
64 |
65 | void Consumer::openHead(Txn* txn) {
66 | _current = _topic->getConsumerHeadFile(*txn, _name, _current);
67 |
68 | char path[4096];
69 | _topic->getChunkFilePath(path, _current);
70 |
71 | mdb_env_create(&_env);
72 | mdb_env_set_mapsize(_env, _opt.chunkSize);
73 | int rc = mdb_env_open(_env, path, MDB_RDONLY | MDB_NOSYNC | MDB_NOSUBDIR, 0664);
74 |
75 | int cleared = 0;
76 | mdb_reader_check(_env, &cleared);
77 |
78 | if (rc != 0) {
79 | mdb_env_close(_env);
80 | _env = nullptr;
81 | printf("Consumer open error.\n%s\n", mdb_strerror(rc));
82 | return;
83 | }
84 |
85 | MDB_txn *otxn;
86 | mdb_txn_begin(_env, NULL, MDB_RDONLY, &otxn);
87 | mdb_dbi_open(otxn, NULL, MDB_CREATE, &_db);
88 | mdb_set_compare(otxn, _db, mdbIntCmp);
89 | mdb_txn_commit(otxn);
90 |
91 | mdb_txn_begin(_env, NULL, MDB_RDONLY, &_rtxn);
92 | _cursor = new MDBCursor(_db, _rtxn);
93 | mdb_txn_reset(_rtxn);
94 | }
95 |
96 | void Consumer::closeCurrent() {
97 | delete _cursor;
98 | mdb_txn_abort(_rtxn);
99 | mdb_dbi_close(_env, _db);
100 | mdb_env_close(_env);
101 | }
102 |
103 | void Consumer::rotate() {
104 | Txn txn(_topic->getEnv(), NULL);
105 | closeCurrent();
106 | openHead(&txn);
107 | txn.commit();
108 | }
109 |
--------------------------------------------------------------------------------
/src/consumer.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 |
3 | #include
4 | #include
5 |
6 | #include "env.h"
7 |
8 | class Topic;
9 |
10 | class Consumer {
11 | public:
12 | typedef std::tuple ItemType;
13 | typedef std::vector BatchType;
14 |
15 | public:
16 | Consumer(const std::string& root, const std::string& topic, const std::string& name, TopicOpt* opt);
17 | ~Consumer();
18 |
19 | private:
20 | Consumer(const Consumer&);
21 | Consumer& operator=(const Consumer&);
22 |
23 | public:
24 | void pop(BatchType& result, size_t cnt);
25 |
26 | private:
27 | void openHead(Txn* txn);
28 | void closeCurrent();
29 | void rotate();
30 |
31 | private:
32 | TopicOpt _opt;
33 | Topic* _topic;
34 | std::string _name;
35 |
36 | uint32_t _current;
37 | uint64_t _lastOffset;
38 | MDB_env* _env;
39 | MDB_dbi _db;
40 | MDB_txn* _rtxn;
41 | MDBCursor* _cursor;
42 | };
43 |
--------------------------------------------------------------------------------
/src/env.cc:
--------------------------------------------------------------------------------
1 | #include "topic.h"
2 | #include "env.h"
3 |
4 | using namespace std;
5 |
6 | Env* EnvManager::getEnv(const string& root, EnvOpt *opt) {
7 | static EnvManager instance;
8 |
9 | std::lock_guard guard(instance._mtx);
10 | EnvPtr& ptr = instance._envMap[root];
11 | if (!ptr.get()) ptr.reset(new Env(root, opt));
12 |
13 | return ptr.get();
14 | }
15 |
16 | Env::Env(const string& root, EnvOpt* opt) : _root(root), _env(nullptr) {
17 | mdb_env_create(&_env);
18 |
19 | if (opt) {
20 | mdb_env_set_mapsize(_env, opt->mapSize);
21 | mdb_env_set_maxdbs(_env, opt->maxTopicNum);
22 | } else {
23 | /* Default opt */
24 | mdb_env_set_mapsize(_env, 256 * 1024 * 1024);
25 | mdb_env_set_maxdbs(_env, 256);
26 | }
27 |
28 | string path = root + "/__meta__";
29 | int rc = mdb_env_open(_env, path.c_str(), MDB_NOSYNC | MDB_NOSUBDIR, 0664);
30 | if (rc != 0) {
31 | mdb_env_close(_env);
32 | _env = nullptr;
33 | printf("Env open error.\n%s\n", mdb_strerror(rc));
34 | return;
35 | }
36 |
37 | int cleared = 0;
38 | mdb_reader_check(_env, &cleared);
39 | }
40 |
41 | Env::~Env() {
42 | _topics.clear();
43 | if (_env) {
44 | mdb_env_close(_env);
45 | _env = nullptr;
46 | }
47 | }
48 |
49 | Topic* Env::getTopic(const string& name) {
50 | std::lock_guard guard(_mtx);
51 | TopicPtr& ptr = _topics[name];
52 | if (!ptr.get()) ptr.reset(new Topic(this, name));
53 |
54 | return ptr.get();
55 | }
56 |
--------------------------------------------------------------------------------
/src/env.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 |
3 | #include
4 | #include