├── .gitignore ├── LICENSE ├── README.md ├── bench_test.go ├── cursor.go ├── env.go ├── env_test.go ├── error_test.go ├── example_test.go ├── lmdb.h ├── mdb.c ├── mdb.go ├── mdb_test.go ├── midl.c ├── midl.h ├── txn.go ├── val.go └── val_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Ferenc Szalai 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Archiveing 2 | ========== 3 | 4 | This project is not actively maintianed anymore. Please look for its forks or alternative implementations for better supported version. 5 | 6 | 7 | gomdb 8 | ===== 9 | 10 | Go wrapper for OpenLDAP Lightning Memory-Mapped Database (LMDB). 11 | Read more about LMDB here: http://symas.com/mdb/ 12 | 13 | GoDoc available here: http://godoc.org/github.com/szferi/gomdb 14 | 15 | Build 16 | ======= 17 | 18 | `go get github.com/szferi/gomdb` 19 | 20 | There is no dependency on LMDB dynamic library. 21 | 22 | On FreeBSD 10, you must explicitly set `CC` (otherwise it will fail with a cryptic error), for example: 23 | 24 | `CC=clang go test -v` 25 | 26 | TODO 27 | ====== 28 | 29 | * write more documentation 30 | * write more unit test 31 | * benchmark 32 | * figure out how can you write go binding for `MDB_comp_func` and `MDB_rel_func` 33 | * Handle go `*Cursor` close with `txn.Commit` and `txn.Abort` transparently 34 | 35 | -------------------------------------------------------------------------------- /bench_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | crand "crypto/rand" 5 | "io/ioutil" 6 | "math/rand" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | // repeatedly put (overwrite) keys. 12 | func BenchmarkTxnPut(b *testing.B) { 13 | initRandSource(b) 14 | env, path := setupBenchDB(b) 15 | defer teardownBenchDB(b, env, path) 16 | 17 | dbi := openBenchDBI(b, env) 18 | 19 | var ps [][]byte 20 | 21 | rc := newRandSourceCursor() 22 | txn, err := env.BeginTxn(nil, 0) 23 | bMust(b, err, "starting transaction") 24 | for i := 0; i < benchDBNumKeys; i++ { 25 | k := makeBenchDBKey(&rc) 26 | v := makeBenchDBVal(&rc) 27 | err := txn.Put(dbi, k, v, 0) 28 | ps = append(ps, k, v) 29 | bTxnMust(b, txn, err, "putting data") 30 | } 31 | err = txn.Commit() 32 | bMust(b, err, "commiting transaction") 33 | 34 | txn, err = env.BeginTxn(nil, 0) 35 | b.ResetTimer() 36 | for i := 0; i < b.N; i++ { 37 | k := ps[rand.Intn(len(ps)/2)*2] 38 | v := makeBenchDBVal(&rc) 39 | err := txn.Put(dbi, k, v, 0) 40 | bTxnMust(b, txn, err, "putting data") 41 | } 42 | b.StopTimer() 43 | err = txn.Commit() 44 | bMust(b, err, "commiting transaction") 45 | } 46 | 47 | // repeatedly get random keys. 48 | func BenchmarkTxnGetRDONLY(b *testing.B) { 49 | initRandSource(b) 50 | env, path := setupBenchDB(b) 51 | defer teardownBenchDB(b, env, path) 52 | 53 | dbi := openBenchDBI(b, env) 54 | 55 | var ps [][]byte 56 | 57 | rc := newRandSourceCursor() 58 | txn, err := env.BeginTxn(nil, 0) 59 | bMust(b, err, "starting transaction") 60 | for i := 0; i < benchDBNumKeys; i++ { 61 | k := makeBenchDBKey(&rc) 62 | v := makeBenchDBVal(&rc) 63 | err := txn.Put(dbi, k, v, 0) 64 | ps = append(ps, k, v) 65 | bTxnMust(b, txn, err, "putting data") 66 | } 67 | err = txn.Commit() 68 | bMust(b, err, "commiting transaction") 69 | 70 | txn, err = env.BeginTxn(nil, RDONLY) 71 | bMust(b, err, "starting transaction") 72 | defer txn.Abort() 73 | b.ResetTimer() 74 | for i := 0; i < b.N; i++ { 75 | _, err := txn.Get(dbi, ps[rand.Intn(len(ps))]) 76 | if err == NotFound { 77 | continue 78 | } 79 | if err != nil { 80 | b.Fatalf("error getting data: %v", err) 81 | } 82 | } 83 | b.StopTimer() 84 | } 85 | 86 | // like BenchmarkTxnGetRDONLY, but txn.GetVal() is called instead. 87 | func BenchmarkTxnGetValRDONLY(b *testing.B) { 88 | initRandSource(b) 89 | env, path := setupBenchDB(b) 90 | defer teardownBenchDB(b, env, path) 91 | 92 | dbi := openBenchDBI(b, env) 93 | 94 | var ps [][]byte 95 | 96 | rc := newRandSourceCursor() 97 | txn, err := env.BeginTxn(nil, 0) 98 | bMust(b, err, "starting transaction") 99 | for i := 0; i < benchDBNumKeys; i++ { 100 | k := makeBenchDBKey(&rc) 101 | v := makeBenchDBVal(&rc) 102 | err := txn.Put(dbi, k, v, 0) 103 | ps = append(ps, k, v) 104 | bTxnMust(b, txn, err, "putting data") 105 | } 106 | err = txn.Commit() 107 | bMust(b, err, "commiting transaction") 108 | 109 | txn, err = env.BeginTxn(nil, RDONLY) 110 | bMust(b, err, "starting transaction") 111 | defer txn.Abort() 112 | b.ResetTimer() 113 | for i := 0; i < b.N; i++ { 114 | _, err := txn.GetVal(dbi, ps[rand.Intn(len(ps))]) 115 | if err == NotFound { 116 | continue 117 | } 118 | if err != nil { 119 | b.Fatalf("error getting data: %v", err) 120 | } 121 | } 122 | b.StopTimer() 123 | } 124 | 125 | // repeatedly scan all the values in a database. 126 | func BenchmarkCursorScanRDONLY(b *testing.B) { 127 | initRandSource(b) 128 | env, path := setupBenchDB(b) 129 | defer teardownBenchDB(b, env, path) 130 | 131 | dbi := openBenchDBI(b, env) 132 | 133 | var ps [][]byte 134 | 135 | rc := newRandSourceCursor() 136 | txn, err := env.BeginTxn(nil, 0) 137 | bMust(b, err, "starting transaction") 138 | for i := 0; i < benchDBNumKeys; i++ { 139 | k := makeBenchDBKey(&rc) 140 | v := makeBenchDBVal(&rc) 141 | err := txn.Put(dbi, k, v, 0) 142 | ps = append(ps, k, v) 143 | bTxnMust(b, txn, err, "putting data") 144 | } 145 | err = txn.Commit() 146 | bMust(b, err, "commiting transaction") 147 | 148 | txn, err = env.BeginTxn(nil, RDONLY) 149 | bMust(b, err, "starting transaction") 150 | defer txn.Abort() 151 | b.ResetTimer() 152 | for i := 0; i < b.N; i++ { 153 | func() { 154 | cur, err := txn.CursorOpen(dbi) 155 | bMust(b, err, "opening cursor") 156 | defer cur.Close() 157 | var count int64 158 | for { 159 | _, _, err := cur.Get(nil, nil, NEXT) 160 | if err == NotFound { 161 | return 162 | } 163 | if err != nil { 164 | b.Fatalf("error getting data: %v", err) 165 | } 166 | count++ 167 | } 168 | if count != benchDBNumKeys { 169 | b.Fatalf("unexpected number of keys: %d", count) 170 | } 171 | }() 172 | } 173 | b.StopTimer() 174 | } 175 | 176 | // like BenchmarkCursoreScanRDONLY, but cursor.GetVal() is called instead. 177 | func BenchmarkCursorScanValRDONLY(b *testing.B) { 178 | initRandSource(b) 179 | env, path := setupBenchDB(b) 180 | defer teardownBenchDB(b, env, path) 181 | 182 | dbi := openBenchDBI(b, env) 183 | 184 | var ps [][]byte 185 | 186 | rc := newRandSourceCursor() 187 | txn, err := env.BeginTxn(nil, 0) 188 | bMust(b, err, "starting transaction") 189 | for i := 0; i < benchDBNumKeys; i++ { 190 | k := makeBenchDBKey(&rc) 191 | v := makeBenchDBVal(&rc) 192 | err := txn.Put(dbi, k, v, 0) 193 | ps = append(ps, k, v) 194 | bTxnMust(b, txn, err, "putting data") 195 | } 196 | err = txn.Commit() 197 | bMust(b, err, "commiting transaction") 198 | 199 | txn, err = env.BeginTxn(nil, RDONLY) 200 | bMust(b, err, "starting transaction") 201 | defer txn.Abort() 202 | b.ResetTimer() 203 | for i := 0; i < b.N; i++ { 204 | func() { 205 | cur, err := txn.CursorOpen(dbi) 206 | bMust(b, err, "opening cursor") 207 | defer cur.Close() 208 | var count int64 209 | for { 210 | _, _, err := cur.GetVal(nil, nil, NEXT) 211 | if err == NotFound { 212 | return 213 | } 214 | if err != nil { 215 | b.Fatalf("error getting data: %v", err) 216 | } 217 | count++ 218 | } 219 | if count != benchDBNumKeys { 220 | b.Fatalf("unexpected number of keys: %d", count) 221 | } 222 | }() 223 | } 224 | b.StopTimer() 225 | } 226 | 227 | func setupBenchDB(b *testing.B) (*Env, string) { 228 | env, err := NewEnv() 229 | bMust(b, err, "creating env") 230 | err = env.SetMaxDBs(26) 231 | bMust(b, err, "setting max dbs") 232 | err = env.SetMapSize(1 << 30) // 1GB 233 | bMust(b, err, "sizing env") 234 | path, err := ioutil.TempDir("", "mdb_test-bench-") 235 | bMust(b, err, "creating temp directory") 236 | err = env.Open(path, 0, 0644) 237 | if err != nil { 238 | teardownBenchDB(b, env, path) 239 | } 240 | bMust(b, err, "opening database") 241 | return env, path 242 | } 243 | 244 | func openBenchDBI(b *testing.B, env *Env) DBI { 245 | txn, err := env.BeginTxn(nil, 0) 246 | bMust(b, err, "starting transaction") 247 | name := "benchmark" 248 | dbi, err := txn.DBIOpen(&name, CREATE) 249 | if err != nil { 250 | txn.Abort() 251 | b.Fatalf("error opening dbi: %v", err) 252 | } 253 | err = txn.Commit() 254 | bMust(b, err, "commiting transaction") 255 | return dbi 256 | } 257 | 258 | func teardownBenchDB(b *testing.B, env *Env, path string) { 259 | env.Close() 260 | os.RemoveAll(path) 261 | } 262 | 263 | func randBytes(n int) []byte { 264 | p := make([]byte, n) 265 | crand.Read(p) 266 | return p 267 | } 268 | 269 | func bMust(b *testing.B, err error, action string) { 270 | if err != nil { 271 | b.Fatalf("error %s: %v", action, err) 272 | } 273 | } 274 | 275 | func bTxnMust(b *testing.B, txn *Txn, err error, action string) { 276 | if err != nil { 277 | txn.Abort() 278 | b.Fatalf("error %s: %v", action, err) 279 | } 280 | } 281 | 282 | const randSourceSize = 500 << 20 // size of the 'entropy pool' for random byte generation. 283 | const benchDBNumKeys = 100000 // number of keys to store in benchmark databases 284 | const benchDBMaxKeyLen = 30 // maximum length for database keys (size is limited by MDB) 285 | const benchDBMaxValLen = 2000 // maximum lengh for database values 286 | 287 | func makeBenchDBKey(c *randSourceCursor) []byte { 288 | return c.NBytes(rand.Intn(benchDBMaxKeyLen) + 1) 289 | } 290 | 291 | func makeBenchDBVal(c *randSourceCursor) []byte { 292 | return c.NBytes(rand.Intn(benchDBMaxValLen) + 1) 293 | } 294 | 295 | // holds a bunch of random bytes so repeated generation af 'random' slices is 296 | // cheap. acts as a ring which can be read from (although doesn't implement io.Reader). 297 | var randSource [randSourceSize]byte 298 | 299 | func initRandSource(b *testing.B) { 300 | if randSource[0] == 0 && randSource[1] == 0 && randSource[2] == 0 && randSource[3] == 0 { 301 | b.Logf("initializing random source data") 302 | n, err := crand.Read(randSource[:]) 303 | bMust(b, err, "initializing random source") 304 | if n < len(randSource) { 305 | b.Fatalf("unable to read enough random source data %d", n) 306 | } 307 | } 308 | } 309 | 310 | // acts as a simple byte slice generator. 311 | type randSourceCursor int 312 | 313 | func newRandSourceCursor() randSourceCursor { 314 | i := rand.Intn(randSourceSize) 315 | return randSourceCursor(i) 316 | } 317 | 318 | func (c *randSourceCursor) NBytes(n int) []byte { 319 | i := int(*c) 320 | if n >= randSourceSize { 321 | panic("rand size too big") 322 | } 323 | *c = (*c + randSourceCursor(n)) % randSourceSize 324 | _n := i + n - randSourceSize 325 | if _n > 0 { 326 | p := make([]byte, n) 327 | m := copy(p, randSource[i:]) 328 | copy(p[m:], randSource[:]) 329 | return p 330 | } 331 | return randSource[i : i+n] 332 | } 333 | -------------------------------------------------------------------------------- /cursor.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | /* 4 | #cgo CFLAGS: -pthread -W -Wall -Wno-unused-parameter -Wbad-function-cast -O2 -g 5 | #cgo freebsd CFLAGS: -DMDB_DSYNC=O_SYNC 6 | #cgo openbsd CFLAGS: -DMDB_DSYNC=O_SYNC 7 | #cgo netbsd CFLAGS: -DMDB_DSYNC=O_SYNC 8 | #include 9 | #include 10 | #include "lmdb.h" 11 | */ 12 | import "C" 13 | 14 | import ( 15 | "errors" 16 | ) 17 | 18 | // MDB_cursor_op 19 | const ( 20 | FIRST = iota 21 | FIRST_DUP 22 | GET_BOTH 23 | GET_RANGE 24 | GET_CURRENT 25 | GET_MULTIPLE 26 | LAST 27 | LAST_DUP 28 | NEXT 29 | NEXT_DUP 30 | NEXT_MULTIPLE 31 | NEXT_NODUP 32 | PREV 33 | PREV_DUP 34 | PREV_NODUP 35 | SET 36 | SET_KEY 37 | SET_RANGE 38 | ) 39 | 40 | func (cursor *Cursor) Close() error { 41 | if cursor._cursor == nil { 42 | return errors.New("Cursor already closed") 43 | } 44 | C.mdb_cursor_close(cursor._cursor) 45 | cursor._cursor = nil 46 | return nil 47 | } 48 | 49 | func (cursor *Cursor) Txn() *Txn { 50 | var _txn *C.MDB_txn 51 | _txn = C.mdb_cursor_txn(cursor._cursor) 52 | if _txn != nil { 53 | return &Txn{_txn} 54 | } 55 | return nil 56 | } 57 | 58 | func (cursor *Cursor) DBI() DBI { 59 | var _dbi C.MDB_dbi 60 | _dbi = C.mdb_cursor_dbi(cursor._cursor) 61 | return DBI(_dbi) 62 | } 63 | 64 | // Retrieves the low-level MDB cursor. 65 | func (cursor *Cursor) MdbCursor() *C.MDB_cursor { 66 | return cursor._cursor 67 | } 68 | 69 | func (cursor *Cursor) Get(set_key, sval []byte, op uint) (key, val []byte, err error) { 70 | k, v, err := cursor.GetVal(set_key, sval, op) 71 | if err != nil { 72 | return nil, nil, err 73 | } 74 | return k.Bytes(), v.Bytes(), nil 75 | } 76 | 77 | func (cursor *Cursor) GetVal(key, val []byte, op uint) (Val, Val, error) { 78 | ckey := Wrap(key) 79 | cval := Wrap(val) 80 | ret := C.mdb_cursor_get(cursor._cursor, (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval), C.MDB_cursor_op(op)) 81 | return ckey, cval, errno(ret) 82 | } 83 | 84 | func (cursor *Cursor) Put(key, val []byte, flags uint) error { 85 | ckey := Wrap(key) 86 | cval := Wrap(val) 87 | ret := C.mdb_cursor_put(cursor._cursor, (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval), C.uint(flags)) 88 | return errno(ret) 89 | } 90 | 91 | func (cursor *Cursor) Del(flags uint) error { 92 | ret := C.mdb_cursor_del(cursor._cursor, C.uint(flags)) 93 | return errno(ret) 94 | } 95 | 96 | func (cursor *Cursor) Count() (uint64, error) { 97 | var _size C.size_t 98 | ret := C.mdb_cursor_count(cursor._cursor, &_size) 99 | if ret != SUCCESS { 100 | return 0, errno(ret) 101 | } 102 | return uint64(_size), nil 103 | } 104 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | /* 4 | #cgo CFLAGS: -pthread -W -Wall -Wno-unused-parameter -Wbad-function-cast -O2 -g 5 | #cgo freebsd CFLAGS: -DMDB_DSYNC=O_SYNC 6 | #cgo openbsd CFLAGS: -DMDB_DSYNC=O_SYNC 7 | #cgo netbsd CFLAGS: -DMDB_DSYNC=O_SYNC 8 | #include 9 | #include 10 | #include "lmdb.h" 11 | */ 12 | import "C" 13 | 14 | import ( 15 | "errors" 16 | "fmt" 17 | "syscall" 18 | "unsafe" 19 | ) 20 | 21 | const SUCCESS = C.MDB_SUCCESS 22 | 23 | // mdb_env Environment Flags 24 | const ( 25 | FIXEDMAP = C.MDB_FIXEDMAP // mmap at a fixed address (experimental) 26 | NOSUBDIR = C.MDB_NOSUBDIR // no environment directory 27 | NOSYNC = C.MDB_NOSYNC // don't fsync after commit 28 | RDONLY = C.MDB_RDONLY // read only 29 | NOMETASYNC = C.MDB_NOMETASYNC // don't fsync metapage after commit 30 | WRITEMAP = C.MDB_WRITEMAP // use writable mmap 31 | MAPASYNC = C.MDB_MAPASYNC // use asynchronous msync when MDB_WRITEMAP is use 32 | NOTLS = C.MDB_NOTLS // tie reader locktable slots to Txn objects instead of threads 33 | ) 34 | 35 | type DBI uint 36 | 37 | type Errno C.int 38 | 39 | // minimum and maximum values produced for the Errno type. syscall.Errnos of 40 | // other values may still be produced. 41 | const minErrno, maxErrno C.int = C.MDB_KEYEXIST, C.MDB_LAST_ERRCODE 42 | 43 | func (e Errno) Error() string { 44 | s := C.GoString(C.mdb_strerror(C.int(e))) 45 | if s == "" { 46 | return fmt.Sprint("mdb errno:", int(e)) 47 | } 48 | return s 49 | } 50 | 51 | // for tests that can't import C 52 | func _errno(ret int) error { 53 | return errno(C.int(ret)) 54 | } 55 | 56 | func errno(ret C.int) error { 57 | if ret == C.MDB_SUCCESS { 58 | return nil 59 | } 60 | if minErrno <= ret && ret <= maxErrno { 61 | return Errno(ret) 62 | } 63 | return syscall.Errno(ret) 64 | } 65 | 66 | // error codes 67 | const ( 68 | KeyExist = Errno(C.MDB_KEYEXIST) 69 | NotFound = Errno(C.MDB_NOTFOUND) 70 | PageNotFound = Errno(C.MDB_PAGE_NOTFOUND) 71 | Corrupted = Errno(C.MDB_CORRUPTED) 72 | Panic = Errno(C.MDB_PANIC) 73 | VersionMismatch = Errno(C.MDB_VERSION_MISMATCH) 74 | Invalid = Errno(C.MDB_INVALID) 75 | MapFull = Errno(C.MDB_MAP_FULL) 76 | DbsFull = Errno(C.MDB_DBS_FULL) 77 | ReadersFull = Errno(C.MDB_READERS_FULL) 78 | TlsFull = Errno(C.MDB_TLS_FULL) 79 | TxnFull = Errno(C.MDB_TXN_FULL) 80 | CursorFull = Errno(C.MDB_CURSOR_FULL) 81 | PageFull = Errno(C.MDB_PAGE_FULL) 82 | MapResized = Errno(C.MDB_MAP_RESIZED) 83 | Incompatibile = Errno(C.MDB_INCOMPATIBLE) 84 | ) 85 | 86 | func Version() string { 87 | var major, minor, patch *C.int 88 | ver_str := C.mdb_version(major, minor, patch) 89 | return C.GoString(ver_str) 90 | } 91 | 92 | // Env is opaque structure for a database environment. 93 | // A DB environment supports multiple databases, all residing in the 94 | // same shared-memory map. 95 | type Env struct { 96 | _env *C.MDB_env 97 | } 98 | 99 | // Create an MDB environment handle. 100 | func NewEnv() (*Env, error) { 101 | var _env *C.MDB_env 102 | ret := C.mdb_env_create(&_env) 103 | if ret != SUCCESS { 104 | return nil, errno(ret) 105 | } 106 | return &Env{_env}, nil 107 | } 108 | 109 | // Open an environment handle. If this function fails Close() must be called to discard the Env handle. 110 | func (env *Env) Open(path string, flags uint, mode uint) error { 111 | cpath := C.CString(path) 112 | defer C.free(unsafe.Pointer(cpath)) 113 | ret := C.mdb_env_open(env._env, cpath, C.uint(NOTLS|flags), C.mdb_mode_t(mode)) 114 | return errno(ret) 115 | } 116 | 117 | func (env *Env) Close() error { 118 | if env._env == nil { 119 | return errors.New("Environment already closed") 120 | } 121 | C.mdb_env_close(env._env) 122 | env._env = nil 123 | return nil 124 | } 125 | 126 | func (env *Env) Copy(path string) error { 127 | cpath := C.CString(path) 128 | defer C.free(unsafe.Pointer(cpath)) 129 | ret := C.mdb_env_copy(env._env, cpath) 130 | return errno(ret) 131 | } 132 | 133 | // Statistics for a database in the environment 134 | type Stat struct { 135 | PSize uint // Size of a database page. This is currently the same for all databases. 136 | Depth uint // Depth (height) of the B-tree 137 | BranchPages uint64 // Number of internal (non-leaf) pages 138 | LeafPages uint64 // Number of leaf pages 139 | OverflowPages uint64 // Number of overflow pages 140 | Entries uint64 // Number of data items 141 | } 142 | 143 | func (env *Env) Stat() (*Stat, error) { 144 | var _stat C.MDB_stat 145 | ret := C.mdb_env_stat(env._env, &_stat) 146 | if ret != SUCCESS { 147 | return nil, errno(ret) 148 | } 149 | stat := Stat{PSize: uint(_stat.ms_psize), 150 | Depth: uint(_stat.ms_depth), 151 | BranchPages: uint64(_stat.ms_branch_pages), 152 | LeafPages: uint64(_stat.ms_leaf_pages), 153 | OverflowPages: uint64(_stat.ms_overflow_pages), 154 | Entries: uint64(_stat.ms_entries)} 155 | return &stat, nil 156 | } 157 | 158 | type Info struct { 159 | MapSize uint64 // Size of the data memory map 160 | LastPNO uint64 // ID of the last used page 161 | LastTxnID uint64 // ID of the last committed transaction 162 | MaxReaders uint // maximum number of threads for the environment 163 | NumReaders uint // maximum number of threads used in the environment 164 | } 165 | 166 | func (env *Env) Info() (*Info, error) { 167 | var _info C.MDB_envinfo 168 | ret := C.mdb_env_info(env._env, &_info) 169 | if ret != SUCCESS { 170 | return nil, errno(ret) 171 | } 172 | info := Info{MapSize: uint64(_info.me_mapsize), 173 | LastPNO: uint64(_info.me_last_pgno), 174 | LastTxnID: uint64(_info.me_last_txnid), 175 | MaxReaders: uint(_info.me_maxreaders), 176 | NumReaders: uint(_info.me_numreaders)} 177 | return &info, nil 178 | } 179 | 180 | func (env *Env) Sync(force int) error { 181 | ret := C.mdb_env_sync(env._env, C.int(force)) 182 | return errno(ret) 183 | } 184 | 185 | func (env *Env) SetFlags(flags uint, onoff int) error { 186 | ret := C.mdb_env_set_flags(env._env, C.uint(flags), C.int(onoff)) 187 | return errno(ret) 188 | } 189 | 190 | func (env *Env) Flags() (uint, error) { 191 | var _flags C.uint 192 | ret := C.mdb_env_get_flags(env._env, &_flags) 193 | if ret != SUCCESS { 194 | return 0, errno(ret) 195 | } 196 | return uint(_flags), nil 197 | } 198 | 199 | func (env *Env) Path() (string, error) { 200 | var path string 201 | cpath := C.CString(path) 202 | defer C.free(unsafe.Pointer(cpath)) 203 | ret := C.mdb_env_get_path(env._env, &cpath) 204 | if ret != SUCCESS { 205 | return "", errno(ret) 206 | } 207 | return C.GoString(cpath), nil 208 | } 209 | 210 | func (env *Env) SetMapSize(size uint64) error { 211 | ret := C.mdb_env_set_mapsize(env._env, C.size_t(size)) 212 | return errno(ret) 213 | } 214 | 215 | func (env *Env) SetMaxReaders(size uint) error { 216 | ret := C.mdb_env_set_maxreaders(env._env, C.uint(size)) 217 | return errno(ret) 218 | } 219 | 220 | func (env *Env) SetMaxDBs(size DBI) error { 221 | ret := C.mdb_env_set_maxdbs(env._env, C.MDB_dbi(size)) 222 | return errno(ret) 223 | } 224 | 225 | func (env *Env) DBIClose(dbi DBI) { 226 | C.mdb_dbi_close(env._env, C.MDB_dbi(dbi)) 227 | } 228 | -------------------------------------------------------------------------------- /env_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestEnvOpen(t *testing.T) { 10 | env, err := NewEnv() 11 | if err != nil { 12 | t.Errorf("Cannot create enviroment: %s", err) 13 | } 14 | err = env.Open("adsjgfadsfjg", 0, 0664) 15 | if err == nil { 16 | t.Errorf("should not be able to open") 17 | } 18 | path, err := ioutil.TempDir("/tmp", "mdb_test") 19 | if err != nil { 20 | t.Errorf("Cannot create temporary directory") 21 | } 22 | err = os.MkdirAll(path, 0770) 23 | if err != nil { 24 | t.Errorf("Cannot create directory: %s", path) 25 | } 26 | err = env.Open(path, 0, 0664) 27 | if err != nil { 28 | t.Errorf("Cannot open environment: %s", err) 29 | } 30 | err = env.Close() 31 | if err != nil { 32 | t.Errorf("Error during close of environment: %s", err) 33 | } 34 | // clean up 35 | os.RemoveAll(path) 36 | } 37 | 38 | func setup(t *testing.T) *Env { 39 | env, err := NewEnv() 40 | if err != nil { 41 | t.Errorf("Cannot create enviroment: %s", err) 42 | } 43 | path, err := ioutil.TempDir("/tmp", "mdb_test") 44 | if err != nil { 45 | t.Errorf("Cannot create temporary directory") 46 | } 47 | err = os.MkdirAll(path, 0770) 48 | if err != nil { 49 | t.Errorf("Cannot create directory: %s", path) 50 | } 51 | err = env.Open(path, 0, 0664) 52 | if err != nil { 53 | t.Errorf("Cannot open environment: %s", err) 54 | } 55 | 56 | return env 57 | } 58 | 59 | func clean(env *Env, t *testing.T) { 60 | path, err := env.Path() 61 | if err != nil { 62 | t.Errorf("Cannot get path") 63 | } 64 | if path == "" { 65 | t.Errorf("Invalid path") 66 | } 67 | t.Logf("Env path: %s", path) 68 | err = env.Close() 69 | if err != nil { 70 | t.Errorf("Error during close of environment: %s", err) 71 | } 72 | // clean up 73 | os.RemoveAll(path) 74 | } 75 | 76 | func TestEnvCopy(t *testing.T) { 77 | env := setup(t) 78 | clean(env, t) 79 | } 80 | -------------------------------------------------------------------------------- /error_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | "testing" 5 | "syscall" 6 | ) 7 | 8 | func TestErrno(t *testing.T) { 9 | zeroerr := errno(0) 10 | if zeroerr != nil { 11 | t.Errorf("errno(0) != nil: %#v", zeroerr) 12 | } 13 | syserr := _errno(int(syscall.EINVAL)) 14 | if syserr != syscall.EINVAL { // fails if syserr is Errno(syscall.EINVAL) 15 | t.Errorf("errno(syscall.EINVAL) != syscall.EINVAL: %#v", syserr) 16 | } 17 | mdberr := _errno(int(KeyExist)) 18 | if mdberr != KeyExist { // fails if syserr is Errno(syscall.EINVAL) 19 | t.Errorf("errno(KeyExist) != KeyExist: %#v", syserr) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | // Most mdb functions/methods can return errors. This example ignores errors 10 | // for brevity. Real code should check all return values. 11 | func Example() { 12 | // create a directory to hold the database 13 | path, _ := ioutil.TempDir("", "mdb_test") 14 | defer os.RemoveAll(path) 15 | 16 | // open the db 17 | env, _ := NewEnv() 18 | env.SetMapSize(1 << 20) // max file size 19 | env.Open(path, 0, 0664) 20 | defer env.Close() 21 | txn, _ := env.BeginTxn(nil, 0) 22 | dbi, _ := txn.DBIOpen(nil, 0) 23 | defer env.DBIClose(dbi) 24 | txn.Commit() 25 | 26 | // write some data 27 | txn, _ = env.BeginTxn(nil, 0) 28 | num_entries := 5 29 | for i := 0; i < num_entries; i++ { 30 | key := fmt.Sprintf("Key-%d", i) 31 | val := fmt.Sprintf("Val-%d", i) 32 | txn.Put(dbi, []byte(key), []byte(val), 0) 33 | } 34 | txn.Commit() 35 | 36 | // inspect the database 37 | stat, _ := env.Stat() 38 | fmt.Println(stat.Entries) 39 | 40 | // scan the database 41 | txn, _ = env.BeginTxn(nil, RDONLY) 42 | defer txn.Abort() 43 | cursor, _ := txn.CursorOpen(dbi) 44 | defer cursor.Close() 45 | for { 46 | bkey, bval, err := cursor.Get(nil, nil, NEXT) 47 | if err == NotFound { 48 | break 49 | } 50 | if err != nil { 51 | panic(err) 52 | } 53 | fmt.Printf("%s: %s\n", bkey, bval) 54 | } 55 | 56 | // random access 57 | bval, _ := txn.Get(dbi, []byte("Key-3")) 58 | fmt.Println(string(bval)) 59 | 60 | // Output: 61 | // 5 62 | // Key-0: Val-0 63 | // Key-1: Val-1 64 | // Key-2: Val-2 65 | // Key-3: Val-3 66 | // Key-4: Val-4 67 | // Val-3 68 | } 69 | -------------------------------------------------------------------------------- /lmdb.h: -------------------------------------------------------------------------------- 1 | /** @file lmdb.h 2 | * @brief Lightning memory-mapped database library 3 | * 4 | * @mainpage Lightning Memory-Mapped Database Manager (LMDB) 5 | * 6 | * @section intro_sec Introduction 7 | * LMDB is a Btree-based database management library modeled loosely on the 8 | * BerkeleyDB API, but much simplified. The entire database is exposed 9 | * in a memory map, and all data fetches return data directly 10 | * from the mapped memory, so no malloc's or memcpy's occur during 11 | * data fetches. As such, the library is extremely simple because it 12 | * requires no page caching layer of its own, and it is extremely high 13 | * performance and memory-efficient. It is also fully transactional with 14 | * full ACID semantics, and when the memory map is read-only, the 15 | * database integrity cannot be corrupted by stray pointer writes from 16 | * application code. 17 | * 18 | * The library is fully thread-aware and supports concurrent read/write 19 | * access from multiple processes and threads. Data pages use a copy-on- 20 | * write strategy so no active data pages are ever overwritten, which 21 | * also provides resistance to corruption and eliminates the need of any 22 | * special recovery procedures after a system crash. Writes are fully 23 | * serialized; only one write transaction may be active at a time, which 24 | * guarantees that writers can never deadlock. The database structure is 25 | * multi-versioned so readers run with no locks; writers cannot block 26 | * readers, and readers don't block writers. 27 | * 28 | * Unlike other well-known database mechanisms which use either write-ahead 29 | * transaction logs or append-only data writes, LMDB requires no maintenance 30 | * during operation. Both write-ahead loggers and append-only databases 31 | * require periodic checkpointing and/or compaction of their log or database 32 | * files otherwise they grow without bound. LMDB tracks free pages within 33 | * the database and re-uses them for new write operations, so the database 34 | * size does not grow without bound in normal use. 35 | * 36 | * The memory map can be used as a read-only or read-write map. It is 37 | * read-only by default as this provides total immunity to corruption. 38 | * Using read-write mode offers much higher write performance, but adds 39 | * the possibility for stray application writes thru pointers to silently 40 | * corrupt the database. Of course if your application code is known to 41 | * be bug-free (...) then this is not an issue. 42 | * 43 | * @section caveats_sec Caveats 44 | * Troubleshooting the lock file, plus semaphores on BSD systems: 45 | * 46 | * - A broken lockfile can cause sync issues. 47 | * Stale reader transactions left behind by an aborted program 48 | * cause further writes to grow the database quickly, and 49 | * stale locks can block further operation. 50 | * 51 | * Fix: Check for stale readers periodically, using the 52 | * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just 53 | * make all programs using the database close it; the lockfile 54 | * is always reset on first open of the environment. 55 | * 56 | * - On BSD systems or others configured with MDB_USE_POSIX_SEM, 57 | * startup can fail due to semaphores owned by another userid. 58 | * 59 | * Fix: Open and close the database as the user which owns the 60 | * semaphores (likely last user) or as root, while no other 61 | * process is using the database. 62 | * 63 | * Restrictions/caveats (in addition to those listed for some functions): 64 | * 65 | * - Only the database owner should normally use the database on 66 | * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. 67 | * Multiple users can cause startup to fail later, as noted above. 68 | * 69 | * - There is normally no pure read-only mode, since readers need write 70 | * access to locks and lock file. Exceptions: On read-only filesystems 71 | * or with the #MDB_NOLOCK flag described under #mdb_env_open(). 72 | * 73 | * - By default, in versions before 0.9.10, unused portions of the data 74 | * file might receive garbage data from memory freed by other code. 75 | * (This does not happen when using the #MDB_WRITEMAP flag.) As of 76 | * 0.9.10 the default behavior is to initialize such memory before 77 | * writing to the data file. Since there may be a slight performance 78 | * cost due to this initialization, applications may disable it using 79 | * the #MDB_NOMEMINIT flag. Applications handling sensitive data 80 | * which must not be written should not use this flag. This flag is 81 | * irrelevant when using #MDB_WRITEMAP. 82 | * 83 | * - A thread can only use one transaction at a time, plus any child 84 | * transactions. Each transaction belongs to one thread. See below. 85 | * The #MDB_NOTLS flag changes this for read-only transactions. 86 | * 87 | * - Use an MDB_env* in the process which opened it, without fork()ing. 88 | * 89 | * - Do not have open an LMDB database twice in the same process at 90 | * the same time. Not even from a plain open() call - close()ing it 91 | * breaks flock() advisory locking. 92 | * 93 | * - Avoid long-lived transactions. Read transactions prevent 94 | * reuse of pages freed by newer write transactions, thus the 95 | * database can grow quickly. Write transactions prevent 96 | * other write transactions, since writes are serialized. 97 | * 98 | * - Avoid suspending a process with active transactions. These 99 | * would then be "long-lived" as above. Also read transactions 100 | * suspended when writers commit could sometimes see wrong data. 101 | * 102 | * ...when several processes can use a database concurrently: 103 | * 104 | * - Avoid aborting a process with an active transaction. 105 | * The transaction becomes "long-lived" as above until a check 106 | * for stale readers is performed or the lockfile is reset, 107 | * since the process may not remove it from the lockfile. 108 | * 109 | * - If you do that anyway, do a periodic check for stale readers. Or 110 | * close the environment once in a while, so the lockfile can get reset. 111 | * 112 | * - Do not use LMDB databases on remote filesystems, even between 113 | * processes on the same host. This breaks flock() on some OSes, 114 | * possibly memory map sync, and certainly sync between programs 115 | * on different hosts. 116 | * 117 | * - Opening a database can fail if another process is opening or 118 | * closing it at exactly the same time. 119 | * 120 | * @author Howard Chu, Symas Corporation. 121 | * 122 | * @copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved. 123 | * 124 | * Redistribution and use in source and binary forms, with or without 125 | * modification, are permitted only as authorized by the OpenLDAP 126 | * Public License. 127 | * 128 | * A copy of this license is available in the file LICENSE in the 129 | * top-level directory of the distribution or, alternatively, at 130 | * . 131 | * 132 | * @par Derived From: 133 | * This code is derived from btree.c written by Martin Hedenfalk. 134 | * 135 | * Copyright (c) 2009, 2010 Martin Hedenfalk 136 | * 137 | * Permission to use, copy, modify, and distribute this software for any 138 | * purpose with or without fee is hereby granted, provided that the above 139 | * copyright notice and this permission notice appear in all copies. 140 | * 141 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 142 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 143 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 144 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 145 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 146 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 147 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 148 | */ 149 | #ifndef _LMDB_H_ 150 | #define _LMDB_H_ 151 | 152 | #include 153 | 154 | #ifdef __cplusplus 155 | extern "C" { 156 | #endif 157 | 158 | /** Unix permissions for creating files, or dummy definition for Windows */ 159 | #ifdef _MSC_VER 160 | typedef int mdb_mode_t; 161 | #else 162 | typedef mode_t mdb_mode_t; 163 | #endif 164 | 165 | /** An abstraction for a file handle. 166 | * On POSIX systems file handles are small integers. On Windows 167 | * they're opaque pointers. 168 | */ 169 | #ifdef _WIN32 170 | typedef void *mdb_filehandle_t; 171 | #else 172 | typedef int mdb_filehandle_t; 173 | #endif 174 | 175 | /** @defgroup mdb LMDB API 176 | * @{ 177 | * @brief OpenLDAP Lightning Memory-Mapped Database Manager 178 | */ 179 | /** @defgroup Version Version Macros 180 | * @{ 181 | */ 182 | /** Library major version */ 183 | #define MDB_VERSION_MAJOR 0 184 | /** Library minor version */ 185 | #define MDB_VERSION_MINOR 9 186 | /** Library patch version */ 187 | #define MDB_VERSION_PATCH 14 188 | 189 | /** Combine args a,b,c into a single integer for easy version comparisons */ 190 | #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) 191 | 192 | /** The full library version as a single integer */ 193 | #define MDB_VERSION_FULL \ 194 | MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) 195 | 196 | /** The release date of this library version */ 197 | #define MDB_VERSION_DATE "July 24, 2014" 198 | 199 | /** A stringifier for the version info */ 200 | #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" 201 | 202 | /** A helper for the stringifier macro */ 203 | #define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) 204 | 205 | /** The full library version as a C string */ 206 | #define MDB_VERSION_STRING \ 207 | MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) 208 | /** @} */ 209 | 210 | /** @brief Opaque structure for a database environment. 211 | * 212 | * A DB environment supports multiple databases, all residing in the same 213 | * shared-memory map. 214 | */ 215 | typedef struct MDB_env MDB_env; 216 | 217 | /** @brief Opaque structure for a transaction handle. 218 | * 219 | * All database operations require a transaction handle. Transactions may be 220 | * read-only or read-write. 221 | */ 222 | typedef struct MDB_txn MDB_txn; 223 | 224 | /** @brief A handle for an individual database in the DB environment. */ 225 | typedef unsigned int MDB_dbi; 226 | 227 | /** @brief Opaque structure for navigating through a database */ 228 | typedef struct MDB_cursor MDB_cursor; 229 | 230 | /** @brief Generic structure used for passing keys and data in and out 231 | * of the database. 232 | * 233 | * Values returned from the database are valid only until a subsequent 234 | * update operation, or the end of the transaction. Do not modify or 235 | * free them, they commonly point into the database itself. 236 | * 237 | * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. 238 | * The same applies to data sizes in databases with the #MDB_DUPSORT flag. 239 | * Other data items can in theory be from 0 to 0xffffffff bytes long. 240 | */ 241 | typedef struct MDB_val { 242 | size_t mv_size; /**< size of the data item */ 243 | void *mv_data; /**< address of the data item */ 244 | } MDB_val; 245 | 246 | /** @brief A callback function used to compare two keys in a database */ 247 | typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); 248 | 249 | /** @brief A callback function used to relocate a position-dependent data item 250 | * in a fixed-address database. 251 | * 252 | * The \b newptr gives the item's desired address in 253 | * the memory map, and \b oldptr gives its previous address. The item's actual 254 | * data resides at the address in \b item. This callback is expected to walk 255 | * through the fields of the record in \b item and modify any 256 | * values based at the \b oldptr address to be relative to the \b newptr address. 257 | * @param[in,out] item The item that is to be relocated. 258 | * @param[in] oldptr The previous address. 259 | * @param[in] newptr The new address to relocate to. 260 | * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). 261 | * @todo This feature is currently unimplemented. 262 | */ 263 | typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); 264 | 265 | /** @defgroup mdb_env Environment Flags 266 | * @{ 267 | */ 268 | /** mmap at a fixed address (experimental) */ 269 | #define MDB_FIXEDMAP 0x01 270 | /** no environment directory */ 271 | #define MDB_NOSUBDIR 0x4000 272 | /** don't fsync after commit */ 273 | #define MDB_NOSYNC 0x10000 274 | /** read only */ 275 | #define MDB_RDONLY 0x20000 276 | /** don't fsync metapage after commit */ 277 | #define MDB_NOMETASYNC 0x40000 278 | /** use writable mmap */ 279 | #define MDB_WRITEMAP 0x80000 280 | /** use asynchronous msync when #MDB_WRITEMAP is used */ 281 | #define MDB_MAPASYNC 0x100000 282 | /** tie reader locktable slots to #MDB_txn objects instead of to threads */ 283 | #define MDB_NOTLS 0x200000 284 | /** don't do any locking, caller must manage their own locks */ 285 | #define MDB_NOLOCK 0x400000 286 | /** don't do readahead (no effect on Windows) */ 287 | #define MDB_NORDAHEAD 0x800000 288 | /** don't initialize malloc'd memory before writing to datafile */ 289 | #define MDB_NOMEMINIT 0x1000000 290 | /** @} */ 291 | 292 | /** @defgroup mdb_dbi_open Database Flags 293 | * @{ 294 | */ 295 | /** use reverse string keys */ 296 | #define MDB_REVERSEKEY 0x02 297 | /** use sorted duplicates */ 298 | #define MDB_DUPSORT 0x04 299 | /** numeric keys in native byte order. 300 | * The keys must all be of the same size. */ 301 | #define MDB_INTEGERKEY 0x08 302 | /** with #MDB_DUPSORT, sorted dup items have fixed size */ 303 | #define MDB_DUPFIXED 0x10 304 | /** with #MDB_DUPSORT, dups are numeric in native byte order */ 305 | #define MDB_INTEGERDUP 0x20 306 | /** with #MDB_DUPSORT, use reverse string dups */ 307 | #define MDB_REVERSEDUP 0x40 308 | /** create DB if not already existing */ 309 | #define MDB_CREATE 0x40000 310 | /** @} */ 311 | 312 | /** @defgroup mdb_put Write Flags 313 | * @{ 314 | */ 315 | /** For put: Don't write if the key already exists. */ 316 | #define MDB_NOOVERWRITE 0x10 317 | /** Only for #MDB_DUPSORT
318 | * For put: don't write if the key and data pair already exist.
319 | * For mdb_cursor_del: remove all duplicate data items. 320 | */ 321 | #define MDB_NODUPDATA 0x20 322 | /** For mdb_cursor_put: overwrite the current key/data pair */ 323 | #define MDB_CURRENT 0x40 324 | /** For put: Just reserve space for data, don't copy it. Return a 325 | * pointer to the reserved space. 326 | */ 327 | #define MDB_RESERVE 0x10000 328 | /** Data is being appended, don't split full pages. */ 329 | #define MDB_APPEND 0x20000 330 | /** Duplicate data is being appended, don't split full pages. */ 331 | #define MDB_APPENDDUP 0x40000 332 | /** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ 333 | #define MDB_MULTIPLE 0x80000 334 | /* @} */ 335 | 336 | /** @defgroup mdb_copy Copy Flags 337 | * @{ 338 | */ 339 | /** Compacting copy: Omit free space from copy, and renumber all 340 | * pages sequentially. 341 | */ 342 | #define MDB_CP_COMPACT 0x01 343 | /* @} */ 344 | 345 | /** @brief Cursor Get operations. 346 | * 347 | * This is the set of all operations for retrieving data 348 | * using a cursor. 349 | */ 350 | typedef enum MDB_cursor_op { 351 | MDB_FIRST, /**< Position at first key/data item */ 352 | MDB_FIRST_DUP, /**< Position at first data item of current key. 353 | Only for #MDB_DUPSORT */ 354 | MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ 355 | MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ 356 | MDB_GET_CURRENT, /**< Return key/data at current cursor position */ 357 | MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items 358 | from current cursor position. Move cursor to prepare 359 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ 360 | MDB_LAST, /**< Position at last key/data item */ 361 | MDB_LAST_DUP, /**< Position at last data item of current key. 362 | Only for #MDB_DUPSORT */ 363 | MDB_NEXT, /**< Position at next data item */ 364 | MDB_NEXT_DUP, /**< Position at next data item of current key. 365 | Only for #MDB_DUPSORT */ 366 | MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items 367 | from next cursor position. Move cursor to prepare 368 | for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ 369 | MDB_NEXT_NODUP, /**< Position at first data item of next key */ 370 | MDB_PREV, /**< Position at previous data item */ 371 | MDB_PREV_DUP, /**< Position at previous data item of current key. 372 | Only for #MDB_DUPSORT */ 373 | MDB_PREV_NODUP, /**< Position at last data item of previous key */ 374 | MDB_SET, /**< Position at specified key */ 375 | MDB_SET_KEY, /**< Position at specified key, return key + data */ 376 | MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ 377 | } MDB_cursor_op; 378 | 379 | /** @defgroup errors Return Codes 380 | * 381 | * BerkeleyDB uses -30800 to -30999, we'll go under them 382 | * @{ 383 | */ 384 | /** Successful result */ 385 | #define MDB_SUCCESS 0 386 | /** key/data pair already exists */ 387 | #define MDB_KEYEXIST (-30799) 388 | /** key/data pair not found (EOF) */ 389 | #define MDB_NOTFOUND (-30798) 390 | /** Requested page not found - this usually indicates corruption */ 391 | #define MDB_PAGE_NOTFOUND (-30797) 392 | /** Located page was wrong type */ 393 | #define MDB_CORRUPTED (-30796) 394 | /** Update of meta page failed, probably I/O error */ 395 | #define MDB_PANIC (-30795) 396 | /** Environment version mismatch */ 397 | #define MDB_VERSION_MISMATCH (-30794) 398 | /** File is not a valid LMDB file */ 399 | #define MDB_INVALID (-30793) 400 | /** Environment mapsize reached */ 401 | #define MDB_MAP_FULL (-30792) 402 | /** Environment maxdbs reached */ 403 | #define MDB_DBS_FULL (-30791) 404 | /** Environment maxreaders reached */ 405 | #define MDB_READERS_FULL (-30790) 406 | /** Too many TLS keys in use - Windows only */ 407 | #define MDB_TLS_FULL (-30789) 408 | /** Txn has too many dirty pages */ 409 | #define MDB_TXN_FULL (-30788) 410 | /** Cursor stack too deep - internal error */ 411 | #define MDB_CURSOR_FULL (-30787) 412 | /** Page has not enough space - internal error */ 413 | #define MDB_PAGE_FULL (-30786) 414 | /** Database contents grew beyond environment mapsize */ 415 | #define MDB_MAP_RESIZED (-30785) 416 | /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ 417 | #define MDB_INCOMPATIBLE (-30784) 418 | /** Invalid reuse of reader locktable slot */ 419 | #define MDB_BAD_RSLOT (-30783) 420 | /** Transaction cannot recover - it must be aborted */ 421 | #define MDB_BAD_TXN (-30782) 422 | /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ 423 | #define MDB_BAD_VALSIZE (-30781) 424 | /** The specified DBI was changed unexpectedly */ 425 | #define MDB_BAD_DBI (-30780) 426 | /** The last defined error code */ 427 | #define MDB_LAST_ERRCODE MDB_BAD_DBI 428 | /** @} */ 429 | 430 | /** @brief Statistics for a database in the environment */ 431 | typedef struct MDB_stat { 432 | unsigned int ms_psize; /**< Size of a database page. 433 | This is currently the same for all databases. */ 434 | unsigned int ms_depth; /**< Depth (height) of the B-tree */ 435 | size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ 436 | size_t ms_leaf_pages; /**< Number of leaf pages */ 437 | size_t ms_overflow_pages; /**< Number of overflow pages */ 438 | size_t ms_entries; /**< Number of data items */ 439 | } MDB_stat; 440 | 441 | /** @brief Information about the environment */ 442 | typedef struct MDB_envinfo { 443 | void *me_mapaddr; /**< Address of map, if fixed */ 444 | size_t me_mapsize; /**< Size of the data memory map */ 445 | size_t me_last_pgno; /**< ID of the last used page */ 446 | size_t me_last_txnid; /**< ID of the last committed transaction */ 447 | unsigned int me_maxreaders; /**< max reader slots in the environment */ 448 | unsigned int me_numreaders; /**< max reader slots used in the environment */ 449 | } MDB_envinfo; 450 | 451 | /** @brief Return the LMDB library version information. 452 | * 453 | * @param[out] major if non-NULL, the library major version number is copied here 454 | * @param[out] minor if non-NULL, the library minor version number is copied here 455 | * @param[out] patch if non-NULL, the library patch version number is copied here 456 | * @retval "version string" The library version as a string 457 | */ 458 | char *mdb_version(int *major, int *minor, int *patch); 459 | 460 | /** @brief Return a string describing a given error code. 461 | * 462 | * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) 463 | * function. If the error code is greater than or equal to 0, then the string 464 | * returned by the system function strerror(3) is returned. If the error code 465 | * is less than 0, an error string corresponding to the LMDB library error is 466 | * returned. See @ref errors for a list of LMDB-specific error codes. 467 | * @param[in] err The error code 468 | * @retval "error message" The description of the error 469 | */ 470 | char *mdb_strerror(int err); 471 | 472 | /** @brief Create an LMDB environment handle. 473 | * 474 | * This function allocates memory for a #MDB_env structure. To release 475 | * the allocated memory and discard the handle, call #mdb_env_close(). 476 | * Before the handle may be used, it must be opened using #mdb_env_open(). 477 | * Various other options may also need to be set before opening the handle, 478 | * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), 479 | * depending on usage requirements. 480 | * @param[out] env The address where the new handle will be stored 481 | * @return A non-zero error value on failure and 0 on success. 482 | */ 483 | int mdb_env_create(MDB_env **env); 484 | 485 | /** @brief Open an environment handle. 486 | * 487 | * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. 488 | * @param[in] env An environment handle returned by #mdb_env_create() 489 | * @param[in] path The directory in which the database files reside. This 490 | * directory must already exist and be writable. 491 | * @param[in] flags Special options for this environment. This parameter 492 | * must be set to 0 or by bitwise OR'ing together one or more of the 493 | * values described here. 494 | * Flags set by mdb_env_set_flags() are also used. 495 | *
    496 | *
  • #MDB_FIXEDMAP 497 | * use a fixed address for the mmap region. This flag must be specified 498 | * when creating the environment, and is stored persistently in the environment. 499 | * If successful, the memory map will always reside at the same virtual address 500 | * and pointers used to reference data items in the database will be constant 501 | * across multiple invocations. This option may not always work, depending on 502 | * how the operating system has allocated memory to shared libraries and other uses. 503 | * The feature is highly experimental. 504 | *
  • #MDB_NOSUBDIR 505 | * By default, LMDB creates its environment in a directory whose 506 | * pathname is given in \b path, and creates its data and lock files 507 | * under that directory. With this option, \b path is used as-is for 508 | * the database main data file. The database lock file is the \b path 509 | * with "-lock" appended. 510 | *
  • #MDB_RDONLY 511 | * Open the environment in read-only mode. No write operations will be 512 | * allowed. LMDB will still modify the lock file - except on read-only 513 | * filesystems, where LMDB does not use locks. 514 | *
  • #MDB_WRITEMAP 515 | * Use a writeable memory map unless MDB_RDONLY is set. This is faster 516 | * and uses fewer mallocs, but loses protection from application bugs 517 | * like wild pointer writes and other bad updates into the database. 518 | * Incompatible with nested transactions. 519 | * Processes with and without MDB_WRITEMAP on the same environment do 520 | * not cooperate well. 521 | *
  • #MDB_NOMETASYNC 522 | * Flush system buffers to disk only once per transaction, omit the 523 | * metadata flush. Defer that until the system flushes files to disk, 524 | * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization 525 | * maintains database integrity, but a system crash may undo the last 526 | * committed transaction. I.e. it preserves the ACI (atomicity, 527 | * consistency, isolation) but not D (durability) database property. 528 | * This flag may be changed at any time using #mdb_env_set_flags(). 529 | *
  • #MDB_NOSYNC 530 | * Don't flush system buffers to disk when committing a transaction. 531 | * This optimization means a system crash can corrupt the database or 532 | * lose the last transactions if buffers are not yet flushed to disk. 533 | * The risk is governed by how often the system flushes dirty buffers 534 | * to disk and how often #mdb_env_sync() is called. However, if the 535 | * filesystem preserves write order and the #MDB_WRITEMAP flag is not 536 | * used, transactions exhibit ACI (atomicity, consistency, isolation) 537 | * properties and only lose D (durability). I.e. database integrity 538 | * is maintained, but a system crash may undo the final transactions. 539 | * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no 540 | * hint for when to write transactions to disk, unless #mdb_env_sync() 541 | * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. 542 | * This flag may be changed at any time using #mdb_env_set_flags(). 543 | *
  • #MDB_MAPASYNC 544 | * When using #MDB_WRITEMAP, use asynchronous flushes to disk. 545 | * As with #MDB_NOSYNC, a system crash can then corrupt the 546 | * database or lose the last transactions. Calling #mdb_env_sync() 547 | * ensures on-disk database integrity until next commit. 548 | * This flag may be changed at any time using #mdb_env_set_flags(). 549 | *
  • #MDB_NOTLS 550 | * Don't use Thread-Local Storage. Tie reader locktable slots to 551 | * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps 552 | * the slot reseved for the #MDB_txn object. A thread may use parallel 553 | * read-only transactions. A read-only transaction may span threads if 554 | * the user synchronizes its use. Applications that multiplex many 555 | * user threads over individual OS threads need this option. Such an 556 | * application must also serialize the write transactions in an OS 557 | * thread, since LMDB's write locking is unaware of the user threads. 558 | *
  • #MDB_NOLOCK 559 | * Don't do any locking. If concurrent access is anticipated, the 560 | * caller must manage all concurrency itself. For proper operation 561 | * the caller must enforce single-writer semantics, and must ensure 562 | * that no readers are using old transactions while a writer is 563 | * active. The simplest approach is to use an exclusive lock so that 564 | * no readers may be active at all when a writer begins. 565 | *
  • #MDB_NORDAHEAD 566 | * Turn off readahead. Most operating systems perform readahead on 567 | * read requests by default. This option turns it off if the OS 568 | * supports it. Turning it off may help random read performance 569 | * when the DB is larger than RAM and system RAM is full. 570 | * The option is not implemented on Windows. 571 | *
  • #MDB_NOMEMINIT 572 | * Don't initialize malloc'd memory before writing to unused spaces 573 | * in the data file. By default, memory for pages written to the data 574 | * file is obtained using malloc. While these pages may be reused in 575 | * subsequent transactions, freshly malloc'd pages will be initialized 576 | * to zeroes before use. This avoids persisting leftover data from other 577 | * code (that used the heap and subsequently freed the memory) into the 578 | * data file. Note that many other system libraries may allocate 579 | * and free memory from the heap for arbitrary uses. E.g., stdio may 580 | * use the heap for file I/O buffers. This initialization step has a 581 | * modest performance cost so some applications may want to disable 582 | * it using this flag. This option can be a problem for applications 583 | * which handle sensitive data like passwords, and it makes memory 584 | * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, 585 | * which writes directly to the mmap instead of using malloc for pages. The 586 | * initialization is also skipped if #MDB_RESERVE is used; the 587 | * caller is expected to overwrite all of the memory that was 588 | * reserved in that case. 589 | * This flag may be changed at any time using #mdb_env_set_flags(). 590 | *
591 | * @param[in] mode The UNIX permissions to set on created files. This parameter 592 | * is ignored on Windows. 593 | * @return A non-zero error value on failure and 0 on success. Some possible 594 | * errors are: 595 | *
    596 | *
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the 597 | * version that created the database environment. 598 | *
  • #MDB_INVALID - the environment file headers are corrupted. 599 | *
  • ENOENT - the directory specified by the path parameter doesn't exist. 600 | *
  • EACCES - the user didn't have permission to access the environment files. 601 | *
  • EAGAIN - the environment was locked by another process. 602 | *
603 | */ 604 | int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); 605 | 606 | /** @brief Copy an LMDB environment to the specified path. 607 | * 608 | * This function may be used to make a backup of an existing environment. 609 | * No lockfile is created, since it gets recreated at need. 610 | * @note This call can trigger significant file size growth if run in 611 | * parallel with write transactions, because it employs a read-only 612 | * transaction. See long-lived transactions under @ref caveats_sec. 613 | * @param[in] env An environment handle returned by #mdb_env_create(). It 614 | * must have already been opened successfully. 615 | * @param[in] path The directory in which the copy will reside. This 616 | * directory must already exist and be writable but must otherwise be 617 | * empty. 618 | * @return A non-zero error value on failure and 0 on success. 619 | */ 620 | int mdb_env_copy(MDB_env *env, const char *path); 621 | 622 | /** @brief Copy an LMDB environment to the specified file descriptor. 623 | * 624 | * This function may be used to make a backup of an existing environment. 625 | * No lockfile is created, since it gets recreated at need. 626 | * @note This call can trigger significant file size growth if run in 627 | * parallel with write transactions, because it employs a read-only 628 | * transaction. See long-lived transactions under @ref caveats_sec. 629 | * @param[in] env An environment handle returned by #mdb_env_create(). It 630 | * must have already been opened successfully. 631 | * @param[in] fd The filedescriptor to write the copy to. It must 632 | * have already been opened for Write access. 633 | * @return A non-zero error value on failure and 0 on success. 634 | */ 635 | int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); 636 | 637 | /** @brief Copy an LMDB environment to the specified path, with options. 638 | * 639 | * This function may be used to make a backup of an existing environment. 640 | * No lockfile is created, since it gets recreated at need. 641 | * @note This call can trigger significant file size growth if run in 642 | * parallel with write transactions, because it employs a read-only 643 | * transaction. See long-lived transactions under @ref caveats_sec. 644 | * @param[in] env An environment handle returned by #mdb_env_create(). It 645 | * must have already been opened successfully. 646 | * @param[in] path The directory in which the copy will reside. This 647 | * directory must already exist and be writable but must otherwise be 648 | * empty. 649 | * @param[in] flags Special options for this operation. This parameter 650 | * must be set to 0 or by bitwise OR'ing together one or more of the 651 | * values described here. 652 | *
    653 | *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free 654 | * pages and sequentially renumber all pages in output. This option 655 | * consumes more CPU and runs more slowly than the default. 656 | *
657 | * @return A non-zero error value on failure and 0 on success. 658 | */ 659 | int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); 660 | 661 | /** @brief Copy an LMDB environment to the specified file descriptor, 662 | * with options. 663 | * 664 | * This function may be used to make a backup of an existing environment. 665 | * No lockfile is created, since it gets recreated at need. See 666 | * #mdb_env_copy2() for further details. 667 | * @note This call can trigger significant file size growth if run in 668 | * parallel with write transactions, because it employs a read-only 669 | * transaction. See long-lived transactions under @ref caveats_sec. 670 | * @param[in] env An environment handle returned by #mdb_env_create(). It 671 | * must have already been opened successfully. 672 | * @param[in] fd The filedescriptor to write the copy to. It must 673 | * have already been opened for Write access. 674 | * @param[in] flags Special options for this operation. 675 | * See #mdb_env_copy2() for options. 676 | * @return A non-zero error value on failure and 0 on success. 677 | */ 678 | int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); 679 | 680 | /** @brief Return statistics about the LMDB environment. 681 | * 682 | * @param[in] env An environment handle returned by #mdb_env_create() 683 | * @param[out] stat The address of an #MDB_stat structure 684 | * where the statistics will be copied 685 | */ 686 | int mdb_env_stat(MDB_env *env, MDB_stat *stat); 687 | 688 | /** @brief Return information about the LMDB environment. 689 | * 690 | * @param[in] env An environment handle returned by #mdb_env_create() 691 | * @param[out] stat The address of an #MDB_envinfo structure 692 | * where the information will be copied 693 | */ 694 | int mdb_env_info(MDB_env *env, MDB_envinfo *stat); 695 | 696 | /** @brief Flush the data buffers to disk. 697 | * 698 | * Data is always written to disk when #mdb_txn_commit() is called, 699 | * but the operating system may keep it buffered. LMDB always flushes 700 | * the OS buffers upon commit as well, unless the environment was 701 | * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. 702 | * @param[in] env An environment handle returned by #mdb_env_create() 703 | * @param[in] force If non-zero, force a synchronous flush. Otherwise 704 | * if the environment has the #MDB_NOSYNC flag set the flushes 705 | * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. 706 | * @return A non-zero error value on failure and 0 on success. Some possible 707 | * errors are: 708 | *
    709 | *
  • EINVAL - an invalid parameter was specified. 710 | *
  • EIO - an error occurred during synchronization. 711 | *
712 | */ 713 | int mdb_env_sync(MDB_env *env, int force); 714 | 715 | /** @brief Close the environment and release the memory map. 716 | * 717 | * Only a single thread may call this function. All transactions, databases, 718 | * and cursors must already be closed before calling this function. Attempts to 719 | * use any such handles after calling this function will cause a SIGSEGV. 720 | * The environment handle will be freed and must not be used again after this call. 721 | * @param[in] env An environment handle returned by #mdb_env_create() 722 | */ 723 | void mdb_env_close(MDB_env *env); 724 | 725 | /** @brief Set environment flags. 726 | * 727 | * This may be used to set some flags in addition to those from 728 | * #mdb_env_open(), or to unset these flags. If several threads 729 | * change the flags at the same time, the result is undefined. 730 | * @param[in] env An environment handle returned by #mdb_env_create() 731 | * @param[in] flags The flags to change, bitwise OR'ed together 732 | * @param[in] onoff A non-zero value sets the flags, zero clears them. 733 | * @return A non-zero error value on failure and 0 on success. Some possible 734 | * errors are: 735 | *
    736 | *
  • EINVAL - an invalid parameter was specified. 737 | *
738 | */ 739 | int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); 740 | 741 | /** @brief Get environment flags. 742 | * 743 | * @param[in] env An environment handle returned by #mdb_env_create() 744 | * @param[out] flags The address of an integer to store the flags 745 | * @return A non-zero error value on failure and 0 on success. Some possible 746 | * errors are: 747 | *
    748 | *
  • EINVAL - an invalid parameter was specified. 749 | *
750 | */ 751 | int mdb_env_get_flags(MDB_env *env, unsigned int *flags); 752 | 753 | /** @brief Return the path that was used in #mdb_env_open(). 754 | * 755 | * @param[in] env An environment handle returned by #mdb_env_create() 756 | * @param[out] path Address of a string pointer to contain the path. This 757 | * is the actual string in the environment, not a copy. It should not be 758 | * altered in any way. 759 | * @return A non-zero error value on failure and 0 on success. Some possible 760 | * errors are: 761 | *
    762 | *
  • EINVAL - an invalid parameter was specified. 763 | *
764 | */ 765 | int mdb_env_get_path(MDB_env *env, const char **path); 766 | 767 | /** @brief Return the filedescriptor for the given environment. 768 | * 769 | * @param[in] env An environment handle returned by #mdb_env_create() 770 | * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. 771 | * @return A non-zero error value on failure and 0 on success. Some possible 772 | * errors are: 773 | *
    774 | *
  • EINVAL - an invalid parameter was specified. 775 | *
776 | */ 777 | int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); 778 | 779 | /** @brief Set the size of the memory map to use for this environment. 780 | * 781 | * The size should be a multiple of the OS page size. The default is 782 | * 10485760 bytes. The size of the memory map is also the maximum size 783 | * of the database. The value should be chosen as large as possible, 784 | * to accommodate future growth of the database. 785 | * This function should be called after #mdb_env_create() and before #mdb_env_open(). 786 | * It may be called at later times if no transactions are active in 787 | * this process. Note that the library does not check for this condition, 788 | * the caller must ensure it explicitly. 789 | * 790 | * The new size takes effect immediately for the current process but 791 | * will not be persisted to any others until a write transaction has been 792 | * committed by the current process. Also, only mapsize increases are 793 | * persisted into the environment. 794 | * 795 | * If the mapsize is increased by another process, and data has grown 796 | * beyond the range of the current mapsize, #mdb_txn_begin() will 797 | * return #MDB_MAP_RESIZED. This function may be called with a size 798 | * of zero to adopt the new size. 799 | * 800 | * Any attempt to set a size smaller than the space already consumed 801 | * by the environment will be silently changed to the current size of the used space. 802 | * @param[in] env An environment handle returned by #mdb_env_create() 803 | * @param[in] size The size in bytes 804 | * @return A non-zero error value on failure and 0 on success. Some possible 805 | * errors are: 806 | *
    807 | *
  • EINVAL - an invalid parameter was specified, or the environment has 808 | * an active write transaction. 809 | *
810 | */ 811 | int mdb_env_set_mapsize(MDB_env *env, size_t size); 812 | 813 | /** @brief Set the maximum number of threads/reader slots for the environment. 814 | * 815 | * This defines the number of slots in the lock table that is used to track readers in the 816 | * the environment. The default is 126. 817 | * Starting a read-only transaction normally ties a lock table slot to the 818 | * current thread until the environment closes or the thread exits. If 819 | * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the 820 | * MDB_txn object until it or the #MDB_env object is destroyed. 821 | * This function may only be called after #mdb_env_create() and before #mdb_env_open(). 822 | * @param[in] env An environment handle returned by #mdb_env_create() 823 | * @param[in] readers The maximum number of reader lock table slots 824 | * @return A non-zero error value on failure and 0 on success. Some possible 825 | * errors are: 826 | *
    827 | *
  • EINVAL - an invalid parameter was specified, or the environment is already open. 828 | *
829 | */ 830 | int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); 831 | 832 | /** @brief Get the maximum number of threads/reader slots for the environment. 833 | * 834 | * @param[in] env An environment handle returned by #mdb_env_create() 835 | * @param[out] readers Address of an integer to store the number of readers 836 | * @return A non-zero error value on failure and 0 on success. Some possible 837 | * errors are: 838 | *
    839 | *
  • EINVAL - an invalid parameter was specified. 840 | *
841 | */ 842 | int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); 843 | 844 | /** @brief Set the maximum number of named databases for the environment. 845 | * 846 | * This function is only needed if multiple databases will be used in the 847 | * environment. Simpler applications that use the environment as a single 848 | * unnamed database can ignore this option. 849 | * This function may only be called after #mdb_env_create() and before #mdb_env_open(). 850 | * 851 | * Currently a moderate number of slots are cheap but a huge number gets 852 | * expensive: 7-120 words per transaction, and every #mdb_dbi_open() 853 | * does a linear search of the opened slots. 854 | * @param[in] env An environment handle returned by #mdb_env_create() 855 | * @param[in] dbs The maximum number of databases 856 | * @return A non-zero error value on failure and 0 on success. Some possible 857 | * errors are: 858 | *
    859 | *
  • EINVAL - an invalid parameter was specified, or the environment is already open. 860 | *
861 | */ 862 | int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); 863 | 864 | /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. 865 | * 866 | * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. 867 | * See @ref MDB_val. 868 | * @param[in] env An environment handle returned by #mdb_env_create() 869 | * @return The maximum size of a key we can write 870 | */ 871 | int mdb_env_get_maxkeysize(MDB_env *env); 872 | 873 | /** @brief Set application information associated with the #MDB_env. 874 | * 875 | * @param[in] env An environment handle returned by #mdb_env_create() 876 | * @param[in] ctx An arbitrary pointer for whatever the application needs. 877 | * @return A non-zero error value on failure and 0 on success. 878 | */ 879 | int mdb_env_set_userctx(MDB_env *env, void *ctx); 880 | 881 | /** @brief Get the application information associated with the #MDB_env. 882 | * 883 | * @param[in] env An environment handle returned by #mdb_env_create() 884 | * @return The pointer set by #mdb_env_set_userctx(). 885 | */ 886 | void *mdb_env_get_userctx(MDB_env *env); 887 | 888 | /** @brief A callback function for most LMDB assert() failures, 889 | * called before printing the message and aborting. 890 | * 891 | * @param[in] env An environment handle returned by #mdb_env_create(). 892 | * @param[in] msg The assertion message, not including newline. 893 | */ 894 | typedef void MDB_assert_func(MDB_env *env, const char *msg); 895 | 896 | /** Set or reset the assert() callback of the environment. 897 | * Disabled if liblmdb is buillt with NDEBUG. 898 | * @note This hack should become obsolete as lmdb's error handling matures. 899 | * @param[in] env An environment handle returned by #mdb_env_create(). 900 | * @param[in] func An #MDB_assert_func function, or 0. 901 | * @return A non-zero error value on failure and 0 on success. 902 | */ 903 | int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); 904 | 905 | /** @brief Create a transaction for use with the environment. 906 | * 907 | * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). 908 | * @note A transaction and its cursors must only be used by a single 909 | * thread, and a thread may only have a single transaction at a time. 910 | * If #MDB_NOTLS is in use, this does not apply to read-only transactions. 911 | * @note Cursors may not span transactions. 912 | * @param[in] env An environment handle returned by #mdb_env_create() 913 | * @param[in] parent If this parameter is non-NULL, the new transaction 914 | * will be a nested transaction, with the transaction indicated by \b parent 915 | * as its parent. Transactions may be nested to any level. A parent 916 | * transaction and its cursors may not issue any other operations than 917 | * mdb_txn_commit and mdb_txn_abort while it has active child transactions. 918 | * @param[in] flags Special options for this transaction. This parameter 919 | * must be set to 0 or by bitwise OR'ing together one or more of the 920 | * values described here. 921 | *
    922 | *
  • #MDB_RDONLY 923 | * This transaction will not perform any write operations. 924 | *
925 | * @param[out] txn Address where the new #MDB_txn handle will be stored 926 | * @return A non-zero error value on failure and 0 on success. Some possible 927 | * errors are: 928 | *
    929 | *
  • #MDB_PANIC - a fatal error occurred earlier and the environment 930 | * must be shut down. 931 | *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's 932 | * mapsize and this environment's map must be resized as well. 933 | * See #mdb_env_set_mapsize(). 934 | *
  • #MDB_READERS_FULL - a read-only transaction was requested and 935 | * the reader lock table is full. See #mdb_env_set_maxreaders(). 936 | *
  • ENOMEM - out of memory. 937 | *
938 | */ 939 | int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); 940 | 941 | /** @brief Returns the transaction's #MDB_env 942 | * 943 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 944 | */ 945 | MDB_env *mdb_txn_env(MDB_txn *txn); 946 | 947 | /** @brief Commit all the operations of a transaction into the database. 948 | * 949 | * The transaction handle is freed. It and its cursors must not be used 950 | * again after this call, except with #mdb_cursor_renew(). 951 | * @note Earlier documentation incorrectly said all cursors would be freed. 952 | * Only write-transactions free cursors. 953 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 954 | * @return A non-zero error value on failure and 0 on success. Some possible 955 | * errors are: 956 | *
    957 | *
  • EINVAL - an invalid parameter was specified. 958 | *
  • ENOSPC - no more disk space. 959 | *
  • EIO - a low-level I/O error occurred while writing. 960 | *
  • ENOMEM - out of memory. 961 | *
962 | */ 963 | int mdb_txn_commit(MDB_txn *txn); 964 | 965 | /** @brief Abandon all the operations of the transaction instead of saving them. 966 | * 967 | * The transaction handle is freed. It and its cursors must not be used 968 | * again after this call, except with #mdb_cursor_renew(). 969 | * @note Earlier documentation incorrectly said all cursors would be freed. 970 | * Only write-transactions free cursors. 971 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 972 | */ 973 | void mdb_txn_abort(MDB_txn *txn); 974 | 975 | /** @brief Reset a read-only transaction. 976 | * 977 | * Abort the transaction like #mdb_txn_abort(), but keep the transaction 978 | * handle. #mdb_txn_renew() may reuse the handle. This saves allocation 979 | * overhead if the process will start a new read-only transaction soon, 980 | * and also locking overhead if #MDB_NOTLS is in use. The reader table 981 | * lock is released, but the table slot stays tied to its thread or 982 | * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free 983 | * its lock table slot if MDB_NOTLS is in use. 984 | * Cursors opened within the transaction must not be used 985 | * again after this call, except with #mdb_cursor_renew(). 986 | * Reader locks generally don't interfere with writers, but they keep old 987 | * versions of database pages allocated. Thus they prevent the old pages 988 | * from being reused when writers commit new data, and so under heavy load 989 | * the database size may grow much more rapidly than otherwise. 990 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 991 | */ 992 | void mdb_txn_reset(MDB_txn *txn); 993 | 994 | /** @brief Renew a read-only transaction. 995 | * 996 | * This acquires a new reader lock for a transaction handle that had been 997 | * released by #mdb_txn_reset(). It must be called before a reset transaction 998 | * may be used again. 999 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1000 | * @return A non-zero error value on failure and 0 on success. Some possible 1001 | * errors are: 1002 | *
    1003 | *
  • #MDB_PANIC - a fatal error occurred earlier and the environment 1004 | * must be shut down. 1005 | *
  • EINVAL - an invalid parameter was specified. 1006 | *
1007 | */ 1008 | int mdb_txn_renew(MDB_txn *txn); 1009 | 1010 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ 1011 | #define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) 1012 | /** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ 1013 | #define mdb_close(env,dbi) mdb_dbi_close(env,dbi) 1014 | 1015 | /** @brief Open a database in the environment. 1016 | * 1017 | * A database handle denotes the name and parameters of a database, 1018 | * independently of whether such a database exists. 1019 | * The database handle may be discarded by calling #mdb_dbi_close(). 1020 | * The old database handle is returned if the database was already open. 1021 | * The handle may only be closed once. 1022 | * The database handle will be private to the current transaction until 1023 | * the transaction is successfully committed. If the transaction is 1024 | * aborted the handle will be closed automatically. 1025 | * After a successful commit the 1026 | * handle will reside in the shared environment, and may be used 1027 | * by other transactions. This function must not be called from 1028 | * multiple concurrent transactions. A transaction that uses this function 1029 | * must finish (either commit or abort) before any other transaction may 1030 | * use this function. 1031 | * 1032 | * To use named databases (with name != NULL), #mdb_env_set_maxdbs() 1033 | * must be called before opening the environment. Database names 1034 | * are kept as keys in the unnamed database. 1035 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1036 | * @param[in] name The name of the database to open. If only a single 1037 | * database is needed in the environment, this value may be NULL. 1038 | * @param[in] flags Special options for this database. This parameter 1039 | * must be set to 0 or by bitwise OR'ing together one or more of the 1040 | * values described here. 1041 | *
    1042 | *
  • #MDB_REVERSEKEY 1043 | * Keys are strings to be compared in reverse order, from the end 1044 | * of the strings to the beginning. By default, Keys are treated as strings and 1045 | * compared from beginning to end. 1046 | *
  • #MDB_DUPSORT 1047 | * Duplicate keys may be used in the database. (Or, from another perspective, 1048 | * keys may have multiple data items, stored in sorted order.) By default 1049 | * keys must be unique and may have only a single data item. 1050 | *
  • #MDB_INTEGERKEY 1051 | * Keys are binary integers in native byte order. Setting this option 1052 | * requires all keys to be the same size, typically sizeof(int) 1053 | * or sizeof(size_t). 1054 | *
  • #MDB_DUPFIXED 1055 | * This flag may only be used in combination with #MDB_DUPSORT. This option 1056 | * tells the library that the data items for this database are all the same 1057 | * size, which allows further optimizations in storage and retrieval. When 1058 | * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE 1059 | * cursor operations may be used to retrieve multiple items at once. 1060 | *
  • #MDB_INTEGERDUP 1061 | * This option specifies that duplicate data items are also integers, and 1062 | * should be sorted as such. 1063 | *
  • #MDB_REVERSEDUP 1064 | * This option specifies that duplicate data items should be compared as 1065 | * strings in reverse order. 1066 | *
  • #MDB_CREATE 1067 | * Create the named database if it doesn't exist. This option is not 1068 | * allowed in a read-only transaction or a read-only environment. 1069 | *
1070 | * @param[out] dbi Address where the new #MDB_dbi handle will be stored 1071 | * @return A non-zero error value on failure and 0 on success. Some possible 1072 | * errors are: 1073 | *
    1074 | *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment 1075 | * and #MDB_CREATE was not specified. 1076 | *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). 1077 | *
1078 | */ 1079 | int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); 1080 | 1081 | /** @brief Retrieve statistics for a database. 1082 | * 1083 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1084 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1085 | * @param[out] stat The address of an #MDB_stat structure 1086 | * where the statistics will be copied 1087 | * @return A non-zero error value on failure and 0 on success. Some possible 1088 | * errors are: 1089 | *
    1090 | *
  • EINVAL - an invalid parameter was specified. 1091 | *
1092 | */ 1093 | int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); 1094 | 1095 | /** @brief Retrieve the DB flags for a database handle. 1096 | * 1097 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1098 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1099 | * @param[out] flags Address where the flags will be returned. 1100 | * @return A non-zero error value on failure and 0 on success. 1101 | */ 1102 | int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); 1103 | 1104 | /** @brief Close a database handle. Normally unnecessary. Use with care: 1105 | * 1106 | * This call is not mutex protected. Handles should only be closed by 1107 | * a single thread, and only if no other threads are going to reference 1108 | * the database handle or one of its cursors any further. Do not close 1109 | * a handle if an existing transaction has modified its database. 1110 | * Doing so can cause misbehavior from database corruption to errors 1111 | * like MDB_BAD_VALSIZE (since the DB name is gone). 1112 | * 1113 | * Closing a database handle is not necessary, but lets #mdb_dbi_open() 1114 | * reuse the handle value. Usually it's better to set a bigger 1115 | * #mdb_env_set_maxdbs(), unless that value would be large. 1116 | * 1117 | * @param[in] env An environment handle returned by #mdb_env_create() 1118 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1119 | */ 1120 | void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); 1121 | 1122 | /** @brief Empty or delete+close a database. 1123 | * 1124 | * See #mdb_dbi_close() for restrictions about closing the DB handle. 1125 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1126 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1127 | * @param[in] del 0 to empty the DB, 1 to delete it from the 1128 | * environment and close the DB handle. 1129 | * @return A non-zero error value on failure and 0 on success. 1130 | */ 1131 | int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); 1132 | 1133 | /** @brief Set a custom key comparison function for a database. 1134 | * 1135 | * The comparison function is called whenever it is necessary to compare a 1136 | * key specified by the application with a key currently stored in the database. 1137 | * If no comparison function is specified, and no special key flags were specified 1138 | * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating 1139 | * before longer keys. 1140 | * @warning This function must be called before any data access functions are used, 1141 | * otherwise data corruption may occur. The same comparison function must be used by every 1142 | * program accessing the database, every time the database is used. 1143 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1144 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1145 | * @param[in] cmp A #MDB_cmp_func function 1146 | * @return A non-zero error value on failure and 0 on success. Some possible 1147 | * errors are: 1148 | *
    1149 | *
  • EINVAL - an invalid parameter was specified. 1150 | *
1151 | */ 1152 | int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); 1153 | 1154 | /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. 1155 | * 1156 | * This comparison function is called whenever it is necessary to compare a data 1157 | * item specified by the application with a data item currently stored in the database. 1158 | * This function only takes effect if the database was opened with the #MDB_DUPSORT 1159 | * flag. 1160 | * If no comparison function is specified, and no special key flags were specified 1161 | * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating 1162 | * before longer items. 1163 | * @warning This function must be called before any data access functions are used, 1164 | * otherwise data corruption may occur. The same comparison function must be used by every 1165 | * program accessing the database, every time the database is used. 1166 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1167 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1168 | * @param[in] cmp A #MDB_cmp_func function 1169 | * @return A non-zero error value on failure and 0 on success. Some possible 1170 | * errors are: 1171 | *
    1172 | *
  • EINVAL - an invalid parameter was specified. 1173 | *
1174 | */ 1175 | int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); 1176 | 1177 | /** @brief Set a relocation function for a #MDB_FIXEDMAP database. 1178 | * 1179 | * @todo The relocation function is called whenever it is necessary to move the data 1180 | * of an item to a different position in the database (e.g. through tree 1181 | * balancing operations, shifts as a result of adds or deletes, etc.). It is 1182 | * intended to allow address/position-dependent data items to be stored in 1183 | * a database in an environment opened with the #MDB_FIXEDMAP option. 1184 | * Currently the relocation feature is unimplemented and setting 1185 | * this function has no effect. 1186 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1187 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1188 | * @param[in] rel A #MDB_rel_func function 1189 | * @return A non-zero error value on failure and 0 on success. Some possible 1190 | * errors are: 1191 | *
    1192 | *
  • EINVAL - an invalid parameter was specified. 1193 | *
1194 | */ 1195 | int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); 1196 | 1197 | /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. 1198 | * 1199 | * See #mdb_set_relfunc and #MDB_rel_func for more details. 1200 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1201 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1202 | * @param[in] ctx An arbitrary pointer for whatever the application needs. 1203 | * It will be passed to the callback function set by #mdb_set_relfunc 1204 | * as its \b relctx parameter whenever the callback is invoked. 1205 | * @return A non-zero error value on failure and 0 on success. Some possible 1206 | * errors are: 1207 | *
    1208 | *
  • EINVAL - an invalid parameter was specified. 1209 | *
1210 | */ 1211 | int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); 1212 | 1213 | /** @brief Get items from a database. 1214 | * 1215 | * This function retrieves key/data pairs from the database. The address 1216 | * and length of the data associated with the specified \b key are returned 1217 | * in the structure to which \b data refers. 1218 | * If the database supports duplicate keys (#MDB_DUPSORT) then the 1219 | * first data item for the key will be returned. Retrieval of other 1220 | * items requires the use of #mdb_cursor_get(). 1221 | * 1222 | * @note The memory pointed to by the returned values is owned by the 1223 | * database. The caller need not dispose of the memory, and may not 1224 | * modify it in any way. For values returned in a read-only transaction 1225 | * any modification attempts will cause a SIGSEGV. 1226 | * @note Values returned from the database are valid only until a 1227 | * subsequent update operation, or the end of the transaction. 1228 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1229 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1230 | * @param[in] key The key to search for in the database 1231 | * @param[out] data The data corresponding to the key 1232 | * @return A non-zero error value on failure and 0 on success. Some possible 1233 | * errors are: 1234 | *
    1235 | *
  • #MDB_NOTFOUND - the key was not in the database. 1236 | *
  • EINVAL - an invalid parameter was specified. 1237 | *
1238 | */ 1239 | int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); 1240 | 1241 | /** @brief Store items into a database. 1242 | * 1243 | * This function stores key/data pairs in the database. The default behavior 1244 | * is to enter the new key/data pair, replacing any previously existing key 1245 | * if duplicates are disallowed, or adding a duplicate data item if 1246 | * duplicates are allowed (#MDB_DUPSORT). 1247 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1248 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1249 | * @param[in] key The key to store in the database 1250 | * @param[in,out] data The data to store 1251 | * @param[in] flags Special options for this operation. This parameter 1252 | * must be set to 0 or by bitwise OR'ing together one or more of the 1253 | * values described here. 1254 | *
    1255 | *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not 1256 | * already appear in the database. This flag may only be specified 1257 | * if the database was opened with #MDB_DUPSORT. The function will 1258 | * return #MDB_KEYEXIST if the key/data pair already appears in the 1259 | * database. 1260 | *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key 1261 | * does not already appear in the database. The function will return 1262 | * #MDB_KEYEXIST if the key already appears in the database, even if 1263 | * the database supports duplicates (#MDB_DUPSORT). The \b data 1264 | * parameter will be set to point to the existing item. 1265 | *
  • #MDB_RESERVE - reserve space for data of the given size, but 1266 | * don't copy the given data. Instead, return a pointer to the 1267 | * reserved space, which the caller can fill in later - before 1268 | * the next update operation or the transaction ends. This saves 1269 | * an extra memcpy if the data is being generated later. 1270 | * LMDB does nothing else with this memory, the caller is expected 1271 | * to modify all of the space requested. 1272 | *
  • #MDB_APPEND - append the given key/data pair to the end of the 1273 | * database. No key comparisons are performed. This option allows 1274 | * fast bulk loading when keys are already known to be in the 1275 | * correct order. Loading unsorted keys with this flag will cause 1276 | * data corruption. 1277 | *
  • #MDB_APPENDDUP - as above, but for sorted dup data. 1278 | *
1279 | * @return A non-zero error value on failure and 0 on success. Some possible 1280 | * errors are: 1281 | *
    1282 | *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). 1283 | *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. 1284 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1285 | *
  • EINVAL - an invalid parameter was specified. 1286 | *
1287 | */ 1288 | int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, 1289 | unsigned int flags); 1290 | 1291 | /** @brief Delete items from a database. 1292 | * 1293 | * This function removes key/data pairs from the database. 1294 | * If the database does not support sorted duplicate data items 1295 | * (#MDB_DUPSORT) the data parameter is ignored. 1296 | * If the database supports sorted duplicates and the data parameter 1297 | * is NULL, all of the duplicate data items for the key will be 1298 | * deleted. Otherwise, if the data parameter is non-NULL 1299 | * only the matching data item will be deleted. 1300 | * This function will return #MDB_NOTFOUND if the specified key/data 1301 | * pair is not in the database. 1302 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1303 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1304 | * @param[in] key The key to delete from the database 1305 | * @param[in] data The data to delete 1306 | * @return A non-zero error value on failure and 0 on success. Some possible 1307 | * errors are: 1308 | *
    1309 | *
  • EACCES - an attempt was made to write in a read-only transaction. 1310 | *
  • EINVAL - an invalid parameter was specified. 1311 | *
1312 | */ 1313 | int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); 1314 | 1315 | /** @brief Create a cursor handle. 1316 | * 1317 | * A cursor is associated with a specific transaction and database. 1318 | * A cursor cannot be used when its database handle is closed. Nor 1319 | * when its transaction has ended, except with #mdb_cursor_renew(). 1320 | * It can be discarded with #mdb_cursor_close(). 1321 | * A cursor in a write-transaction can be closed before its transaction 1322 | * ends, and will otherwise be closed when its transaction ends. 1323 | * A cursor in a read-only transaction must be closed explicitly, before 1324 | * or after its transaction ends. It can be reused with 1325 | * #mdb_cursor_renew() before finally closing it. 1326 | * @note Earlier documentation said that cursors in every transaction 1327 | * were closed when the transaction committed or aborted. 1328 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1329 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1330 | * @param[out] cursor Address where the new #MDB_cursor handle will be stored 1331 | * @return A non-zero error value on failure and 0 on success. Some possible 1332 | * errors are: 1333 | *
    1334 | *
  • EINVAL - an invalid parameter was specified. 1335 | *
1336 | */ 1337 | int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); 1338 | 1339 | /** @brief Close a cursor handle. 1340 | * 1341 | * The cursor handle will be freed and must not be used again after this call. 1342 | * Its transaction must still be live if it is a write-transaction. 1343 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1344 | */ 1345 | void mdb_cursor_close(MDB_cursor *cursor); 1346 | 1347 | /** @brief Renew a cursor handle. 1348 | * 1349 | * A cursor is associated with a specific transaction and database. 1350 | * Cursors that are only used in read-only 1351 | * transactions may be re-used, to avoid unnecessary malloc/free overhead. 1352 | * The cursor may be associated with a new read-only transaction, and 1353 | * referencing the same database handle as it was created with. 1354 | * This may be done whether the previous transaction is live or dead. 1355 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1356 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1357 | * @return A non-zero error value on failure and 0 on success. Some possible 1358 | * errors are: 1359 | *
    1360 | *
  • EINVAL - an invalid parameter was specified. 1361 | *
1362 | */ 1363 | int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); 1364 | 1365 | /** @brief Return the cursor's transaction handle. 1366 | * 1367 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1368 | */ 1369 | MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); 1370 | 1371 | /** @brief Return the cursor's database handle. 1372 | * 1373 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1374 | */ 1375 | MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); 1376 | 1377 | /** @brief Retrieve by cursor. 1378 | * 1379 | * This function retrieves key/data pairs from the database. The address and length 1380 | * of the key are returned in the object to which \b key refers (except for the 1381 | * case of the #MDB_SET option, in which the \b key object is unchanged), and 1382 | * the address and length of the data are returned in the object to which \b data 1383 | * refers. 1384 | * See #mdb_get() for restrictions on using the output values. 1385 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1386 | * @param[in,out] key The key for a retrieved item 1387 | * @param[in,out] data The data of a retrieved item 1388 | * @param[in] op A cursor operation #MDB_cursor_op 1389 | * @return A non-zero error value on failure and 0 on success. Some possible 1390 | * errors are: 1391 | *
    1392 | *
  • #MDB_NOTFOUND - no matching key found. 1393 | *
  • EINVAL - an invalid parameter was specified. 1394 | *
1395 | */ 1396 | int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, 1397 | MDB_cursor_op op); 1398 | 1399 | /** @brief Store by cursor. 1400 | * 1401 | * This function stores key/data pairs into the database. 1402 | * The cursor is positioned at the new item, or on failure usually near it. 1403 | * @note Earlier documentation incorrectly said errors would leave the 1404 | * state of the cursor unchanged. 1405 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1406 | * @param[in] key The key operated on. 1407 | * @param[in] data The data operated on. 1408 | * @param[in] flags Options for this operation. This parameter 1409 | * must be set to 0 or one of the values described here. 1410 | *
    1411 | *
  • #MDB_CURRENT - replace the item at the current cursor position. 1412 | * The \b key parameter must still be provided, and must match it. 1413 | * If using sorted duplicates (#MDB_DUPSORT) the data item must still 1414 | * sort into the same place. This is intended to be used when the 1415 | * new data is the same size as the old. Otherwise it will simply 1416 | * perform a delete of the old record followed by an insert. 1417 | *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not 1418 | * already appear in the database. This flag may only be specified 1419 | * if the database was opened with #MDB_DUPSORT. The function will 1420 | * return #MDB_KEYEXIST if the key/data pair already appears in the 1421 | * database. 1422 | *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key 1423 | * does not already appear in the database. The function will return 1424 | * #MDB_KEYEXIST if the key already appears in the database, even if 1425 | * the database supports duplicates (#MDB_DUPSORT). 1426 | *
  • #MDB_RESERVE - reserve space for data of the given size, but 1427 | * don't copy the given data. Instead, return a pointer to the 1428 | * reserved space, which the caller can fill in later. This saves 1429 | * an extra memcpy if the data is being generated later. 1430 | *
  • #MDB_APPEND - append the given key/data pair to the end of the 1431 | * database. No key comparisons are performed. This option allows 1432 | * fast bulk loading when keys are already known to be in the 1433 | * correct order. Loading unsorted keys with this flag will cause 1434 | * data corruption. 1435 | *
  • #MDB_APPENDDUP - as above, but for sorted dup data. 1436 | *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a 1437 | * single request. This flag may only be specified if the database 1438 | * was opened with #MDB_DUPFIXED. The \b data argument must be an 1439 | * array of two MDB_vals. The mv_size of the first MDB_val must be 1440 | * the size of a single data element. The mv_data of the first MDB_val 1441 | * must point to the beginning of the array of contiguous data elements. 1442 | * The mv_size of the second MDB_val must be the count of the number 1443 | * of data elements to store. On return this field will be set to 1444 | * the count of the number of elements actually written. The mv_data 1445 | * of the second MDB_val is unused. 1446 | *
1447 | * @return A non-zero error value on failure and 0 on success. Some possible 1448 | * errors are: 1449 | *
    1450 | *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). 1451 | *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. 1452 | *
  • EACCES - an attempt was made to modify a read-only database. 1453 | *
  • EINVAL - an invalid parameter was specified. 1454 | *
1455 | */ 1456 | int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, 1457 | unsigned int flags); 1458 | 1459 | /** @brief Delete current key/data pair 1460 | * 1461 | * This function deletes the key/data pair to which the cursor refers. 1462 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1463 | * @param[in] flags Options for this operation. This parameter 1464 | * must be set to 0 or one of the values described here. 1465 | *
    1466 | *
  • #MDB_NODUPDATA - delete all of the data items for the current key. 1467 | * This flag may only be specified if the database was opened with #MDB_DUPSORT. 1468 | *
1469 | * @return A non-zero error value on failure and 0 on success. Some possible 1470 | * errors are: 1471 | *
    1472 | *
  • EACCES - an attempt was made to modify a read-only database. 1473 | *
  • EINVAL - an invalid parameter was specified. 1474 | *
1475 | */ 1476 | int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); 1477 | 1478 | /** @brief Return count of duplicates for current key. 1479 | * 1480 | * This call is only valid on databases that support sorted duplicate 1481 | * data items #MDB_DUPSORT. 1482 | * @param[in] cursor A cursor handle returned by #mdb_cursor_open() 1483 | * @param[out] countp Address where the count will be stored 1484 | * @return A non-zero error value on failure and 0 on success. Some possible 1485 | * errors are: 1486 | *
    1487 | *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. 1488 | *
1489 | */ 1490 | int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); 1491 | 1492 | /** @brief Compare two data items according to a particular database. 1493 | * 1494 | * This returns a comparison as if the two data items were keys in the 1495 | * specified database. 1496 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1497 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1498 | * @param[in] a The first item to compare 1499 | * @param[in] b The second item to compare 1500 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b 1501 | */ 1502 | int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); 1503 | 1504 | /** @brief Compare two data items according to a particular database. 1505 | * 1506 | * This returns a comparison as if the two items were data items of 1507 | * the specified database. The database must have the #MDB_DUPSORT flag. 1508 | * @param[in] txn A transaction handle returned by #mdb_txn_begin() 1509 | * @param[in] dbi A database handle returned by #mdb_dbi_open() 1510 | * @param[in] a The first item to compare 1511 | * @param[in] b The second item to compare 1512 | * @return < 0 if a < b, 0 if a == b, > 0 if a > b 1513 | */ 1514 | int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); 1515 | 1516 | /** @brief A callback function used to print a message from the library. 1517 | * 1518 | * @param[in] msg The string to be printed. 1519 | * @param[in] ctx An arbitrary context pointer for the callback. 1520 | * @return < 0 on failure, >= 0 on success. 1521 | */ 1522 | typedef int (MDB_msg_func)(const char *msg, void *ctx); 1523 | 1524 | /** @brief Dump the entries in the reader lock table. 1525 | * 1526 | * @param[in] env An environment handle returned by #mdb_env_create() 1527 | * @param[in] func A #MDB_msg_func function 1528 | * @param[in] ctx Anything the message function needs 1529 | * @return < 0 on failure, >= 0 on success. 1530 | */ 1531 | int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); 1532 | 1533 | /** @brief Check for stale entries in the reader lock table. 1534 | * 1535 | * @param[in] env An environment handle returned by #mdb_env_create() 1536 | * @param[out] dead Number of stale slots that were cleared 1537 | * @return 0 on success, non-zero on failure. 1538 | */ 1539 | int mdb_reader_check(MDB_env *env, int *dead); 1540 | /** @} */ 1541 | 1542 | #ifdef __cplusplus 1543 | } 1544 | #endif 1545 | /** @page tools LMDB Command Line Tools 1546 | The following describes the command line tools that are available for LMDB. 1547 | \li \ref mdb_copy_1 1548 | \li \ref mdb_dump_1 1549 | \li \ref mdb_load_1 1550 | \li \ref mdb_stat_1 1551 | */ 1552 | 1553 | #endif /* _LMDB_H_ */ 1554 | -------------------------------------------------------------------------------- /mdb.go: -------------------------------------------------------------------------------- 1 | /* 2 | A thin wrapper for the lmdb C library. These are low-level bindings for the C 3 | API. The C documentation should be used as a reference while developing 4 | (http://symas.com/mdb/doc/group__mdb.html). 5 | 6 | Errors 7 | 8 | The errors returned by the package API will with few exceptions be of type 9 | Errno or syscall.Errno. The only errors of type Errno returned are those 10 | defined in lmdb.h. Other errno values like EINVAL will by of type 11 | syscall.Errno. 12 | */ 13 | package mdb 14 | -------------------------------------------------------------------------------- /mdb_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestTest1(t *testing.T) { 11 | env, err := NewEnv() 12 | if err != nil { 13 | t.Fatalf("Cannot create environment: %s", err) 14 | } 15 | err = env.SetMapSize(10485760) 16 | if err != nil { 17 | t.Fatalf("Cannot set mapsize: %s", err) 18 | } 19 | path, err := ioutil.TempDir("/tmp", "mdb_test") 20 | if err != nil { 21 | t.Fatalf("Cannot create temporary directory") 22 | } 23 | err = os.MkdirAll(path, 0770) 24 | defer os.RemoveAll(path) 25 | if err != nil { 26 | t.Fatalf("Cannot create directory: %s", path) 27 | } 28 | err = env.Open(path, 0, 0664) 29 | defer env.Close() 30 | if err != nil { 31 | t.Fatalf("Cannot open environment: %s", err) 32 | } 33 | var txn *Txn 34 | txn, err = env.BeginTxn(nil, 0) 35 | if err != nil { 36 | t.Fatalf("Cannot begin transaction: %s", err) 37 | } 38 | var dbi DBI 39 | dbi, err = txn.DBIOpen(nil, 0) 40 | defer env.DBIClose(dbi) 41 | if err != nil { 42 | t.Fatalf("Cannot create DBI %s", err) 43 | } 44 | var data = map[string]string{} 45 | var key string 46 | var val string 47 | num_entries := 10 48 | for i := 0; i < num_entries; i++ { 49 | key = fmt.Sprintf("Key-%d", i) 50 | val = fmt.Sprintf("Val-%d", i) 51 | data[key] = val 52 | err = txn.Put(dbi, []byte(key), []byte(val), NOOVERWRITE) 53 | if err != nil { 54 | txn.Abort() 55 | t.Fatalf("Error during put: %s", err) 56 | } 57 | } 58 | err = txn.Commit() 59 | if err != nil { 60 | txn.Abort() 61 | t.Fatalf("Cannot commit %s", err) 62 | } 63 | stat, err := env.Stat() 64 | if err != nil { 65 | t.Fatalf("Cannot get stat %s", err) 66 | } 67 | t.Logf("%+v", stat) 68 | if stat.Entries != uint64(num_entries) { 69 | t.Errorf("Less entry in the database than expected: %d <> %d", stat.Entries, num_entries) 70 | } 71 | txn, err = env.BeginTxn(nil, 0) 72 | if err != nil { 73 | t.Fatalf("Cannot begin transaction: %s", err) 74 | } 75 | var cursor *Cursor 76 | cursor, err = txn.CursorOpen(dbi) 77 | if err != nil { 78 | cursor.Close() 79 | txn.Abort() 80 | t.Fatalf("Error during cursor open %s", err) 81 | } 82 | var bkey, bval []byte 83 | var rc error 84 | for { 85 | bkey, bval, rc = cursor.Get(nil, nil, NEXT) 86 | if rc != nil { 87 | break 88 | } 89 | skey := string(bkey) 90 | sval := string(bval) 91 | t.Logf("Val: %s", sval) 92 | t.Logf("Key: %s", skey) 93 | var d string 94 | var ok bool 95 | if d, ok = data[skey]; !ok { 96 | t.Errorf("Cannot found: %q", skey) 97 | } 98 | if d != sval { 99 | t.Errorf("Data missmatch: %q <> %q", sval, d) 100 | } 101 | } 102 | cursor.Close() 103 | bval, err = txn.Get(dbi, []byte("Key-0")) 104 | if err != nil { 105 | txn.Abort() 106 | t.Fatalf("Error during txn get %s", err) 107 | } 108 | if string(bval) != "Val-0" { 109 | txn.Abort() 110 | t.Fatalf("Invalid txn get %s", string(bval)) 111 | } 112 | txn.Abort() 113 | } 114 | -------------------------------------------------------------------------------- /midl.c: -------------------------------------------------------------------------------- 1 | /** @file midl.c 2 | * @brief ldap bdb back-end ID List functions */ 3 | /* $OpenLDAP$ */ 4 | /* This work is part of OpenLDAP Software . 5 | * 6 | * Copyright 2000-2014 The OpenLDAP Foundation. 7 | * All rights reserved. 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted only as authorized by the OpenLDAP 11 | * Public License. 12 | * 13 | * A copy of this license is available in the file LICENSE in the 14 | * top-level directory of the distribution or, alternatively, at 15 | * . 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "midl.h" 24 | 25 | /** @defgroup internal LMDB Internals 26 | * @{ 27 | */ 28 | /** @defgroup idls ID List Management 29 | * @{ 30 | */ 31 | #define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) 32 | 33 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) 34 | { 35 | /* 36 | * binary search of id in ids 37 | * if found, returns position of id 38 | * if not found, returns first position greater than id 39 | */ 40 | unsigned base = 0; 41 | unsigned cursor = 1; 42 | int val = 0; 43 | unsigned n = ids[0]; 44 | 45 | while( 0 < n ) { 46 | unsigned pivot = n >> 1; 47 | cursor = base + pivot + 1; 48 | val = CMP( ids[cursor], id ); 49 | 50 | if( val < 0 ) { 51 | n = pivot; 52 | 53 | } else if ( val > 0 ) { 54 | base = cursor; 55 | n -= pivot + 1; 56 | 57 | } else { 58 | return cursor; 59 | } 60 | } 61 | 62 | if( val > 0 ) { 63 | ++cursor; 64 | } 65 | return cursor; 66 | } 67 | 68 | #if 0 /* superseded by append/sort */ 69 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) 70 | { 71 | unsigned x, i; 72 | 73 | x = mdb_midl_search( ids, id ); 74 | assert( x > 0 ); 75 | 76 | if( x < 1 ) { 77 | /* internal error */ 78 | return -2; 79 | } 80 | 81 | if ( x <= ids[0] && ids[x] == id ) { 82 | /* duplicate */ 83 | assert(0); 84 | return -1; 85 | } 86 | 87 | if ( ++ids[0] >= MDB_IDL_DB_MAX ) { 88 | /* no room */ 89 | --ids[0]; 90 | return -2; 91 | 92 | } else { 93 | /* insert id */ 94 | for (i=ids[0]; i>x; i--) 95 | ids[i] = ids[i-1]; 96 | ids[x] = id; 97 | } 98 | 99 | return 0; 100 | } 101 | #endif 102 | 103 | MDB_IDL mdb_midl_alloc(int num) 104 | { 105 | MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); 106 | if (ids) { 107 | *ids++ = num; 108 | *ids = 0; 109 | } 110 | return ids; 111 | } 112 | 113 | void mdb_midl_free(MDB_IDL ids) 114 | { 115 | if (ids) 116 | free(ids-1); 117 | } 118 | 119 | int mdb_midl_shrink( MDB_IDL *idp ) 120 | { 121 | MDB_IDL ids = *idp; 122 | if (*(--ids) > MDB_IDL_UM_MAX && 123 | (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) 124 | { 125 | *ids++ = MDB_IDL_UM_MAX; 126 | *idp = ids; 127 | return 1; 128 | } 129 | return 0; 130 | } 131 | 132 | static int mdb_midl_grow( MDB_IDL *idp, int num ) 133 | { 134 | MDB_IDL idn = *idp-1; 135 | /* grow it */ 136 | idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); 137 | if (!idn) 138 | return ENOMEM; 139 | *idn++ += num; 140 | *idp = idn; 141 | return 0; 142 | } 143 | 144 | int mdb_midl_need( MDB_IDL *idp, unsigned num ) 145 | { 146 | MDB_IDL ids = *idp; 147 | num += ids[0]; 148 | if (num > ids[-1]) { 149 | num = (num + num/4 + (256 + 2)) & -256; 150 | if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) 151 | return ENOMEM; 152 | *ids++ = num - 2; 153 | *idp = ids; 154 | } 155 | return 0; 156 | } 157 | 158 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) 159 | { 160 | MDB_IDL ids = *idp; 161 | /* Too big? */ 162 | if (ids[0] >= ids[-1]) { 163 | if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) 164 | return ENOMEM; 165 | ids = *idp; 166 | } 167 | ids[0]++; 168 | ids[ids[0]] = id; 169 | return 0; 170 | } 171 | 172 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) 173 | { 174 | MDB_IDL ids = *idp; 175 | /* Too big? */ 176 | if (ids[0] + app[0] >= ids[-1]) { 177 | if (mdb_midl_grow(idp, app[0])) 178 | return ENOMEM; 179 | ids = *idp; 180 | } 181 | memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); 182 | ids[0] += app[0]; 183 | return 0; 184 | } 185 | 186 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) 187 | { 188 | MDB_ID *ids = *idp, len = ids[0]; 189 | /* Too big? */ 190 | if (len + n > ids[-1]) { 191 | if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) 192 | return ENOMEM; 193 | ids = *idp; 194 | } 195 | ids[0] = len + n; 196 | ids += len; 197 | while (n) 198 | ids[n--] = id++; 199 | return 0; 200 | } 201 | 202 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) 203 | { 204 | MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; 205 | idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ 206 | old_id = idl[j]; 207 | while (i) { 208 | merge_id = merge[i--]; 209 | for (; old_id < merge_id; old_id = idl[--j]) 210 | idl[k--] = old_id; 211 | idl[k--] = merge_id; 212 | } 213 | idl[0] = total; 214 | } 215 | 216 | /* Quicksort + Insertion sort for small arrays */ 217 | 218 | #define SMALL 8 219 | #define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } 220 | 221 | void 222 | mdb_midl_sort( MDB_IDL ids ) 223 | { 224 | /* Max possible depth of int-indexed tree * 2 items/level */ 225 | int istack[sizeof(int)*CHAR_BIT * 2]; 226 | int i,j,k,l,ir,jstack; 227 | MDB_ID a, itmp; 228 | 229 | ir = (int)ids[0]; 230 | l = 1; 231 | jstack = 0; 232 | for(;;) { 233 | if (ir - l < SMALL) { /* Insertion sort */ 234 | for (j=l+1;j<=ir;j++) { 235 | a = ids[j]; 236 | for (i=j-1;i>=1;i--) { 237 | if (ids[i] >= a) break; 238 | ids[i+1] = ids[i]; 239 | } 240 | ids[i+1] = a; 241 | } 242 | if (jstack == 0) break; 243 | ir = istack[jstack--]; 244 | l = istack[jstack--]; 245 | } else { 246 | k = (l + ir) >> 1; /* Choose median of left, center, right */ 247 | MIDL_SWAP(ids[k], ids[l+1]); 248 | if (ids[l] < ids[ir]) { 249 | MIDL_SWAP(ids[l], ids[ir]); 250 | } 251 | if (ids[l+1] < ids[ir]) { 252 | MIDL_SWAP(ids[l+1], ids[ir]); 253 | } 254 | if (ids[l] < ids[l+1]) { 255 | MIDL_SWAP(ids[l], ids[l+1]); 256 | } 257 | i = l+1; 258 | j = ir; 259 | a = ids[l+1]; 260 | for(;;) { 261 | do i++; while(ids[i] > a); 262 | do j--; while(ids[j] < a); 263 | if (j < i) break; 264 | MIDL_SWAP(ids[i],ids[j]); 265 | } 266 | ids[l+1] = ids[j]; 267 | ids[j] = a; 268 | jstack += 2; 269 | if (ir-i+1 >= j-l) { 270 | istack[jstack] = ir; 271 | istack[jstack-1] = i; 272 | ir = j-1; 273 | } else { 274 | istack[jstack] = j-1; 275 | istack[jstack-1] = l; 276 | l = i; 277 | } 278 | } 279 | } 280 | } 281 | 282 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) 283 | { 284 | /* 285 | * binary search of id in ids 286 | * if found, returns position of id 287 | * if not found, returns first position greater than id 288 | */ 289 | unsigned base = 0; 290 | unsigned cursor = 1; 291 | int val = 0; 292 | unsigned n = (unsigned)ids[0].mid; 293 | 294 | while( 0 < n ) { 295 | unsigned pivot = n >> 1; 296 | cursor = base + pivot + 1; 297 | val = CMP( id, ids[cursor].mid ); 298 | 299 | if( val < 0 ) { 300 | n = pivot; 301 | 302 | } else if ( val > 0 ) { 303 | base = cursor; 304 | n -= pivot + 1; 305 | 306 | } else { 307 | return cursor; 308 | } 309 | } 310 | 311 | if( val > 0 ) { 312 | ++cursor; 313 | } 314 | return cursor; 315 | } 316 | 317 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) 318 | { 319 | unsigned x, i; 320 | 321 | x = mdb_mid2l_search( ids, id->mid ); 322 | 323 | if( x < 1 ) { 324 | /* internal error */ 325 | return -2; 326 | } 327 | 328 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) { 329 | /* duplicate */ 330 | return -1; 331 | } 332 | 333 | if ( ids[0].mid >= MDB_IDL_UM_MAX ) { 334 | /* too big */ 335 | return -2; 336 | 337 | } else { 338 | /* insert id */ 339 | ids[0].mid++; 340 | for (i=(unsigned)ids[0].mid; i>x; i--) 341 | ids[i] = ids[i-1]; 342 | ids[x] = *id; 343 | } 344 | 345 | return 0; 346 | } 347 | 348 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) 349 | { 350 | /* Too big? */ 351 | if (ids[0].mid >= MDB_IDL_UM_MAX) { 352 | return -2; 353 | } 354 | ids[0].mid++; 355 | ids[ids[0].mid] = *id; 356 | return 0; 357 | } 358 | 359 | /** @} */ 360 | /** @} */ 361 | -------------------------------------------------------------------------------- /midl.h: -------------------------------------------------------------------------------- 1 | /** @file midl.h 2 | * @brief LMDB ID List header file. 3 | * 4 | * This file was originally part of back-bdb but has been 5 | * modified for use in libmdb. Most of the macros defined 6 | * in this file are unused, just left over from the original. 7 | * 8 | * This file is only used internally in libmdb and its definitions 9 | * are not exposed publicly. 10 | */ 11 | /* $OpenLDAP$ */ 12 | /* This work is part of OpenLDAP Software . 13 | * 14 | * Copyright 2000-2014 The OpenLDAP Foundation. 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted only as authorized by the OpenLDAP 19 | * Public License. 20 | * 21 | * A copy of this license is available in the file LICENSE in the 22 | * top-level directory of the distribution or, alternatively, at 23 | * . 24 | */ 25 | 26 | #ifndef _MDB_MIDL_H_ 27 | #define _MDB_MIDL_H_ 28 | 29 | #include 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | /** @defgroup internal LMDB Internals 36 | * @{ 37 | */ 38 | 39 | /** @defgroup idls ID List Management 40 | * @{ 41 | */ 42 | /** A generic unsigned ID number. These were entryIDs in back-bdb. 43 | * Preferably it should have the same size as a pointer. 44 | */ 45 | typedef size_t MDB_ID; 46 | 47 | /** An IDL is an ID List, a sorted array of IDs. The first 48 | * element of the array is a counter for how many actual 49 | * IDs are in the list. In the original back-bdb code, IDLs are 50 | * sorted in ascending order. For libmdb IDLs are sorted in 51 | * descending order. 52 | */ 53 | typedef MDB_ID *MDB_IDL; 54 | 55 | /* IDL sizes - likely should be even bigger 56 | * limiting factors: sizeof(ID), thread stack size 57 | */ 58 | #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ 59 | #define MDB_IDL_DB_SIZE (1< 9 | #include 10 | #include "lmdb.h" 11 | */ 12 | import "C" 13 | 14 | import ( 15 | "math" 16 | "runtime" 17 | "unsafe" 18 | ) 19 | 20 | // DBIOpen Database Flags 21 | const ( 22 | REVERSEKEY = C.MDB_REVERSEKEY // use reverse string keys 23 | DUPSORT = C.MDB_DUPSORT // use sorted duplicates 24 | INTEGERKEY = C.MDB_INTEGERKEY // numeric keys in native byte order. The keys must all be of the same size. 25 | DUPFIXED = C.MDB_DUPFIXED // with DUPSORT, sorted dup items have fixed size 26 | INTEGERDUP = C.MDB_INTEGERDUP // with DUPSORT, dups are numeric in native byte order 27 | REVERSEDUP = C.MDB_REVERSEDUP // with DUPSORT, use reverse string dups 28 | CREATE = C.MDB_CREATE // create DB if not already existing 29 | ) 30 | 31 | // put flags 32 | const ( 33 | NODUPDATA = C.MDB_NODUPDATA 34 | NOOVERWRITE = C.MDB_NOOVERWRITE 35 | RESERVE = C.MDB_RESERVE 36 | APPEND = C.MDB_APPEND 37 | APPENDDUP = C.MDB_APPENDDUP 38 | ) 39 | 40 | // Txn is Opaque structure for a transaction handle. 41 | // All database operations require a transaction handle. 42 | // Transactions may be read-only or read-write. 43 | type Txn struct { 44 | _txn *C.MDB_txn 45 | } 46 | 47 | func (env *Env) BeginTxn(parent *Txn, flags uint) (*Txn, error) { 48 | var _txn *C.MDB_txn 49 | var ptxn *C.MDB_txn 50 | if parent == nil { 51 | ptxn = nil 52 | } else { 53 | ptxn = parent._txn 54 | } 55 | if flags&RDONLY == 0 { 56 | runtime.LockOSThread() 57 | } 58 | ret := C.mdb_txn_begin(env._env, ptxn, C.uint(flags), &_txn) 59 | if ret != SUCCESS { 60 | runtime.UnlockOSThread() 61 | return nil, errno(ret) 62 | } 63 | return &Txn{_txn}, nil 64 | } 65 | 66 | func (txn *Txn) Commit() error { 67 | ret := C.mdb_txn_commit(txn._txn) 68 | runtime.UnlockOSThread() 69 | // The transaction handle is freed if there was no error 70 | if ret == C.MDB_SUCCESS { 71 | txn._txn = nil 72 | } 73 | return errno(ret) 74 | } 75 | 76 | func (txn *Txn) Abort() { 77 | if txn._txn == nil { 78 | return 79 | } 80 | C.mdb_txn_abort(txn._txn) 81 | runtime.UnlockOSThread() 82 | // The transaction handle is always freed. 83 | txn._txn = nil 84 | } 85 | 86 | func (txn *Txn) Reset() { 87 | C.mdb_txn_reset(txn._txn) 88 | } 89 | 90 | func (txn *Txn) Renew() error { 91 | ret := C.mdb_txn_renew(txn._txn) 92 | return errno(ret) 93 | } 94 | 95 | func (txn *Txn) DBIOpen(name *string, flags uint) (DBI, error) { 96 | var _dbi C.MDB_dbi 97 | var cname *C.char 98 | if name == nil { 99 | cname = nil 100 | } else { 101 | cname = C.CString(*name) 102 | defer C.free(unsafe.Pointer(cname)) 103 | } 104 | ret := C.mdb_dbi_open(txn._txn, cname, C.uint(flags), &_dbi) 105 | if ret != SUCCESS { 106 | return DBI(math.NaN()), errno(ret) 107 | } 108 | return DBI(_dbi), nil 109 | } 110 | 111 | func (txn *Txn) Stat(dbi DBI) (*Stat, error) { 112 | var _stat C.MDB_stat 113 | ret := C.mdb_stat(txn._txn, C.MDB_dbi(dbi), &_stat) 114 | if ret != SUCCESS { 115 | return nil, errno(ret) 116 | } 117 | stat := Stat{PSize: uint(_stat.ms_psize), 118 | Depth: uint(_stat.ms_depth), 119 | BranchPages: uint64(_stat.ms_branch_pages), 120 | LeafPages: uint64(_stat.ms_leaf_pages), 121 | OverflowPages: uint64(_stat.ms_overflow_pages), 122 | Entries: uint64(_stat.ms_entries)} 123 | return &stat, nil 124 | } 125 | 126 | func (txn *Txn) Drop(dbi DBI, del int) error { 127 | ret := C.mdb_drop(txn._txn, C.MDB_dbi(dbi), C.int(del)) 128 | return errno(ret) 129 | } 130 | 131 | func (txn *Txn) Get(dbi DBI, key []byte) ([]byte, error) { 132 | val, err := txn.GetVal(dbi, key) 133 | if err != nil { 134 | return nil, err 135 | } 136 | return val.Bytes(), nil 137 | } 138 | 139 | func (txn *Txn) GetVal(dbi DBI, key []byte) (Val, error) { 140 | ckey := Wrap(key) 141 | var cval Val 142 | ret := C.mdb_get(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval)) 143 | return cval, errno(ret) 144 | } 145 | 146 | func (txn *Txn) Put(dbi DBI, key []byte, val []byte, flags uint) error { 147 | ckey := Wrap(key) 148 | cval := Wrap(val) 149 | ret := C.mdb_put(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval), C.uint(flags)) 150 | return errno(ret) 151 | } 152 | 153 | func (txn *Txn) Del(dbi DBI, key, val []byte) error { 154 | ckey := Wrap(key) 155 | if val == nil { 156 | ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), nil) 157 | return errno(ret) 158 | } 159 | cval := Wrap(val) 160 | ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval)) 161 | return errno(ret) 162 | } 163 | 164 | type Cursor struct { 165 | _cursor *C.MDB_cursor 166 | } 167 | 168 | func (txn *Txn) CursorOpen(dbi DBI) (*Cursor, error) { 169 | var _cursor *C.MDB_cursor 170 | ret := C.mdb_cursor_open(txn._txn, C.MDB_dbi(dbi), &_cursor) 171 | if ret != SUCCESS { 172 | return nil, errno(ret) 173 | } 174 | return &Cursor{_cursor}, nil 175 | } 176 | 177 | func (txn *Txn) CursorRenew(cursor *Cursor) error { 178 | ret := C.mdb_cursor_renew(txn._txn, cursor._cursor) 179 | return errno(ret) 180 | } 181 | 182 | /* 183 | type CmpFunc func(a, b []byte) int 184 | 185 | func (txn *Txn) SetCompare(dbi DBI, cmp CmpFunc) error { 186 | f := func(a, b *C.MDB_val) C.int { 187 | ga := C.GoBytes(a.mv_data, C.int(a.mv_size)) 188 | gb := C.GoBytes(a.mv_data, C.int(a.mv_size)) 189 | return C.int(cmp(ga, gb)) 190 | } 191 | ret := C.mdb_set_compare(txn._txn, C.MDB_dbi(dbi), *unsafe.Pointer(&f)) 192 | return errno(ret) 193 | } 194 | */ 195 | // func (txn *Txn) SetDupSort(dbi DBI, comp *C.MDB_comp_func) error 196 | // func (txn *Txn) SetRelFunc(dbi DBI, rel *C.MDB_rel_func) error 197 | // func (txn *Txn) SetRelCtx(dbi DBI, void *) error 198 | -------------------------------------------------------------------------------- /val.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | /* 4 | #cgo CFLAGS: -pthread -W -Wall -Wno-unused-parameter -Wbad-function-cast -O2 -g 5 | #cgo CFLAGS: -I/usr/local 6 | 7 | #include 8 | #include 9 | #include "lmdb.h" 10 | */ 11 | import "C" 12 | 13 | import ( 14 | "reflect" 15 | "unsafe" 16 | ) 17 | 18 | // MDB_val 19 | type Val C.MDB_val 20 | 21 | // Create a Val that points to p's data. the Val's data must not be freed 22 | // manually and C references must not survive the garbage collection of p (and 23 | // the returned Val). 24 | func Wrap(p []byte) Val { 25 | if len(p) == 0 { 26 | return Val(C.MDB_val{}) 27 | } 28 | return Val(C.MDB_val{ 29 | mv_size: C.size_t(len(p)), 30 | mv_data: unsafe.Pointer(&p[0]), 31 | }) 32 | } 33 | 34 | // If val is nil, a empty slice is retured. 35 | func (val Val) Bytes() []byte { 36 | return C.GoBytes(val.mv_data, C.int(val.mv_size)) 37 | } 38 | 39 | // If val is nil, a empty slice is retured. 40 | func (val Val) BytesNoCopy() []byte { 41 | hdr := reflect.SliceHeader{ 42 | Data: uintptr(unsafe.Pointer(val.mv_data)), 43 | Len: int(val.mv_size), 44 | Cap: int(val.mv_size), 45 | } 46 | return *(*[]byte)(unsafe.Pointer(&hdr)) 47 | } 48 | 49 | // If val is nil, an empty string is returned. 50 | func (val Val) String() string { 51 | return C.GoStringN((*C.char)(val.mv_data), C.int(val.mv_size)) 52 | } 53 | -------------------------------------------------------------------------------- /val_test.go: -------------------------------------------------------------------------------- 1 | package mdb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestVal(t *testing.T) { 8 | orig := "hey hey" 9 | val := Wrap([]byte(orig)) 10 | 11 | s := val.String() 12 | if s != orig { 13 | t.Errorf("String() not the same as original data: %q", s) 14 | } 15 | 16 | p := val.Bytes() 17 | if string(p) != orig { 18 | t.Errorf("Bytes() not the same as original data: %q", p) 19 | } 20 | } 21 | 22 | func TestValNoCopy(t *testing.T) { 23 | orig := "hey hey" 24 | val := Wrap([]byte(orig)) 25 | 26 | s := val.String() 27 | if s != orig { 28 | t.Errorf("String() not the same as original data: %q", s) 29 | } 30 | 31 | p := val.BytesNoCopy() 32 | if string(p) != orig { 33 | t.Errorf("Bytes() not the same as original data: %q", p) 34 | } 35 | } 36 | --------------------------------------------------------------------------------