├── .github ├── renovate.json ├── versions │ └── go └── workflows │ └── go.yml ├── .gitignore ├── .travis ├── .gitignore └── Makefile ├── LICENSE ├── README.md ├── batch.go ├── cache.go ├── comparator.go ├── conv.go ├── db.go ├── doc.go ├── env.go ├── examples └── comparator_example.go ├── filterpolicy.go ├── go.mod ├── go.sum ├── iterator.go ├── leveldb_test.go ├── options.go └── version.go /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ], 5 | "postUpdateOptions": ["gomodTidy"] 6 | } 7 | -------------------------------------------------------------------------------- /.github/versions/go: -------------------------------------------------------------------------------- 1 | 1.13.3 2 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | push: 7 | branches: 8 | - master 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v1 18 | 19 | - name: Read Go versions 20 | run: echo "##[set-output name=go_version;]$(cat .github/versions/go)" 21 | id: go_versions 22 | 23 | - name: Set up Go 24 | uses: actions/setup-go@v1 25 | with: 26 | go-version: ${{ steps.go_versions.outputs.go_version }} 27 | id: go 28 | 29 | - name: Build and test 30 | run: cd .travis && make 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.6 4 | *.out 5 | _testmain.go 6 | _obj 7 | -------------------------------------------------------------------------------- /.travis/.gitignore: -------------------------------------------------------------------------------- 1 | archives 2 | root 3 | 4 | -------------------------------------------------------------------------------- /.travis/Makefile: -------------------------------------------------------------------------------- 1 | all: test 2 | 3 | LEVELDB_VERSION ?= v1.18 4 | SNAPPY_VERSION ?= 1.1.7 5 | 6 | export CFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/include 7 | export CXXFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/build 8 | export LDFLAGS ?= -L$(PWD)/root/snappy-$(SNAPPY_VERSION)/build 9 | export CGO_CFLAGS ?= -I$(PWD)/root/snappy-$(SNAPPY_VERSION)/build -I$(PWD)/root/leveldb/include 10 | export CGO_LDFLAGS ?= -L$(PWD)/root/snappy-$(SNAPPY_VERSION)/build -L$(PWD)/root/leveldb -lsnappy 11 | export GOPATH ?= $(PWD)/root/go 12 | export LD_LIBRARY_PATH := $(PWD)/root/snappy-$(SNAPPY_VERSION)/build:$(PWD)/root/leveldb:$(LD_LIBRARY_PATH) 13 | 14 | archives/snappy-$(SNAPPY_VERSION).tar.gz: archives 15 | curl -L https://github.com/google/snappy/archive/$(SNAPPY_VERSION).tar.gz > $@ 16 | 17 | archives: 18 | mkdir -v archives 19 | 20 | levigo: root/snappy-$(SNAPPY_VERSION)/STAMP root/leveldb/STAMP 21 | cd ../ && go get -d . 22 | cd ../ && go build . 23 | cd ../ && go test -test.v=true . 24 | 25 | root: 26 | mkdir -v root 27 | 28 | root/leveldb: root 29 | cd root && git clone https://github.com/google/leveldb.git 30 | 31 | root/leveldb/STAMP: root/leveldb root/snappy-$(SNAPPY_VERSION)/STAMP 32 | cd root/leveldb && git checkout $(LEVELDB_VERSION) 33 | $(MAKE) -C root/leveldb 34 | touch $@ 35 | 36 | root/snappy-$(SNAPPY_VERSION): archives/snappy-$(SNAPPY_VERSION).tar.gz root 37 | tar xzvf archives/snappy-$(SNAPPY_VERSION).tar.gz -C root 38 | 39 | root/snappy-$(SNAPPY_VERSION)/STAMP: root/snappy-$(SNAPPY_VERSION) 40 | mkdir -p root/snappy-$(SNAPPY_VERSION)/build && cd root/snappy-$(SNAPPY_VERSION)/build && cmake -DCMAKE_INSTALL_PREFIX:PATH=$(pwd) ../ && make 41 | touch $@ 42 | 43 | test: levigo 44 | 45 | clean: 46 | -rm -rf archives 47 | -rm -rf root 48 | 49 | .PHONY: levigo test 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Jeffrey M Hodges 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![GoDoc](https://godoc.org/github.com/jmhodges/levigo?status.svg)](https://godoc.org/github.com/jmhodges/levigo) 2 | 3 | # levigo 4 | 5 | levigo is a Go wrapper for LevelDB. 6 | 7 | The API has been godoc'ed and [is available on the 8 | web](http://godoc.org/github.com/jmhodges/levigo). 9 | 10 | Questions answered at `golang-nuts@googlegroups.com`. 11 | 12 | ## Building 13 | 14 | You'll need the shared library build of 15 | [LevelDB](http://code.google.com/p/leveldb/) installed on your machine. The 16 | current LevelDB will build it by default. 17 | 18 | The minimum version of LevelDB required is currently 1.7. If you require the 19 | use of an older version of LevelDB, see the [fork of levigo for LevelDB 20 | 1.4](https://github.com/jmhodges/levigo_leveldb_1.4). Prefer putting in the 21 | work to be up to date as LevelDB moves very quickly. 22 | 23 | Now, if you build LevelDB and put the shared library and headers in one of the 24 | standard places for your OS, you'll be able to simply run: 25 | 26 | go get github.com/jmhodges/levigo 27 | 28 | But, suppose you put the shared LevelDB library somewhere weird like 29 | /path/to/lib and the headers were installed in /path/to/include. To install 30 | levigo remotely, you'll run: 31 | 32 | CGO_CFLAGS="-I/path/to/leveldb/include" CGO_LDFLAGS="-L/path/to/leveldb/lib" go get github.com/jmhodges/levigo 33 | 34 | and there you go. 35 | 36 | In order to build with snappy, you'll have to explicitly add "-lsnappy" to the 37 | `CGO_LDFLAGS`. Supposing that both snappy and leveldb are in weird places, 38 | you'll run something like: 39 | 40 | CGO_CFLAGS="-I/path/to/leveldb/include -I/path/to/snappy/include" 41 | CGO_LDFLAGS="-L/path/to/leveldb/lib -L/path/to/snappy/lib -lsnappy" go get github.com/jmhodges/levigo 42 | 43 | (and make sure the -lsnappy is after the snappy library path!). 44 | 45 | Of course, these same rules apply when doing `go build`, as well. 46 | 47 | ## Caveats 48 | 49 | Comparators and WriteBatch iterators must be written in C in your own 50 | library. This seems like a pain in the ass, but remember that you'll have the 51 | LevelDB C API available to your in your client package when you import levigo. 52 | 53 | An example of writing your own Comparator can be found in 54 | . 55 | 56 | ## Status 57 | 58 | Build: [![Build Status](https://travis-ci.org/jmhodges/levigo.svg)](https://travis-ci.org/jmhodges/levigo) 59 | 60 | Documentation: [![GoDoc](https://godoc.org/github.com/jmhodges/levigo?status.svg)](https://godoc.org/github.com/jmhodges/levigo) 61 | 62 | Lint: [Go Lint](http://go-lint.appspot.com/github.com/jmhodges/levigo) 63 | -------------------------------------------------------------------------------- /batch.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb -lstdc++ 4 | // #include "leveldb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // WriteBatch is a batching of Puts, and Deletes to be written atomically to a 12 | // database. A WriteBatch is written when passed to DB.Write. 13 | // 14 | // To prevent memory leaks, call Close when the program no longer needs the 15 | // WriteBatch object. 16 | type WriteBatch struct { 17 | wbatch *C.leveldb_writebatch_t 18 | } 19 | 20 | // NewWriteBatch creates a fully allocated WriteBatch. 21 | func NewWriteBatch() *WriteBatch { 22 | wb := C.leveldb_writebatch_create() 23 | return &WriteBatch{wb} 24 | } 25 | 26 | // Close releases the underlying memory of a WriteBatch. 27 | func (w *WriteBatch) Close() { 28 | C.leveldb_writebatch_destroy(w.wbatch) 29 | } 30 | 31 | // Put places a key-value pair into the WriteBatch for writing later. 32 | // 33 | // Both the key and value byte slices may be reused as WriteBatch takes a copy 34 | // of them before returning. 35 | // 36 | func (w *WriteBatch) Put(key, value []byte) { 37 | // leveldb_writebatch_put, and _delete call memcpy() (by way of 38 | // Memtable::Add) when called, so we do not need to worry about these 39 | // []byte being reclaimed by GC. 40 | var k, v *C.char 41 | if len(key) != 0 { 42 | k = (*C.char)(unsafe.Pointer(&key[0])) 43 | } 44 | if len(value) != 0 { 45 | v = (*C.char)(unsafe.Pointer(&value[0])) 46 | } 47 | 48 | lenk := len(key) 49 | lenv := len(value) 50 | 51 | C.leveldb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv)) 52 | } 53 | 54 | // Delete queues a deletion of the data at key to be deleted later. 55 | // 56 | // The key byte slice may be reused safely. Delete takes a copy of 57 | // them before returning. 58 | func (w *WriteBatch) Delete(key []byte) { 59 | C.leveldb_writebatch_delete(w.wbatch, 60 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) 61 | } 62 | 63 | // Clear removes all the enqueued Put and Deletes in the WriteBatch. 64 | func (w *WriteBatch) Clear() { 65 | C.leveldb_writebatch_clear(w.wbatch) 66 | } 67 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include 5 | // #include "leveldb/c.h" 6 | import "C" 7 | 8 | // Cache is a cache used to store data read from data in memory. 9 | // 10 | // Typically, NewLRUCache is all you will need, but advanced users may 11 | // implement their own *C.leveldb_cache_t and create a Cache. 12 | // 13 | // To prevent memory leaks, a Cache must have Close called on it when it is 14 | // no longer needed by the program. Note: if the process is shutting down, 15 | // this may not be necessary and could be avoided to shorten shutdown time. 16 | type Cache struct { 17 | Cache *C.leveldb_cache_t 18 | } 19 | 20 | // NewLRUCache creates a new Cache object with the capacity given. 21 | // 22 | // To prevent memory leaks, Close should be called on the Cache when the 23 | // program no longer needs it. Note: if the process is shutting down, this may 24 | // not be necessary and could be avoided to shorten shutdown time. 25 | func NewLRUCache(capacity int) *Cache { 26 | return &Cache{C.leveldb_cache_create_lru(C.size_t(capacity))} 27 | } 28 | 29 | // Close deallocates the underlying memory of the Cache object. 30 | func (c *Cache) Close() { 31 | C.leveldb_cache_destroy(c.Cache) 32 | } 33 | -------------------------------------------------------------------------------- /comparator.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include "leveldb/c.h" 5 | import "C" 6 | 7 | // DestroyComparator deallocates a *C.leveldb_comparator_t. 8 | // 9 | // This is provided as a convienience to advanced users that have implemented 10 | // their own comparators in C in their own code. 11 | func DestroyComparator(cmp *C.leveldb_comparator_t) { 12 | C.leveldb_comparator_destroy(cmp) 13 | } 14 | -------------------------------------------------------------------------------- /conv.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #include "leveldb/c.h" 4 | import "C" 5 | 6 | func boolToUchar(b bool) C.uchar { 7 | uc := C.uchar(0) 8 | if b { 9 | uc = C.uchar(1) 10 | } 11 | return uc 12 | } 13 | 14 | func ucharToBool(uc C.uchar) bool { 15 | if uc == C.uchar(0) { 16 | return false 17 | } 18 | return true 19 | } 20 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | /* 4 | #cgo LDFLAGS: -lleveldb 5 | #include 6 | #include "leveldb/c.h" 7 | 8 | // This function exists only to clean up lack-of-const warnings when 9 | // leveldb_approximate_sizes is called from Go-land. 10 | void levigo_leveldb_approximate_sizes( 11 | leveldb_t* db, 12 | int num_ranges, 13 | char** range_start_key, const size_t* range_start_key_len, 14 | char** range_limit_key, const size_t* range_limit_key_len, 15 | uint64_t* sizes) { 16 | leveldb_approximate_sizes(db, 17 | num_ranges, 18 | (const char* const*)range_start_key, 19 | range_start_key_len, 20 | (const char* const*)range_limit_key, 21 | range_limit_key_len, 22 | sizes); 23 | } 24 | */ 25 | import "C" 26 | 27 | import ( 28 | "errors" 29 | "unsafe" 30 | ) 31 | 32 | // DatabaseError wraps general internal LevelDB errors for user consumption. 33 | type DatabaseError string 34 | 35 | func (e DatabaseError) Error() string { 36 | return "levigo: " + string(e) 37 | } 38 | 39 | // ErrDBClosed is returned by DB.Close when its been called previously. 40 | var ErrDBClosed = errors.New("database is closed") 41 | 42 | // DB is a reusable handle to a LevelDB database on disk, created by Open. 43 | // 44 | // To avoid memory and file descriptor leaks, call Close when the process no 45 | // longer needs the handle. Calls to any DB method made after Close will 46 | // panic. 47 | // 48 | // The DB instance may be shared between goroutines. The usual data race 49 | // conditions will occur if the same key is written to from more than one, of 50 | // course. 51 | type DB struct { 52 | Ldb *C.leveldb_t 53 | 54 | // TLDR: Closed is not racey, it's a best attempt. If `-race` says it's racey, 55 | // then it's your code that is racey, not levigo. 56 | // 57 | // This indicates if the DB is closed or not. LevelDB provides it's own closed 58 | // detection that appears in the form of a sigtrap panic, which can be quite 59 | // confusing. So rather than users hit that, we attempt to give them a better 60 | // panic by checking this flag in functions that LevelDB would have done 61 | // it's own panic. This is not protected by a mutex because it's a best case 62 | // attempt to catch invalid usage. If access in racey and gets to LevelDB, 63 | // so be it, the user will see the sigtrap panic. 64 | // Because LevelDB has it's own mutex to detect the usage, we didn't want to 65 | // put another one up here and drive performance down even more. 66 | // So if you use `-race` and it says closed is racey, it's your code that is 67 | // racey, not levigo. 68 | closed bool 69 | } 70 | 71 | // Range is a range of keys in the database. GetApproximateSizes calls with it 72 | // begin at the key Start and end right before the key Limit. 73 | type Range struct { 74 | Start []byte 75 | Limit []byte 76 | } 77 | 78 | // Snapshot provides a consistent view of read operations in a DB. 79 | // 80 | // Snapshot is used in read operations by setting it on a 81 | // ReadOptions. Snapshots are created by calling DB.NewSnapshot. 82 | // 83 | // To prevent memory leaks and resource strain in the database, the snapshot 84 | // returned must be released with DB.ReleaseSnapshot method on the DB that 85 | // created it. 86 | type Snapshot struct { 87 | snap *C.leveldb_snapshot_t 88 | } 89 | 90 | // Open opens a database. 91 | // 92 | // Creating a new database is done by calling SetCreateIfMissing(true) on the 93 | // Options passed to Open. 94 | // 95 | // It is usually wise to set a Cache object on the Options with SetCache to 96 | // keep recently used data from that database in memory. 97 | func Open(dbname string, o *Options) (*DB, error) { 98 | var errStr *C.char 99 | ldbname := C.CString(dbname) 100 | defer C.free(unsafe.Pointer(ldbname)) 101 | 102 | leveldb := C.leveldb_open(o.Opt, ldbname, &errStr) 103 | if errStr != nil { 104 | gs := C.GoString(errStr) 105 | C.leveldb_free(unsafe.Pointer(errStr)) 106 | return nil, DatabaseError(gs) 107 | } 108 | return &DB{leveldb, false}, nil 109 | } 110 | 111 | // DestroyDatabase removes a database entirely, removing everything from the 112 | // filesystem. 113 | func DestroyDatabase(dbname string, o *Options) error { 114 | var errStr *C.char 115 | ldbname := C.CString(dbname) 116 | defer C.free(unsafe.Pointer(ldbname)) 117 | 118 | C.leveldb_destroy_db(o.Opt, ldbname, &errStr) 119 | if errStr != nil { 120 | gs := C.GoString(errStr) 121 | C.leveldb_free(unsafe.Pointer(errStr)) 122 | return DatabaseError(gs) 123 | } 124 | return nil 125 | } 126 | 127 | // RepairDatabase attempts to repair a database. 128 | // 129 | // If the database is unrepairable, an error is returned. 130 | func RepairDatabase(dbname string, o *Options) error { 131 | var errStr *C.char 132 | ldbname := C.CString(dbname) 133 | defer C.free(unsafe.Pointer(ldbname)) 134 | 135 | C.leveldb_repair_db(o.Opt, ldbname, &errStr) 136 | if errStr != nil { 137 | gs := C.GoString(errStr) 138 | C.leveldb_free(unsafe.Pointer(errStr)) 139 | return DatabaseError(gs) 140 | } 141 | return nil 142 | } 143 | 144 | // Put writes data associated with a key to the database. 145 | // 146 | // If a nil []byte is passed in as value, it will be returned by Get 147 | // as an zero-length slice. The WriteOptions passed in can be reused 148 | // by multiple calls to this and if the WriteOptions is left unchanged. 149 | // 150 | // The key and value byte slices may be reused safely. Put takes a copy of 151 | // them before returning. 152 | func (db *DB) Put(wo *WriteOptions, key, value []byte) error { 153 | if db.closed { 154 | panic(ErrDBClosed) 155 | } 156 | 157 | var errStr *C.char 158 | // leveldb_put, _get, and _delete call memcpy() (by way of Memtable::Add) 159 | // when called, so we do not need to worry about these []byte being 160 | // reclaimed by GC. 161 | var k, v *C.char 162 | if len(key) != 0 { 163 | k = (*C.char)(unsafe.Pointer(&key[0])) 164 | } 165 | if len(value) != 0 { 166 | v = (*C.char)(unsafe.Pointer(&value[0])) 167 | } 168 | 169 | lenk := len(key) 170 | lenv := len(value) 171 | C.leveldb_put( 172 | db.Ldb, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr) 173 | 174 | if errStr != nil { 175 | gs := C.GoString(errStr) 176 | C.leveldb_free(unsafe.Pointer(errStr)) 177 | return DatabaseError(gs) 178 | } 179 | return nil 180 | } 181 | 182 | // Get returns the data associated with the key from the database. 183 | // 184 | // If the key does not exist in the database, a nil []byte is returned. If the 185 | // key does exist, but the data is zero-length in the database, a zero-length 186 | // []byte will be returned. 187 | // 188 | // The key byte slice may be reused safely. Get takes a copy of 189 | // them before returning. 190 | func (db *DB) Get(ro *ReadOptions, key []byte) ([]byte, error) { 191 | if db.closed { 192 | panic(ErrDBClosed) 193 | } 194 | 195 | var errStr *C.char 196 | var vallen C.size_t 197 | var k *C.char 198 | if len(key) != 0 { 199 | k = (*C.char)(unsafe.Pointer(&key[0])) 200 | } 201 | 202 | value := C.leveldb_get( 203 | db.Ldb, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) 204 | 205 | if errStr != nil { 206 | gs := C.GoString(errStr) 207 | C.leveldb_free(unsafe.Pointer(errStr)) 208 | return nil, DatabaseError(gs) 209 | } 210 | 211 | if value == nil { 212 | return nil, nil 213 | } 214 | 215 | defer C.leveldb_free(unsafe.Pointer(value)) 216 | return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil 217 | } 218 | 219 | // Delete removes the data associated with the key from the database. 220 | // 221 | // The key byte slice may be reused safely. Delete takes a copy of 222 | // them before returning. The WriteOptions passed in can be reused by 223 | // multiple calls to this and if the WriteOptions is left unchanged. 224 | func (db *DB) Delete(wo *WriteOptions, key []byte) error { 225 | if db.closed { 226 | panic(ErrDBClosed) 227 | } 228 | 229 | var errStr *C.char 230 | var k *C.char 231 | if len(key) != 0 { 232 | k = (*C.char)(unsafe.Pointer(&key[0])) 233 | } 234 | 235 | C.leveldb_delete( 236 | db.Ldb, wo.Opt, k, C.size_t(len(key)), &errStr) 237 | 238 | if errStr != nil { 239 | gs := C.GoString(errStr) 240 | C.leveldb_free(unsafe.Pointer(errStr)) 241 | return DatabaseError(gs) 242 | } 243 | return nil 244 | } 245 | 246 | // Write atomically writes a WriteBatch to disk. The WriteOptions 247 | // passed in can be reused by multiple calls to this and other methods. 248 | func (db *DB) Write(wo *WriteOptions, w *WriteBatch) error { 249 | if db.closed { 250 | panic(ErrDBClosed) 251 | } 252 | 253 | var errStr *C.char 254 | C.leveldb_write(db.Ldb, wo.Opt, w.wbatch, &errStr) 255 | if errStr != nil { 256 | gs := C.GoString(errStr) 257 | C.leveldb_free(unsafe.Pointer(errStr)) 258 | return DatabaseError(gs) 259 | } 260 | return nil 261 | } 262 | 263 | // NewIterator returns an Iterator over the the database that uses the 264 | // ReadOptions given. 265 | // 266 | // Often, this is used for large, offline bulk reads while serving live 267 | // traffic. In that case, it may be wise to disable caching so that the data 268 | // processed by the returned Iterator does not displace the already cached 269 | // data. This can be done by calling SetFillCache(false) on the ReadOptions 270 | // before passing it here. 271 | // 272 | // Similarly, ReadOptions.SetSnapshot is also useful. 273 | // 274 | // The ReadOptions passed in can be reused by multiple calls to this 275 | // and other methods if the ReadOptions is left unchanged. 276 | func (db *DB) NewIterator(ro *ReadOptions) *Iterator { 277 | if db.closed { 278 | panic(ErrDBClosed) 279 | } 280 | 281 | it := C.leveldb_create_iterator(db.Ldb, ro.Opt) 282 | return &Iterator{Iter: it} 283 | } 284 | 285 | // GetApproximateSizes returns the approximate number of bytes of file system 286 | // space used by one or more key ranges. 287 | // 288 | // The keys counted will begin at Range.Start and end on the key before 289 | // Range.Limit. 290 | func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { 291 | starts := make([]*C.char, len(ranges)) 292 | limits := make([]*C.char, len(ranges)) 293 | startLens := make([]C.size_t, len(ranges)) 294 | limitLens := make([]C.size_t, len(ranges)) 295 | for i, r := range ranges { 296 | starts[i] = C.CString(string(r.Start)) 297 | startLens[i] = C.size_t(len(r.Start)) 298 | limits[i] = C.CString(string(r.Limit)) 299 | limitLens[i] = C.size_t(len(r.Limit)) 300 | } 301 | sizes := make([]uint64, len(ranges)) 302 | numranges := C.int(len(ranges)) 303 | startsPtr := &starts[0] 304 | limitsPtr := &limits[0] 305 | startLensPtr := &startLens[0] 306 | limitLensPtr := &limitLens[0] 307 | sizesPtr := (*C.uint64_t)(&sizes[0]) 308 | C.levigo_leveldb_approximate_sizes( 309 | db.Ldb, numranges, startsPtr, startLensPtr, 310 | limitsPtr, limitLensPtr, sizesPtr) 311 | for i := range ranges { 312 | C.free(unsafe.Pointer(starts[i])) 313 | C.free(unsafe.Pointer(limits[i])) 314 | } 315 | return sizes 316 | } 317 | 318 | // PropertyValue returns the value of a database property. 319 | // 320 | // Examples of properties include "leveldb.stats", "leveldb.sstables", 321 | // and "leveldb.num-files-at-level0". 322 | func (db *DB) PropertyValue(propName string) string { 323 | if db.closed { 324 | panic(ErrDBClosed) 325 | } 326 | 327 | cname := C.CString(propName) 328 | value := C.GoString(C.leveldb_property_value(db.Ldb, cname)) 329 | C.free(unsafe.Pointer(cname)) 330 | return value 331 | } 332 | 333 | // NewSnapshot creates a new snapshot of the database. 334 | // 335 | // The Snapshot, when used in a ReadOptions, provides a consistent 336 | // view of state of the database at the the snapshot was created. 337 | // 338 | // To prevent memory leaks and resource strain in the database, the snapshot 339 | // returned must be released with DB.ReleaseSnapshot method on the DB that 340 | // created it. 341 | // 342 | // See the LevelDB documentation for details. 343 | func (db *DB) NewSnapshot() *Snapshot { 344 | if db.closed { 345 | panic(ErrDBClosed) 346 | } 347 | 348 | return &Snapshot{C.leveldb_create_snapshot(db.Ldb)} 349 | } 350 | 351 | // ReleaseSnapshot removes the snapshot from the database's list of snapshots, 352 | // and deallocates it. 353 | func (db *DB) ReleaseSnapshot(snap *Snapshot) { 354 | if db.closed { 355 | panic(ErrDBClosed) 356 | } 357 | 358 | C.leveldb_release_snapshot(db.Ldb, snap.snap) 359 | } 360 | 361 | // CompactRange runs a manual compaction on the Range of keys given. This is 362 | // not likely to be needed for typical usage. 363 | func (db *DB) CompactRange(r Range) { 364 | if db.closed { 365 | panic(ErrDBClosed) 366 | } 367 | 368 | var start, limit *C.char 369 | if len(r.Start) != 0 { 370 | start = (*C.char)(unsafe.Pointer(&r.Start[0])) 371 | } 372 | if len(r.Limit) != 0 { 373 | limit = (*C.char)(unsafe.Pointer(&r.Limit[0])) 374 | } 375 | C.leveldb_compact_range( 376 | db.Ldb, start, C.size_t(len(r.Start)), limit, C.size_t(len(r.Limit))) 377 | } 378 | 379 | // Close closes the database, rendering it unusable for I/O, by deallocating 380 | // the underlying handle. 381 | // 382 | // Any attempts to use the DB after Close is called will panic. 383 | func (db *DB) Close() { 384 | if db.closed { 385 | return 386 | } 387 | 388 | db.closed = true 389 | C.leveldb_close(db.Ldb) 390 | } 391 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package levigo provides the ability to create and access LevelDB databases. 4 | 5 | levigo.Open opens and creates databases. 6 | 7 | opts := levigo.NewOptions() 8 | opts.SetCache(levigo.NewLRUCache(3<<30)) 9 | opts.SetCreateIfMissing(true) 10 | db, err := levigo.Open("/path/to/db", opts) 11 | 12 | The DB struct returned by Open provides DB.Get, DB.Put and DB.Delete to modify 13 | and query the database. 14 | 15 | ro := levigo.NewReadOptions() 16 | wo := levigo.NewWriteOptions() 17 | // if ro and wo are not used again, be sure to Close them. 18 | data, err := db.Get(ro, []byte("key")) 19 | ... 20 | err = db.Put(wo, []byte("anotherkey"), data) 21 | ... 22 | err = db.Delete(wo, []byte("key")) 23 | 24 | For bulk reads, use an Iterator. If you want to avoid disturbing your live 25 | traffic while doing the bulk read, be sure to call SetFillCache(false) on the 26 | ReadOptions you use when creating the Iterator. 27 | 28 | ro := levigo.NewReadOptions() 29 | ro.SetFillCache(false) 30 | it := db.NewIterator(ro) 31 | defer it.Close() 32 | for it.Seek(mykey); it.Valid(); it.Next() { 33 | munge(it.Key(), it.Value()) 34 | } 35 | if err := it.GetError(); err != nil { 36 | ... 37 | } 38 | 39 | Batched, atomic writes can be performed with a WriteBatch and 40 | DB.Write. 41 | 42 | wb := levigo.NewWriteBatch() 43 | // defer wb.Close or use wb.Clear and reuse. 44 | wb.Delete([]byte("removed")) 45 | wb.Put([]byte("added"), []byte("data")) 46 | wb.Put([]byte("anotheradded"), []byte("more")) 47 | err := db.Write(wo, wb) 48 | 49 | If your working dataset does not fit in memory, you'll want to add a bloom 50 | filter to your database. NewBloomFilter and Options.SetFilterPolicy is what 51 | you want. NewBloomFilter is amount of bits in the filter to use per key in 52 | your database. 53 | 54 | filter := levigo.NewBloomFilter(10) 55 | opts.SetFilterPolicy(filter) 56 | db, err := levigo.Open("/path/to/db", opts) 57 | 58 | If you're using a custom comparator in your code, be aware you may have to 59 | make your own filter policy object. 60 | 61 | This documentation is not a complete discussion of LevelDB. Please read the 62 | LevelDB documentation for information on 63 | its operation. You'll find lots of goodies there. 64 | */ 65 | package levigo 66 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include "leveldb/c.h" 5 | import "C" 6 | 7 | // Env is a system call environment used by a database. 8 | // 9 | // Typically, NewDefaultEnv is all you need. Advanced users may create their 10 | // own Env with a *C.leveldb_env_t of their own creation. 11 | // 12 | // To prevent memory leaks, an Env must have Close called on it when it is 13 | // no longer needed by the program. 14 | type Env struct { 15 | Env *C.leveldb_env_t 16 | } 17 | 18 | // NewDefaultEnv creates a default environment for use in an Options. 19 | // 20 | // To prevent memory leaks, the Env returned should be deallocated with 21 | // Close. 22 | func NewDefaultEnv() *Env { 23 | return &Env{C.leveldb_create_default_env()} 24 | } 25 | 26 | // Close deallocates the Env, freeing the underlying struct. 27 | func (env *Env) Close() { 28 | C.leveldb_env_destroy(env.Env) 29 | } 30 | -------------------------------------------------------------------------------- /examples/comparator_example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | #cgo LDFLAGS: -lleveldb 5 | #include 6 | #include 7 | 8 | static void CmpDestroy(void* arg) { } 9 | 10 | static int CmpCompare(void* arg, const char* a, size_t alen, 11 | const char* b, size_t blen) { 12 | int n = (alen < blen) ? alen : blen; 13 | int r = memcmp(a, b, n); 14 | if (r == 0) { 15 | if (alen < blen) r = -1; 16 | else if (alen > blen) r = +1; 17 | } 18 | return r; 19 | } 20 | 21 | static const char* CmpName(void* arg) { 22 | return "foo"; 23 | } 24 | 25 | static leveldb_comparator_t* CmpFooNew() { 26 | return leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName); 27 | } 28 | 29 | */ 30 | import "C" 31 | 32 | type Comparator struct { 33 | Comparator *C.leveldb_comparator_t 34 | } 35 | 36 | func NewFooComparator() *Comparator { 37 | return &Comparator{C.CmpFooNew()} 38 | } 39 | 40 | func (cmp *Comparator) Close() { 41 | C.leveldb_comparator_destroy(cmp.Comparator) 42 | } 43 | 44 | func main() { 45 | NewFooComparator().Close() 46 | } 47 | -------------------------------------------------------------------------------- /filterpolicy.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include 5 | // #include "leveldb/c.h" 6 | import "C" 7 | 8 | // FilterPolicy is a factory type that allows the LevelDB database to create a 9 | // filter, such as a bloom filter, that is stored in the sstables and used by 10 | // DB.Get to reduce reads. 11 | // 12 | // An instance of this struct may be supplied to Options when opening a 13 | // DB. Typical usage is to call NewBloomFilter to get an instance. 14 | // 15 | // To prevent memory leaks, a FilterPolicy must have Close called on it when 16 | // it is no longer needed by the program. 17 | type FilterPolicy struct { 18 | Policy *C.leveldb_filterpolicy_t 19 | } 20 | 21 | // NewBloomFilter creates a filter policy that will create a bloom filter when 22 | // necessary with the given number of bits per key. 23 | // 24 | // See the FilterPolicy documentation for more. 25 | func NewBloomFilter(bitsPerKey int) *FilterPolicy { 26 | policy := C.leveldb_filterpolicy_create_bloom(C.int(bitsPerKey)) 27 | return &FilterPolicy{policy} 28 | } 29 | 30 | // Close reaps the resources associated with this FilterPolicy. 31 | func (fp *FilterPolicy) Close() { 32 | C.leveldb_filterpolicy_destroy(fp.Policy) 33 | } 34 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jmhodges/levigo 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jmhodges/levigo/ed89ec741d960a3189615c8fbe92a0327b261eda/go.sum -------------------------------------------------------------------------------- /iterator.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include 5 | // #include "leveldb/c.h" 6 | import "C" 7 | 8 | import ( 9 | "unsafe" 10 | ) 11 | 12 | // IteratorError wraps general internal LevelDB iterator errors for user consumption. 13 | type IteratorError string 14 | 15 | func (e IteratorError) Error() string { 16 | return "levigo: " + string(e) 17 | } 18 | 19 | // Iterator is a read-only iterator through a LevelDB database. It provides a 20 | // way to seek to specific keys and iterate through the keyspace from that 21 | // point, as well as access the values of those keys. 22 | // 23 | // Care must be taken when using an Iterator. If the method Valid returns 24 | // false, calls to Key, Value, Next, and Prev will result in panics. However, 25 | // Seek, SeekToFirst, SeekToLast, GetError, Valid, and Close will still be 26 | // safe to call. 27 | // 28 | // GetError will only return an error in the event of a LevelDB error. It will 29 | // return a nil on iterators that are simply invalid. Given that behavior, 30 | // GetError is not a replacement for a Valid. 31 | // 32 | // A typical use looks like: 33 | // 34 | // db := levigo.Open(...) 35 | // 36 | // it := db.NewIterator(readOpts) 37 | // defer it.Close() 38 | // for it.Seek(mykey); it.Valid(); it.Next() { 39 | // useKeyAndValue(it.Key(), it.Value()) 40 | // } 41 | // if err := it.GetError() { 42 | // ... 43 | // } 44 | // 45 | // To prevent memory leaks, an Iterator must have Close called on it when it 46 | // is no longer needed by the program. 47 | type Iterator struct { 48 | Iter *C.leveldb_iterator_t 49 | } 50 | 51 | // Valid returns false only when an Iterator has iterated past either the 52 | // first or the last key in the database. 53 | func (it *Iterator) Valid() bool { 54 | return ucharToBool(C.leveldb_iter_valid(it.Iter)) 55 | } 56 | 57 | // Key returns a copy the key in the database the iterator currently holds. 58 | // 59 | // If Valid returns false, this method will panic. 60 | func (it *Iterator) Key() []byte { 61 | var klen C.size_t 62 | kdata := C.leveldb_iter_key(it.Iter, &klen) 63 | if kdata == nil { 64 | return nil 65 | } 66 | // Unlike DB.Get, the key, kdata, returned is not meant to be freed by the 67 | // client. It's a direct reference to data managed by the iterator_t 68 | // instead of a copy. So, we must not free it here but simply copy it 69 | // with GoBytes. 70 | return C.GoBytes(unsafe.Pointer(kdata), C.int(klen)) 71 | } 72 | 73 | // Value returns a copy of the value in the database the iterator currently 74 | // holds. 75 | // 76 | // If Valid returns false, this method will panic. 77 | func (it *Iterator) Value() []byte { 78 | var vlen C.size_t 79 | vdata := C.leveldb_iter_value(it.Iter, &vlen) 80 | if vdata == nil { 81 | return nil 82 | } 83 | // Unlike DB.Get, the value, vdata, returned is not meant to be freed by 84 | // the client. It's a direct reference to data managed by the iterator_t 85 | // instead of a copy. So, we must not free it here but simply copy it with 86 | // GoBytes. 87 | return C.GoBytes(unsafe.Pointer(vdata), C.int(vlen)) 88 | } 89 | 90 | // Next moves the iterator to the next sequential key in the database, as 91 | // defined by the Comparator in the ReadOptions used to create this Iterator. 92 | // 93 | // If Valid returns false, this method will panic. 94 | func (it *Iterator) Next() { 95 | C.leveldb_iter_next(it.Iter) 96 | } 97 | 98 | // Prev moves the iterator to the previous sequential key in the database, as 99 | // defined by the Comparator in the ReadOptions used to create this Iterator. 100 | // 101 | // If Valid returns false, this method will panic. 102 | func (it *Iterator) Prev() { 103 | C.leveldb_iter_prev(it.Iter) 104 | } 105 | 106 | // SeekToFirst moves the iterator to the first key in the database, as defined 107 | // by the Comparator in the ReadOptions used to create this Iterator. 108 | // 109 | // This method is safe to call when Valid returns false. 110 | func (it *Iterator) SeekToFirst() { 111 | C.leveldb_iter_seek_to_first(it.Iter) 112 | } 113 | 114 | // SeekToLast moves the iterator to the last key in the database, as defined 115 | // by the Comparator in the ReadOptions used to create this Iterator. 116 | // 117 | // This method is safe to call when Valid returns false. 118 | func (it *Iterator) SeekToLast() { 119 | C.leveldb_iter_seek_to_last(it.Iter) 120 | } 121 | 122 | // Seek moves the iterator the position of the key given or, if the key 123 | // doesn't exist, the next key that does exist in the database. If the key 124 | // doesn't exist, and there is no next key, the Iterator becomes invalid. 125 | // 126 | // This method is safe to call when Valid returns false. 127 | func (it *Iterator) Seek(key []byte) { 128 | C.leveldb_iter_seek(it.Iter, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) 129 | } 130 | 131 | // GetError returns an IteratorError from LevelDB if it had one during 132 | // iteration. 133 | // 134 | // This method is safe to call when Valid returns false. 135 | func (it *Iterator) GetError() error { 136 | var errStr *C.char 137 | C.leveldb_iter_get_error(it.Iter, &errStr) 138 | if errStr != nil { 139 | gs := C.GoString(errStr) 140 | C.leveldb_free(unsafe.Pointer(errStr)) 141 | return IteratorError(gs) 142 | } 143 | return nil 144 | } 145 | 146 | // Close deallocates the given Iterator, freeing the underlying C struct. 147 | func (it *Iterator) Close() { 148 | C.leveldb_iter_destroy(it.Iter) 149 | it.Iter = nil 150 | } 151 | -------------------------------------------------------------------------------- /leveldb_test.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func init() { 14 | rand.Seed(int64(time.Now().Nanosecond())) 15 | } 16 | 17 | // This testcase is a port of leveldb's c_test.c. 18 | func TestC(t *testing.T) { 19 | if GetLevelDBMajorVersion() <= 0 { 20 | t.Errorf("Major version cannot be less than zero") 21 | } 22 | 23 | dbname := tempDir(t) 24 | defer deleteDBDirectory(t, dbname) 25 | env := NewDefaultEnv() 26 | cache := NewLRUCache(1 << 20) 27 | 28 | options := NewOptions() 29 | // options.SetComparator(cmp) 30 | options.SetErrorIfExists(true) 31 | options.SetCache(cache) 32 | options.SetEnv(env) 33 | options.SetInfoLog(nil) 34 | options.SetWriteBufferSize(1 << 20) 35 | options.SetParanoidChecks(true) 36 | options.SetMaxOpenFiles(10) 37 | options.SetBlockSize(1024) 38 | options.SetBlockRestartInterval(8) 39 | options.SetCompression(NoCompression) 40 | 41 | roptions := NewReadOptions() 42 | roptions.SetVerifyChecksums(true) 43 | roptions.SetFillCache(false) 44 | 45 | woptions := NewWriteOptions() 46 | woptions.SetSync(true) 47 | 48 | _ = DestroyDatabase(dbname, options) 49 | 50 | db, err := Open(dbname, options) 51 | if err == nil { 52 | t.Errorf("Open on missing db should have failed") 53 | } 54 | 55 | options.SetCreateIfMissing(true) 56 | db, err = Open(dbname, options) 57 | if err != nil { 58 | t.Fatalf("Open failed: %v", err) 59 | } 60 | 61 | putKey := []byte("foo") 62 | putValue := []byte("hello") 63 | err = db.Put(woptions, putKey, putValue) 64 | if err != nil { 65 | t.Errorf("Put failed: %v", err) 66 | } 67 | 68 | CheckGet(t, "after Put", db, roptions, putKey, putValue) 69 | 70 | wb := NewWriteBatch() 71 | wb.Put([]byte("foo"), []byte("a")) 72 | wb.Clear() 73 | wb.Put([]byte("bar"), []byte("b")) 74 | wb.Put([]byte("box"), []byte("c")) 75 | wb.Delete([]byte("bar")) 76 | err = db.Write(woptions, wb) 77 | if err != nil { 78 | t.Errorf("Write batch failed: %v", err) 79 | } 80 | CheckGet(t, "after WriteBatch", db, roptions, []byte("foo"), []byte("hello")) 81 | CheckGet(t, "after WriteBatch", db, roptions, []byte("bar"), nil) 82 | CheckGet(t, "after WriteBatch", db, roptions, []byte("box"), []byte("c")) 83 | // TODO: WriteBatch iteration isn't easy. Suffers same problems as 84 | // Comparator. 85 | // wbiter := &TestWBIter{t: t} 86 | // wb.Iterate(wbiter) 87 | // if wbiter.pos != 3 { 88 | // t.Errorf("After Iterate, on the wrong pos: %d", wbiter.pos) 89 | // } 90 | wb.Close() 91 | 92 | iter := db.NewIterator(roptions) 93 | if iter.Valid() { 94 | t.Errorf("Read iterator should not be valid, yet") 95 | } 96 | iter.SeekToFirst() 97 | if !iter.Valid() { 98 | t.Errorf("Read iterator should be valid after seeking to first record") 99 | } 100 | CheckIter(t, iter, []byte("box"), []byte("c")) 101 | iter.Next() 102 | CheckIter(t, iter, []byte("foo"), []byte("hello")) 103 | iter.Prev() 104 | CheckIter(t, iter, []byte("box"), []byte("c")) 105 | iter.Prev() 106 | if iter.Valid() { 107 | t.Errorf("Read iterator should not be valid after go back past the first record") 108 | } 109 | iter.SeekToLast() 110 | CheckIter(t, iter, []byte("foo"), []byte("hello")) 111 | iter.Seek([]byte("b")) 112 | CheckIter(t, iter, []byte("box"), []byte("c")) 113 | if iter.GetError() != nil { 114 | t.Errorf("Read iterator has an error we didn't expect: %v", iter.GetError()) 115 | } 116 | iter.Close() 117 | 118 | // approximate sizes 119 | n := 20000 120 | for i := 0; i < n; i++ { 121 | keybuf := []byte(fmt.Sprintf("k%020d", i)) 122 | valbuf := []byte(fmt.Sprintf("v%020d", i)) 123 | err := db.Put(woptions, keybuf, valbuf) 124 | if err != nil { 125 | t.Errorf("Put error in approximate size test: %v", err) 126 | } 127 | } 128 | 129 | ranges := []Range{ 130 | {[]byte("a"), []byte("k00000000000000010000")}, 131 | {[]byte("k00000000000000010000"), []byte("z")}, 132 | } 133 | sizes := db.GetApproximateSizes(ranges) 134 | if len(sizes) == 2 { 135 | if sizes[0] <= 0 { 136 | t.Errorf("First size range was %d", sizes[0]) 137 | } 138 | if sizes[1] <= 0 { 139 | t.Errorf("Second size range was %d", sizes[1]) 140 | } 141 | } else { 142 | t.Errorf("Expected 2 approx. sizes back, got %d", len(sizes)) 143 | } 144 | 145 | // property 146 | prop := db.PropertyValue("nosuchprop") 147 | if prop != "" { 148 | t.Errorf("property nosuchprop should not have a value") 149 | } 150 | prop = db.PropertyValue("leveldb.stats") 151 | if prop == "" { 152 | t.Errorf("property leveldb.stats should have a value") 153 | } 154 | 155 | // snapshot 156 | snap := db.NewSnapshot() 157 | err = db.Delete(woptions, []byte("foo")) 158 | if err != nil { 159 | t.Errorf("Delete during snapshot test errored: %v", err) 160 | } 161 | roptions.SetSnapshot(snap) 162 | CheckGet(t, "from snapshot", db, roptions, []byte("foo"), []byte("hello")) 163 | roptions.SetSnapshot(nil) 164 | CheckGet(t, "from snapshot", db, roptions, []byte("foo"), nil) 165 | db.ReleaseSnapshot(snap) 166 | 167 | // repair 168 | db.Close() 169 | options.SetCreateIfMissing(false) 170 | options.SetErrorIfExists(false) 171 | err = RepairDatabase(dbname, options) 172 | if err != nil { 173 | t.Errorf("Repairing db failed: %v", err) 174 | } 175 | db, err = Open(dbname, options) 176 | if err != nil { 177 | t.Errorf("Unable to open repaired db: %v", err) 178 | } 179 | CheckGet(t, "repair", db, roptions, []byte("foo"), nil) 180 | CheckGet(t, "repair", db, roptions, []byte("bar"), nil) 181 | CheckGet(t, "repair", db, roptions, []byte("box"), []byte("c")) 182 | options.SetCreateIfMissing(true) 183 | options.SetErrorIfExists(true) 184 | 185 | // filter 186 | policy := NewBloomFilter(10) 187 | db.Close() 188 | DestroyDatabase(dbname, options) 189 | options.SetFilterPolicy(policy) 190 | db, err = Open(dbname, options) 191 | if err != nil { 192 | t.Fatalf("Unable to recreate db for filter tests: %v", err) 193 | } 194 | err = db.Put(woptions, []byte("foo"), []byte("foovalue")) 195 | if err != nil { 196 | t.Errorf("Unable to put 'foo' with filter: %v", err) 197 | } 198 | err = db.Put(woptions, []byte("bar"), []byte("barvalue")) 199 | if err != nil { 200 | t.Errorf("Unable to put 'bar' with filter: %v", err) 201 | } 202 | db.CompactRange(Range{nil, nil}) 203 | CheckGet(t, "filter", db, roptions, []byte("foo"), []byte("foovalue")) 204 | CheckGet(t, "filter", db, roptions, []byte("bar"), []byte("barvalue")) 205 | options.SetFilterPolicy(nil) 206 | policy.Close() 207 | 208 | // cleanup 209 | db.Close() 210 | options.Close() 211 | roptions.Close() 212 | woptions.Close() 213 | cache.Close() 214 | // DestroyComparator(cmp) 215 | env.Close() 216 | } 217 | 218 | func TestNilSlicesInDb(t *testing.T) { 219 | dbname := tempDir(t) 220 | defer deleteDBDirectory(t, dbname) 221 | options := NewOptions() 222 | options.SetErrorIfExists(true) 223 | options.SetCreateIfMissing(true) 224 | ro := NewReadOptions() 225 | _ = DestroyDatabase(dbname, options) 226 | db, err := Open(dbname, options) 227 | if err != nil { 228 | t.Fatalf("Database could not be opened: %v", err) 229 | } 230 | defer db.Close() 231 | val, err := db.Get(ro, []byte("missing")) 232 | if err != nil { 233 | t.Errorf("Get failed: %v", err) 234 | } 235 | if val != nil { 236 | t.Errorf("A key not in the db should return nil, not %v", val) 237 | } 238 | wo := NewWriteOptions() 239 | db.Put(wo, nil, []byte("love")) 240 | val, err = db.Get(ro, nil) 241 | if !bytes.Equal([]byte("love"), val) { 242 | t.Errorf("Get should see the nil key: %v", val) 243 | } 244 | val, err = db.Get(ro, []byte{}) 245 | if !bytes.Equal([]byte("love"), val) { 246 | t.Errorf("Get shouldn't distinguish between nil key and empty slice key: %v", val) 247 | } 248 | 249 | err = db.Put(wo, []byte("nilvalue"), nil) 250 | if err != nil { 251 | t.Errorf("nil value Put errored: %v", err) 252 | } 253 | // Compare with the []byte("missing") case. We expect Get to return a 254 | // []byte{} here, but expect a nil returned there. 255 | CheckGet(t, "nil value Put", db, ro, []byte("nilvalue"), []byte{}) 256 | 257 | err = db.Put(wo, []byte("emptyvalue"), []byte{}) 258 | if err != nil { 259 | t.Errorf("empty value Put errored: %v", err) 260 | } 261 | CheckGet(t, "empty value Put", db, ro, []byte("emptyvalue"), []byte{}) 262 | 263 | err = db.Delete(wo, nil) 264 | if err != nil { 265 | t.Errorf("nil key Delete errored: %v", err) 266 | } 267 | err = db.Delete(wo, []byte{}) 268 | if err != nil { 269 | t.Errorf("empty slice key Delete errored: %v", err) 270 | } 271 | 272 | } 273 | 274 | func TestIterationValidityLimits(t *testing.T) { 275 | dbname := tempDir(t) 276 | defer deleteDBDirectory(t, dbname) 277 | options := NewOptions() 278 | options.SetErrorIfExists(true) 279 | options.SetCreateIfMissing(true) 280 | ro := NewReadOptions() 281 | wo := NewWriteOptions() 282 | _ = DestroyDatabase(dbname, options) 283 | db, err := Open(dbname, options) 284 | if err != nil { 285 | t.Fatalf("Database could not be opened: %v", err) 286 | } 287 | defer db.Close() 288 | db.Put(wo, []byte("bat"), []byte("somedata")) 289 | db.Put(wo, []byte("done"), []byte("somedata")) 290 | it := db.NewIterator(ro) 291 | defer it.Close() 292 | if it.Valid() { 293 | t.Errorf("new Iterator was valid") 294 | } 295 | it.Seek([]byte("bat")) 296 | if !it.Valid() { 297 | t.Errorf("Seek to %#v failed.", []byte("bat")) 298 | } 299 | if !bytes.Equal([]byte("bat"), it.Key()) { 300 | t.Errorf("did not seek to []byte(\"bat\")") 301 | } 302 | key := it.Key() 303 | it.Next() 304 | if bytes.Equal(key, it.Key()) { 305 | t.Errorf("key should be a copy of last key") 306 | } 307 | it.Next() 308 | if it.Valid() { 309 | t.Errorf("iterating off the db should result in an invalid iterator") 310 | } 311 | err = it.GetError() 312 | if err != nil { 313 | t.Errorf("should not have seen an error on an invalid iterator") 314 | } 315 | it.Seek([]byte("bat")) 316 | if !it.Valid() { 317 | t.Errorf("Iterator should be valid again") 318 | } 319 | } 320 | 321 | func CheckGet(t *testing.T, where string, db *DB, roptions *ReadOptions, key, expected []byte) { 322 | getValue, err := db.Get(roptions, key) 323 | 324 | if err != nil { 325 | t.Errorf("%s, Get failed: %v", where, err) 326 | } 327 | if !bytes.Equal(getValue, expected) { 328 | t.Errorf("%s, expected Get value %v, got %v", where, expected, getValue) 329 | } 330 | } 331 | 332 | func WBIterCheckEqual(t *testing.T, where string, which string, pos int, expected, given []byte) { 333 | if !bytes.Equal(expected, given) { 334 | t.Errorf("%s at pos %d, %s expected: %v, got: %v", where, pos, which, expected, given) 335 | } 336 | } 337 | 338 | func CheckIter(t *testing.T, it *Iterator, key, value []byte) { 339 | if !bytes.Equal(key, it.Key()) { 340 | t.Errorf("Iterator: expected key %v, got %v", key, it.Key()) 341 | } 342 | if !bytes.Equal(value, it.Value()) { 343 | t.Errorf("Iterator: expected value %v, got %v", value, it.Value()) 344 | } 345 | } 346 | 347 | func deleteDBDirectory(t *testing.T, dirPath string) { 348 | err := os.RemoveAll(dirPath) 349 | if err != nil { 350 | t.Errorf("Unable to remove database directory: %s", dirPath) 351 | } 352 | } 353 | 354 | func tempDir(t *testing.T) string { 355 | bottom := fmt.Sprintf("levigo-test-%d", rand.Int()) 356 | path := filepath.Join(os.TempDir(), bottom) 357 | deleteDBDirectory(t, path) 358 | return path 359 | } 360 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | // #cgo LDFLAGS: -lleveldb 4 | // #include "leveldb/c.h" 5 | import "C" 6 | 7 | // CompressionOpt is a value for Options.SetCompression. 8 | type CompressionOpt int 9 | 10 | // Known compression arguments for Options.SetCompression. 11 | const ( 12 | NoCompression = CompressionOpt(0) 13 | SnappyCompression = CompressionOpt(1) 14 | ) 15 | 16 | // Options represent all of the available options when opening a database with 17 | // Open. Options should be created with NewOptions. 18 | // 19 | // It is usually with to call SetCache with a cache object. Otherwise, all 20 | // data will be read off disk. 21 | // 22 | // To prevent memory leaks, Close must be called on an Options when the 23 | // program no longer needs it. 24 | type Options struct { 25 | Opt *C.leveldb_options_t 26 | } 27 | 28 | // ReadOptions represent all of the available options when reading from a 29 | // database. 30 | // 31 | // To prevent memory leaks, Close must called on a ReadOptions when the 32 | // program no longer needs it. 33 | type ReadOptions struct { 34 | Opt *C.leveldb_readoptions_t 35 | } 36 | 37 | // WriteOptions represent all of the available options when writing from a 38 | // database. 39 | // 40 | // To prevent memory leaks, Close must called on a WriteOptions when the 41 | // program no longer needs it. 42 | type WriteOptions struct { 43 | Opt *C.leveldb_writeoptions_t 44 | } 45 | 46 | // NewOptions allocates a new Options object. 47 | func NewOptions() *Options { 48 | opt := C.leveldb_options_create() 49 | return &Options{opt} 50 | } 51 | 52 | // NewReadOptions allocates a new ReadOptions object. 53 | func NewReadOptions() *ReadOptions { 54 | opt := C.leveldb_readoptions_create() 55 | return &ReadOptions{opt} 56 | } 57 | 58 | // NewWriteOptions allocates a new WriteOptions object. 59 | func NewWriteOptions() *WriteOptions { 60 | opt := C.leveldb_writeoptions_create() 61 | return &WriteOptions{opt} 62 | } 63 | 64 | // Close deallocates the Options, freeing its underlying C struct. 65 | func (o *Options) Close() { 66 | C.leveldb_options_destroy(o.Opt) 67 | } 68 | 69 | // SetComparator sets the comparator to be used for all read and write 70 | // operations. 71 | // 72 | // The comparator that created a database must be the same one (technically, 73 | // one with the same name string) that is used to perform read and write 74 | // operations. 75 | // 76 | // The default comparator is usually sufficient. 77 | func (o *Options) SetComparator(cmp *C.leveldb_comparator_t) { 78 | C.leveldb_options_set_comparator(o.Opt, cmp) 79 | } 80 | 81 | // SetErrorIfExists causes the opening of a database that already exists to 82 | // throw an error if true. 83 | func (o *Options) SetErrorIfExists(errorIfExists bool) { 84 | eie := boolToUchar(errorIfExists) 85 | C.leveldb_options_set_error_if_exists(o.Opt, eie) 86 | } 87 | 88 | // SetCache places a cache object in the database when a database is opened. 89 | // 90 | // This is usually wise to use. See also ReadOptions.SetFillCache. 91 | func (o *Options) SetCache(cache *Cache) { 92 | C.leveldb_options_set_cache(o.Opt, cache.Cache) 93 | } 94 | 95 | // SetEnv sets the Env object for the new database handle. 96 | func (o *Options) SetEnv(env *Env) { 97 | C.leveldb_options_set_env(o.Opt, env.Env) 98 | } 99 | 100 | // SetInfoLog sets a *C.leveldb_logger_t object as the informational logger 101 | // for the database. 102 | func (o *Options) SetInfoLog(log *C.leveldb_logger_t) { 103 | C.leveldb_options_set_info_log(o.Opt, log) 104 | } 105 | 106 | // SetWriteBufferSize sets the number of bytes the database will build up in 107 | // memory (backed by an unsorted log on disk) before converting to a sorted 108 | // on-disk file. 109 | func (o *Options) SetWriteBufferSize(s int) { 110 | C.leveldb_options_set_write_buffer_size(o.Opt, C.size_t(s)) 111 | } 112 | 113 | // SetParanoidChecks causes the database to do aggressive checking of the data 114 | // it is processing and will stop early if it detects errors if true. 115 | // 116 | // See the LevelDB documentation docs for details. 117 | func (o *Options) SetParanoidChecks(pc bool) { 118 | C.leveldb_options_set_paranoid_checks(o.Opt, boolToUchar(pc)) 119 | } 120 | 121 | // SetMaxOpenFiles sets the number of files than can be used at once by the 122 | // database. 123 | // 124 | // See the LevelDB documentation for details. 125 | func (o *Options) SetMaxOpenFiles(n int) { 126 | C.leveldb_options_set_max_open_files(o.Opt, C.int(n)) 127 | } 128 | 129 | // SetBlockSize sets the approximate size of user data packed per block. 130 | // 131 | // The default is roughly 4096 uncompressed bytes. A better setting depends on 132 | // your use case. See the LevelDB documentation for details. 133 | func (o *Options) SetBlockSize(s int) { 134 | C.leveldb_options_set_block_size(o.Opt, C.size_t(s)) 135 | } 136 | 137 | // SetBlockRestartInterval is the number of keys between restarts points for 138 | // delta encoding keys. 139 | // 140 | // Most clients should leave this parameter alone. See the LevelDB 141 | // documentation for details. 142 | func (o *Options) SetBlockRestartInterval(n int) { 143 | C.leveldb_options_set_block_restart_interval(o.Opt, C.int(n)) 144 | } 145 | 146 | // SetCompression sets whether to compress blocks using the specified 147 | // compresssion algorithm. 148 | // 149 | // The default value is SnappyCompression and it is fast enough that it is 150 | // unlikely you want to turn it off. The other option is NoCompression. 151 | // 152 | // If the LevelDB library was built without Snappy compression enabled, the 153 | // SnappyCompression setting will be ignored. 154 | func (o *Options) SetCompression(t CompressionOpt) { 155 | C.leveldb_options_set_compression(o.Opt, C.int(t)) 156 | } 157 | 158 | // SetCreateIfMissing causes Open to create a new database on disk if it does 159 | // not already exist. 160 | func (o *Options) SetCreateIfMissing(b bool) { 161 | C.leveldb_options_set_create_if_missing(o.Opt, boolToUchar(b)) 162 | } 163 | 164 | // SetFilterPolicy causes Open to create a new database that will uses filter 165 | // created from the filter policy passed in. 166 | func (o *Options) SetFilterPolicy(fp *FilterPolicy) { 167 | var policy *C.leveldb_filterpolicy_t 168 | if fp != nil { 169 | policy = fp.Policy 170 | } 171 | C.leveldb_options_set_filter_policy(o.Opt, policy) 172 | } 173 | 174 | // Close deallocates the ReadOptions, freeing its underlying C struct. 175 | func (ro *ReadOptions) Close() { 176 | C.leveldb_readoptions_destroy(ro.Opt) 177 | } 178 | 179 | // SetVerifyChecksums controls whether all data read with this ReadOptions 180 | // will be verified against corresponding checksums. 181 | // 182 | // It defaults to false. See the LevelDB documentation for details. 183 | func (ro *ReadOptions) SetVerifyChecksums(b bool) { 184 | C.leveldb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b)) 185 | } 186 | 187 | // SetFillCache controls whether reads performed with this ReadOptions will 188 | // fill the Cache of the server. It defaults to true. 189 | // 190 | // It is useful to turn this off on ReadOptions for DB.Iterator (and DB.Get) 191 | // calls used in offline threads to prevent bulk scans from flushing out live 192 | // user data in the cache. 193 | // 194 | // See also Options.SetCache 195 | func (ro *ReadOptions) SetFillCache(b bool) { 196 | C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) 197 | } 198 | 199 | // SetSnapshot causes reads to provided as they were when the passed in 200 | // Snapshot was created by DB.NewSnapshot. This is useful for getting 201 | // consistent reads during a bulk operation. 202 | // 203 | // See the LevelDB documentation for details. 204 | func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { 205 | var s *C.leveldb_snapshot_t 206 | if snap != nil { 207 | s = snap.snap 208 | } 209 | C.leveldb_readoptions_set_snapshot(ro.Opt, s) 210 | } 211 | 212 | // Close deallocates the WriteOptions, freeing its underlying C struct. 213 | func (wo *WriteOptions) Close() { 214 | C.leveldb_writeoptions_destroy(wo.Opt) 215 | } 216 | 217 | // SetSync controls whether each write performed with this WriteOptions will 218 | // be flushed from the operating system buffer cache before the write is 219 | // considered complete. 220 | // 221 | // If called with true, this will significantly slow down writes. If called 222 | // with false, and the host machine crashes, some recent writes may be 223 | // lost. The default is false. 224 | // 225 | // See the LevelDB documentation for details. 226 | func (wo *WriteOptions) SetSync(b bool) { 227 | C.leveldb_writeoptions_set_sync(wo.Opt, boolToUchar(b)) 228 | } 229 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package levigo 2 | 3 | /* 4 | #cgo LDFLAGS: -lleveldb 5 | #include "leveldb/c.h" 6 | */ 7 | import "C" 8 | 9 | // GetLevelDBMajorVersion returns the underlying LevelDB implementation's major 10 | // version. 11 | func GetLevelDBMajorVersion() int { 12 | return int(C.leveldb_major_version()) 13 | } 14 | 15 | // GetLevelDBMinorVersion returns the underlying LevelDB implementation's minor 16 | // version. 17 | func GetLevelDBMinorVersion() int { 18 | return int(C.leveldb_minor_version()) 19 | } 20 | --------------------------------------------------------------------------------