├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── array.go ├── backup.go ├── cache.go ├── cf_handle.go ├── cf_test.go ├── checkpoint.go ├── checkpoint_test.go ├── compaction_filter.go ├── compaction_filter_test.go ├── comparator.go ├── comparator_test.go ├── cow.go ├── cow_test.go ├── db.go ├── db_external_file_test.go ├── db_test.go ├── dbpath.go ├── doc.go ├── dynflag.go ├── env.go ├── filter_policy.go ├── filter_policy_test.go ├── gorocksdb.c ├── gorocksdb.h ├── iterator.go ├── iterator_test.go ├── memory_usage.go ├── memory_usage_test.go ├── merge_operator.go ├── merge_operator_test.go ├── options.go ├── options_block_based_table.go ├── options_compaction.go ├── options_compression.go ├── options_env.go ├── options_flush.go ├── options_ingest.go ├── options_read.go ├── options_transaction.go ├── options_transactiondb.go ├── options_write.go ├── ratelimiter.go ├── slice.go ├── slice_transform.go ├── slice_transform_test.go ├── snapshot.go ├── sst_file_writer.go ├── staticflag_linux.go ├── transaction.go ├── transactiondb.go ├── transactiondb_test.go ├── util.go ├── wal_iterator.go ├── write_batch.go └── write_batch_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: go 3 | go: 4 | - 1.12.x 5 | - 1.13.x 6 | - tip 7 | 8 | before_install: 9 | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 10 | - sudo apt-get update -qq 11 | - sudo apt-get install gcc-6 g++-6 libsnappy-dev zlib1g-dev libbz2-dev -qq 12 | - export CXX="g++-6" CC="gcc-6" 13 | 14 | - wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags2_2.0-1.1ubuntu1_amd64.deb 15 | - sudo dpkg -i libgflags2_2.0-1.1ubuntu1_amd64.deb 16 | - wget https://launchpad.net/ubuntu/+archive/primary/+files/libgflags-dev_2.0-1.1ubuntu1_amd64.deb 17 | - sudo dpkg -i libgflags-dev_2.0-1.1ubuntu1_amd64.deb 18 | 19 | install: 20 | - git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb 21 | - pushd /tmp/rocksdb 22 | - make clean 23 | - make shared_lib -j`nproc` 24 | - sudo cp --preserve=links ./librocksdb.* /usr/lib/ 25 | - sudo cp -r ./include/rocksdb/ /usr/include/ 26 | - popd 27 | - go get -t ./... 28 | 29 | script: 30 | - go test -v ./ 31 | 32 | notifications: 33 | email: 34 | on_success: change 35 | on_failure: always 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2016 Thomas Adam 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gorocksdb, a Go wrapper for RocksDB 2 | 3 | [![Build Status](https://travis-ci.org/tecbot/gorocksdb.svg)](https://travis-ci.org/tecbot/gorocksdb) [![GoDoc](https://godoc.org/github.com/tecbot/gorocksdb?status.svg)](http://godoc.org/github.com/tecbot/gorocksdb) 4 | 5 | ## Install 6 | 7 | You'll need to build [RocksDB](https://github.com/facebook/rocksdb) v5.16+ on your machine. 8 | 9 | After that, you can install gorocksdb using the following command: 10 | 11 | CGO_CFLAGS="-I/path/to/rocksdb/include" \ 12 | CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd" \ 13 | go get github.com/tecbot/gorocksdb 14 | 15 | Please note that this package might upgrade the required RocksDB version at any moment. 16 | Vendoring is thus highly recommended if you require high stability. 17 | 18 | *The [embedded CockroachDB RocksDB](https://github.com/cockroachdb/c-rocksdb) is no longer supported in gorocksdb.* 19 | -------------------------------------------------------------------------------- /array.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "stdlib.h" 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "reflect" 8 | "unsafe" 9 | ) 10 | 11 | type charsSlice []*C.char 12 | type sizeTSlice []C.size_t 13 | type columnFamilySlice []*C.rocksdb_column_family_handle_t 14 | 15 | func (s charsSlice) c() **C.char { 16 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 17 | return (**C.char)(unsafe.Pointer(sH.Data)) 18 | } 19 | 20 | func (s sizeTSlice) c() *C.size_t { 21 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 22 | return (*C.size_t)(unsafe.Pointer(sH.Data)) 23 | } 24 | 25 | func (s columnFamilySlice) c() **C.rocksdb_column_family_handle_t { 26 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 27 | return (**C.rocksdb_column_family_handle_t)(unsafe.Pointer(sH.Data)) 28 | } 29 | 30 | // bytesSliceToCSlices converts a slice of byte slices to two slices with C 31 | // datatypes. One containing pointers to copies of the byte slices and one 32 | // containing their sizes. 33 | // IMPORTANT: All the contents of the charsSlice array are malloced and 34 | // should be freed using the Destroy method of charsSlice. 35 | func byteSlicesToCSlices(vals [][]byte) (charsSlice, sizeTSlice) { 36 | if len(vals) == 0 { 37 | return nil, nil 38 | } 39 | 40 | chars := make(charsSlice, len(vals)) 41 | sizes := make(sizeTSlice, len(vals)) 42 | for i, val := range vals { 43 | chars[i] = (*C.char)(C.CBytes(val)) 44 | sizes[i] = C.size_t(len(val)) 45 | } 46 | 47 | return chars, sizes 48 | } 49 | 50 | func (s charsSlice) Destroy() { 51 | for _, chars := range s { 52 | C.free(unsafe.Pointer(chars)) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /backup.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "errors" 8 | "unsafe" 9 | ) 10 | 11 | // BackupEngineInfo represents the information about the backups 12 | // in a backup engine instance. Use this to get the state of the 13 | // backup like number of backups and their ids and timestamps etc. 14 | type BackupEngineInfo struct { 15 | c *C.rocksdb_backup_engine_info_t 16 | } 17 | 18 | // GetCount gets the number backsup available. 19 | func (b *BackupEngineInfo) GetCount() int { 20 | return int(C.rocksdb_backup_engine_info_count(b.c)) 21 | } 22 | 23 | // GetTimestamp gets the timestamp at which the backup index was taken. 24 | func (b *BackupEngineInfo) GetTimestamp(index int) int64 { 25 | return int64(C.rocksdb_backup_engine_info_timestamp(b.c, C.int(index))) 26 | } 27 | 28 | // GetBackupId gets an id that uniquely identifies a backup 29 | // regardless of its position. 30 | func (b *BackupEngineInfo) GetBackupId(index int) int64 { 31 | return int64(C.rocksdb_backup_engine_info_backup_id(b.c, C.int(index))) 32 | } 33 | 34 | // GetSize get the size of the backup in bytes. 35 | func (b *BackupEngineInfo) GetSize(index int) int64 { 36 | return int64(C.rocksdb_backup_engine_info_size(b.c, C.int(index))) 37 | } 38 | 39 | // GetNumFiles gets the number of files in the backup index. 40 | func (b *BackupEngineInfo) GetNumFiles(index int) int32 { 41 | return int32(C.rocksdb_backup_engine_info_number_files(b.c, C.int(index))) 42 | } 43 | 44 | // Destroy destroys the backup engine info instance. 45 | func (b *BackupEngineInfo) Destroy() { 46 | C.rocksdb_backup_engine_info_destroy(b.c) 47 | b.c = nil 48 | } 49 | 50 | // RestoreOptions captures the options to be used during 51 | // restoration of a backup. 52 | type RestoreOptions struct { 53 | c *C.rocksdb_restore_options_t 54 | } 55 | 56 | // NewRestoreOptions creates a RestoreOptions instance. 57 | func NewRestoreOptions() *RestoreOptions { 58 | return &RestoreOptions{ 59 | c: C.rocksdb_restore_options_create(), 60 | } 61 | } 62 | 63 | // SetKeepLogFiles is used to set or unset the keep_log_files option 64 | // If true, restore won't overwrite the existing log files in wal_dir. It will 65 | // also move all log files from archive directory to wal_dir. 66 | // By default, this is false. 67 | func (ro *RestoreOptions) SetKeepLogFiles(v int) { 68 | C.rocksdb_restore_options_set_keep_log_files(ro.c, C.int(v)) 69 | } 70 | 71 | // Destroy destroys this RestoreOptions instance. 72 | func (ro *RestoreOptions) Destroy() { 73 | C.rocksdb_restore_options_destroy(ro.c) 74 | } 75 | 76 | // BackupEngine is a reusable handle to a RocksDB Backup, created by 77 | // OpenBackupEngine. 78 | type BackupEngine struct { 79 | c *C.rocksdb_backup_engine_t 80 | path string 81 | opts *Options 82 | } 83 | 84 | // OpenBackupEngine opens a backup engine with specified options. 85 | func OpenBackupEngine(opts *Options, path string) (*BackupEngine, error) { 86 | var cErr *C.char 87 | cpath := C.CString(path) 88 | defer C.free(unsafe.Pointer(cpath)) 89 | 90 | be := C.rocksdb_backup_engine_open(opts.c, cpath, &cErr) 91 | if cErr != nil { 92 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 93 | return nil, errors.New(C.GoString(cErr)) 94 | } 95 | return &BackupEngine{ 96 | c: be, 97 | path: path, 98 | opts: opts, 99 | }, nil 100 | } 101 | 102 | // UnsafeGetBackupEngine returns the underlying c backup engine. 103 | func (b *BackupEngine) UnsafeGetBackupEngine() unsafe.Pointer { 104 | return unsafe.Pointer(b.c) 105 | } 106 | 107 | // CreateNewBackupFlush takes a new backup from db. If flush is set to true, 108 | // it flushes the WAL before taking the backup. 109 | func (b *BackupEngine) CreateNewBackupFlush(db *DB, flush bool) error { 110 | var cErr *C.char 111 | 112 | C.rocksdb_backup_engine_create_new_backup_flush(b.c, db.c, boolToChar(flush), &cErr) 113 | if cErr != nil { 114 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 115 | return errors.New(C.GoString(cErr)) 116 | } 117 | 118 | return nil 119 | } 120 | 121 | // CreateNewBackup takes a new backup from db. 122 | func (b *BackupEngine) CreateNewBackup(db *DB) error { 123 | return b.CreateNewBackupFlush(db, false) 124 | } 125 | 126 | // GetInfo gets an object that gives information about 127 | // the backups that have already been taken 128 | func (b *BackupEngine) GetInfo() *BackupEngineInfo { 129 | return &BackupEngineInfo{ 130 | c: C.rocksdb_backup_engine_get_backup_info(b.c), 131 | } 132 | } 133 | 134 | // RestoreDBFromLatestBackup restores the latest backup to dbDir. walDir 135 | // is where the write ahead logs are restored to and usually the same as dbDir. 136 | func (b *BackupEngine) RestoreDBFromLatestBackup(dbDir, walDir string, ro *RestoreOptions) error { 137 | var cErr *C.char 138 | cDbDir := C.CString(dbDir) 139 | cWalDir := C.CString(walDir) 140 | defer func() { 141 | C.free(unsafe.Pointer(cDbDir)) 142 | C.free(unsafe.Pointer(cWalDir)) 143 | }() 144 | 145 | C.rocksdb_backup_engine_restore_db_from_latest_backup(b.c, cDbDir, cWalDir, ro.c, &cErr) 146 | if cErr != nil { 147 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 148 | return errors.New(C.GoString(cErr)) 149 | } 150 | return nil 151 | } 152 | 153 | // PurgeOldBackups deletes all backups older than the latest 'n' backups 154 | func (b *BackupEngine) PurgeOldBackups(n uint32) error { 155 | var cErr *C.char 156 | C.rocksdb_backup_engine_purge_old_backups(b.c, C.uint32_t(n), &cErr) 157 | if cErr != nil { 158 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 159 | return errors.New(C.GoString(cErr)) 160 | } 161 | return nil 162 | } 163 | 164 | // Close close the backup engine and cleans up state 165 | // The backups already taken remain on storage. 166 | func (b *BackupEngine) Close() { 167 | C.rocksdb_backup_engine_close(b.c) 168 | b.c = nil 169 | } 170 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // Cache is a cache used to store data read from data in memory. 7 | type Cache struct { 8 | c *C.rocksdb_cache_t 9 | } 10 | 11 | // NewLRUCache creates a new LRU Cache object with the capacity given. 12 | func NewLRUCache(capacity uint64) *Cache { 13 | return NewNativeCache(C.rocksdb_cache_create_lru(C.size_t(capacity))) 14 | } 15 | 16 | // NewNativeCache creates a Cache object. 17 | func NewNativeCache(c *C.rocksdb_cache_t) *Cache { 18 | return &Cache{c} 19 | } 20 | 21 | // GetUsage returns the Cache memory usage. 22 | func (c *Cache) GetUsage() uint64 { 23 | return uint64(C.rocksdb_cache_get_usage(c.c)) 24 | } 25 | 26 | // GetPinnedUsage returns the Cache pinned memory usage. 27 | func (c *Cache) GetPinnedUsage() uint64 { 28 | return uint64(C.rocksdb_cache_get_pinned_usage(c.c)) 29 | } 30 | 31 | // Destroy deallocates the Cache object. 32 | func (c *Cache) Destroy() { 33 | C.rocksdb_cache_destroy(c.c) 34 | c.c = nil 35 | } 36 | -------------------------------------------------------------------------------- /cf_handle.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // ColumnFamilyHandle represents a handle to a ColumnFamily. 9 | type ColumnFamilyHandle struct { 10 | c *C.rocksdb_column_family_handle_t 11 | } 12 | 13 | // NewNativeColumnFamilyHandle creates a ColumnFamilyHandle object. 14 | func NewNativeColumnFamilyHandle(c *C.rocksdb_column_family_handle_t) *ColumnFamilyHandle { 15 | return &ColumnFamilyHandle{c} 16 | } 17 | 18 | // UnsafeGetCFHandler returns the underlying c column family handle. 19 | func (h *ColumnFamilyHandle) UnsafeGetCFHandler() unsafe.Pointer { 20 | return unsafe.Pointer(h.c) 21 | } 22 | 23 | // Destroy calls the destructor of the underlying column family handle. 24 | func (h *ColumnFamilyHandle) Destroy() { 25 | C.rocksdb_column_family_handle_destroy(h.c) 26 | } 27 | 28 | type ColumnFamilyHandles []*ColumnFamilyHandle 29 | 30 | func (cfs ColumnFamilyHandles) toCSlice() columnFamilySlice { 31 | cCFs := make(columnFamilySlice, len(cfs)) 32 | for i, cf := range cfs { 33 | cCFs[i] = cf.c 34 | } 35 | return cCFs 36 | } 37 | -------------------------------------------------------------------------------- /cf_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/facebookgo/ensure" 8 | ) 9 | 10 | func TestColumnFamilyOpen(t *testing.T) { 11 | dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyOpen") 12 | ensure.Nil(t, err) 13 | 14 | givenNames := []string{"default", "guide"} 15 | opts := NewDefaultOptions() 16 | opts.SetCreateIfMissingColumnFamilies(true) 17 | opts.SetCreateIfMissing(true) 18 | db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) 19 | ensure.Nil(t, err) 20 | defer db.Close() 21 | ensure.DeepEqual(t, len(cfh), 2) 22 | cfh[0].Destroy() 23 | cfh[1].Destroy() 24 | 25 | actualNames, err := ListColumnFamilies(opts, dir) 26 | ensure.Nil(t, err) 27 | ensure.SameElements(t, actualNames, givenNames) 28 | } 29 | 30 | func TestColumnFamilyCreateDrop(t *testing.T) { 31 | dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyCreate") 32 | ensure.Nil(t, err) 33 | 34 | opts := NewDefaultOptions() 35 | opts.SetCreateIfMissingColumnFamilies(true) 36 | opts.SetCreateIfMissing(true) 37 | db, err := OpenDb(opts, dir) 38 | ensure.Nil(t, err) 39 | defer db.Close() 40 | cf, err := db.CreateColumnFamily(opts, "guide") 41 | ensure.Nil(t, err) 42 | defer cf.Destroy() 43 | 44 | actualNames, err := ListColumnFamilies(opts, dir) 45 | ensure.Nil(t, err) 46 | ensure.SameElements(t, actualNames, []string{"default", "guide"}) 47 | 48 | ensure.Nil(t, db.DropColumnFamily(cf)) 49 | 50 | actualNames, err = ListColumnFamilies(opts, dir) 51 | ensure.Nil(t, err) 52 | ensure.SameElements(t, actualNames, []string{"default"}) 53 | } 54 | 55 | func TestColumnFamilyBatchPutGet(t *testing.T) { 56 | dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") 57 | ensure.Nil(t, err) 58 | 59 | givenNames := []string{"default", "guide"} 60 | opts := NewDefaultOptions() 61 | opts.SetCreateIfMissingColumnFamilies(true) 62 | opts.SetCreateIfMissing(true) 63 | db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) 64 | ensure.Nil(t, err) 65 | defer db.Close() 66 | ensure.DeepEqual(t, len(cfh), 2) 67 | defer cfh[0].Destroy() 68 | defer cfh[1].Destroy() 69 | 70 | wo := NewDefaultWriteOptions() 71 | defer wo.Destroy() 72 | ro := NewDefaultReadOptions() 73 | defer ro.Destroy() 74 | 75 | givenKey0 := []byte("hello0") 76 | givenVal0 := []byte("world0") 77 | givenKey1 := []byte("hello1") 78 | givenVal1 := []byte("world1") 79 | 80 | b0 := NewWriteBatch() 81 | defer b0.Destroy() 82 | b0.PutCF(cfh[0], givenKey0, givenVal0) 83 | ensure.Nil(t, db.Write(wo, b0)) 84 | actualVal0, err := db.GetCF(ro, cfh[0], givenKey0) 85 | defer actualVal0.Free() 86 | ensure.Nil(t, err) 87 | ensure.DeepEqual(t, actualVal0.Data(), givenVal0) 88 | 89 | b1 := NewWriteBatch() 90 | defer b1.Destroy() 91 | b1.PutCF(cfh[1], givenKey1, givenVal1) 92 | ensure.Nil(t, db.Write(wo, b1)) 93 | actualVal1, err := db.GetCF(ro, cfh[1], givenKey1) 94 | defer actualVal1.Free() 95 | ensure.Nil(t, err) 96 | ensure.DeepEqual(t, actualVal1.Data(), givenVal1) 97 | 98 | actualVal, err := db.GetCF(ro, cfh[0], givenKey1) 99 | ensure.Nil(t, err) 100 | ensure.DeepEqual(t, actualVal.Size(), 0) 101 | actualVal, err = db.GetCF(ro, cfh[1], givenKey0) 102 | ensure.Nil(t, err) 103 | ensure.DeepEqual(t, actualVal.Size(), 0) 104 | } 105 | 106 | func TestColumnFamilyPutGetDelete(t *testing.T) { 107 | dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") 108 | ensure.Nil(t, err) 109 | 110 | givenNames := []string{"default", "guide"} 111 | opts := NewDefaultOptions() 112 | opts.SetCreateIfMissingColumnFamilies(true) 113 | opts.SetCreateIfMissing(true) 114 | db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) 115 | ensure.Nil(t, err) 116 | defer db.Close() 117 | ensure.DeepEqual(t, len(cfh), 2) 118 | defer cfh[0].Destroy() 119 | defer cfh[1].Destroy() 120 | 121 | wo := NewDefaultWriteOptions() 122 | defer wo.Destroy() 123 | ro := NewDefaultReadOptions() 124 | defer ro.Destroy() 125 | 126 | givenKey0 := []byte("hello0") 127 | givenVal0 := []byte("world0") 128 | givenKey1 := []byte("hello1") 129 | givenVal1 := []byte("world1") 130 | 131 | ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey0, givenVal0)) 132 | actualVal0, err := db.GetCF(ro, cfh[0], givenKey0) 133 | defer actualVal0.Free() 134 | ensure.Nil(t, err) 135 | ensure.DeepEqual(t, actualVal0.Data(), givenVal0) 136 | 137 | ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey1, givenVal1)) 138 | actualVal1, err := db.GetCF(ro, cfh[1], givenKey1) 139 | defer actualVal1.Free() 140 | ensure.Nil(t, err) 141 | ensure.DeepEqual(t, actualVal1.Data(), givenVal1) 142 | 143 | actualVal, err := db.GetCF(ro, cfh[0], givenKey1) 144 | ensure.Nil(t, err) 145 | ensure.DeepEqual(t, actualVal.Size(), 0) 146 | actualVal, err = db.GetCF(ro, cfh[1], givenKey0) 147 | ensure.Nil(t, err) 148 | ensure.DeepEqual(t, actualVal.Size(), 0) 149 | 150 | ensure.Nil(t, db.DeleteCF(wo, cfh[0], givenKey0)) 151 | actualVal, err = db.GetCF(ro, cfh[0], givenKey0) 152 | ensure.Nil(t, err) 153 | ensure.DeepEqual(t, actualVal.Size(), 0) 154 | } 155 | 156 | func newTestDBCF(t *testing.T, name string) (db *DB, cfh []*ColumnFamilyHandle, cleanup func()) { 157 | dir, err := ioutil.TempDir("", "gorocksdb-TestColumnFamilyPutGet") 158 | ensure.Nil(t, err) 159 | 160 | givenNames := []string{"default", "guide"} 161 | opts := NewDefaultOptions() 162 | opts.SetCreateIfMissingColumnFamilies(true) 163 | opts.SetCreateIfMissing(true) 164 | db, cfh, err = OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) 165 | ensure.Nil(t, err) 166 | cleanup = func() { 167 | for _, cf := range cfh { 168 | cf.Destroy() 169 | } 170 | db.Close() 171 | } 172 | return db, cfh, cleanup 173 | } 174 | 175 | func TestColumnFamilyMultiGet(t *testing.T) { 176 | db, cfh, cleanup := newTestDBCF(t, "TestDBMultiGet") 177 | defer cleanup() 178 | 179 | var ( 180 | givenKey1 = []byte("hello1") 181 | givenKey2 = []byte("hello2") 182 | givenKey3 = []byte("hello3") 183 | givenVal1 = []byte("world1") 184 | givenVal2 = []byte("world2") 185 | givenVal3 = []byte("world3") 186 | wo = NewDefaultWriteOptions() 187 | ro = NewDefaultReadOptions() 188 | ) 189 | 190 | // create 191 | ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey1, givenVal1)) 192 | ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey2, givenVal2)) 193 | ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey3, givenVal3)) 194 | 195 | // column family 0 only has givenKey1 196 | values, err := db.MultiGetCF(ro, cfh[0], []byte("noexist"), givenKey1, givenKey2, givenKey3) 197 | defer values.Destroy() 198 | ensure.Nil(t, err) 199 | ensure.DeepEqual(t, len(values), 4) 200 | 201 | ensure.DeepEqual(t, values[0].Data(), []byte(nil)) 202 | ensure.DeepEqual(t, values[1].Data(), givenVal1) 203 | ensure.DeepEqual(t, values[2].Data(), []byte(nil)) 204 | ensure.DeepEqual(t, values[3].Data(), []byte(nil)) 205 | 206 | // column family 1 only has givenKey2 and givenKey3 207 | values, err = db.MultiGetCF(ro, cfh[1], []byte("noexist"), givenKey1, givenKey2, givenKey3) 208 | defer values.Destroy() 209 | ensure.Nil(t, err) 210 | ensure.DeepEqual(t, len(values), 4) 211 | 212 | ensure.DeepEqual(t, values[0].Data(), []byte(nil)) 213 | ensure.DeepEqual(t, values[1].Data(), []byte(nil)) 214 | ensure.DeepEqual(t, values[2].Data(), givenVal2) 215 | ensure.DeepEqual(t, values[3].Data(), givenVal3) 216 | 217 | // getting them all from the right CF should return them all 218 | values, err = db.MultiGetCFMultiCF(ro, 219 | ColumnFamilyHandles{cfh[0], cfh[1], cfh[1]}, 220 | [][]byte{givenKey1, givenKey2, givenKey3}, 221 | ) 222 | defer values.Destroy() 223 | ensure.Nil(t, err) 224 | ensure.DeepEqual(t, len(values), 3) 225 | 226 | ensure.DeepEqual(t, values[0].Data(), givenVal1) 227 | ensure.DeepEqual(t, values[1].Data(), givenVal2) 228 | ensure.DeepEqual(t, values[2].Data(), givenVal3) 229 | } 230 | -------------------------------------------------------------------------------- /checkpoint.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "errors" 9 | "unsafe" 10 | ) 11 | 12 | // Checkpoint provides Checkpoint functionality. 13 | // Checkpoints provide persistent snapshots of RocksDB databases. 14 | type Checkpoint struct { 15 | c *C.rocksdb_checkpoint_t 16 | } 17 | 18 | // NewNativeCheckpoint creates a new checkpoint. 19 | func NewNativeCheckpoint(c *C.rocksdb_checkpoint_t) *Checkpoint { 20 | return &Checkpoint{c} 21 | } 22 | 23 | // CreateCheckpoint builds an openable snapshot of RocksDB on the same disk, which 24 | // accepts an output directory on the same disk, and under the directory 25 | // (1) hard-linked SST files pointing to existing live SST files 26 | // SST files will be copied if output directory is on a different filesystem 27 | // (2) a copied manifest files and other files 28 | // The directory should not already exist and will be created by this API. 29 | // The directory will be an absolute path 30 | // log_size_for_flush: if the total log file size is equal or larger than 31 | // this value, then a flush is triggered for all the column families. The 32 | // default value is 0, which means flush is always triggered. If you move 33 | // away from the default, the checkpoint may not contain up-to-date data 34 | // if WAL writing is not always enabled. 35 | // Flush will always trigger if it is 2PC. 36 | func (checkpoint *Checkpoint) CreateCheckpoint(checkpoint_dir string, log_size_for_flush uint64) error { 37 | var ( 38 | cErr *C.char 39 | ) 40 | 41 | cDir := C.CString(checkpoint_dir) 42 | defer C.free(unsafe.Pointer(cDir)) 43 | 44 | C.rocksdb_checkpoint_create(checkpoint.c, cDir, C.uint64_t(log_size_for_flush), &cErr) 45 | if cErr != nil { 46 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 47 | return errors.New(C.GoString(cErr)) 48 | } 49 | return nil 50 | } 51 | 52 | // Destroy deallocates the Checkpoint object. 53 | func (checkpoint *Checkpoint) Destroy() { 54 | C.rocksdb_checkpoint_object_destroy(checkpoint.c) 55 | checkpoint.c = nil 56 | } 57 | -------------------------------------------------------------------------------- /checkpoint_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/facebookgo/ensure" 9 | ) 10 | 11 | func TestCheckpoint(t *testing.T) { 12 | 13 | suffix := "checkpoint" 14 | dir, err := ioutil.TempDir("", "gorocksdb-"+suffix) 15 | ensure.Nil(t, err) 16 | err = os.RemoveAll(dir) 17 | ensure.Nil(t, err) 18 | 19 | db := newTestDB(t, "TestCheckpoint", nil) 20 | defer db.Close() 21 | 22 | // insert keys 23 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 24 | givenVal := []byte("val") 25 | wo := NewDefaultWriteOptions() 26 | for _, k := range givenKeys { 27 | ensure.Nil(t, db.Put(wo, k, givenVal)) 28 | } 29 | 30 | var dbCheck *DB 31 | var checkpoint *Checkpoint 32 | 33 | checkpoint, err = db.NewCheckpoint() 34 | defer checkpoint.Destroy() 35 | ensure.NotNil(t, checkpoint) 36 | ensure.Nil(t, err) 37 | 38 | err = checkpoint.CreateCheckpoint(dir, 0) 39 | ensure.Nil(t, err) 40 | 41 | opts := NewDefaultOptions() 42 | opts.SetCreateIfMissing(true) 43 | dbCheck, err = OpenDb(opts, dir) 44 | defer dbCheck.Close() 45 | ensure.Nil(t, err) 46 | 47 | // test keys 48 | var value *Slice 49 | ro := NewDefaultReadOptions() 50 | for _, k := range givenKeys { 51 | value, err = dbCheck.Get(ro, k) 52 | defer value.Free() 53 | ensure.Nil(t, err) 54 | ensure.DeepEqual(t, value.Data(), givenVal) 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /compaction_filter.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // A CompactionFilter can be used to filter keys during compaction time. 7 | type CompactionFilter interface { 8 | // If the Filter function returns false, it indicates 9 | // that the kv should be preserved, while a return value of true 10 | // indicates that this key-value should be removed from the 11 | // output of the compaction. The application can inspect 12 | // the existing value of the key and make decision based on it. 13 | // 14 | // When the value is to be preserved, the application has the option 15 | // to modify the existing value and pass it back through a new value. 16 | // To retain the previous value, simply return nil 17 | // 18 | // If multithreaded compaction is being used *and* a single CompactionFilter 19 | // instance was supplied via SetCompactionFilter, this the Filter function may be 20 | // called from different threads concurrently. The application must ensure 21 | // that the call is thread-safe. 22 | Filter(level int, key, val []byte) (remove bool, newVal []byte) 23 | 24 | // The name of the compaction filter, for logging 25 | Name() string 26 | } 27 | 28 | // NewNativeCompactionFilter creates a CompactionFilter object. 29 | func NewNativeCompactionFilter(c *C.rocksdb_compactionfilter_t) CompactionFilter { 30 | return nativeCompactionFilter{c} 31 | } 32 | 33 | type nativeCompactionFilter struct { 34 | c *C.rocksdb_compactionfilter_t 35 | } 36 | 37 | func (c nativeCompactionFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) { 38 | return false, nil 39 | } 40 | func (c nativeCompactionFilter) Name() string { return "" } 41 | 42 | // Hold references to compaction filters. 43 | var compactionFilters = NewCOWList() 44 | 45 | type compactionFilterWrapper struct { 46 | name *C.char 47 | filter CompactionFilter 48 | } 49 | 50 | func registerCompactionFilter(filter CompactionFilter) int { 51 | return compactionFilters.Append(compactionFilterWrapper{C.CString(filter.Name()), filter}) 52 | } 53 | 54 | //export gorocksdb_compactionfilter_filter 55 | func gorocksdb_compactionfilter_filter(idx int, cLevel C.int, cKey *C.char, cKeyLen C.size_t, cVal *C.char, cValLen C.size_t, cNewVal **C.char, cNewValLen *C.size_t, cValChanged *C.uchar) C.int { 56 | key := charToByte(cKey, cKeyLen) 57 | val := charToByte(cVal, cValLen) 58 | 59 | remove, newVal := compactionFilters.Get(idx).(compactionFilterWrapper).filter.Filter(int(cLevel), key, val) 60 | if remove { 61 | return C.int(1) 62 | } else if newVal != nil { 63 | *cNewVal = byteToChar(newVal) 64 | *cNewValLen = C.size_t(len(newVal)) 65 | *cValChanged = C.uchar(1) 66 | } 67 | return C.int(0) 68 | } 69 | 70 | //export gorocksdb_compactionfilter_name 71 | func gorocksdb_compactionfilter_name(idx int) *C.char { 72 | return compactionFilters.Get(idx).(compactionFilterWrapper).name 73 | } 74 | -------------------------------------------------------------------------------- /compaction_filter_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/facebookgo/ensure" 8 | ) 9 | 10 | func TestCompactionFilter(t *testing.T) { 11 | var ( 12 | changeKey = []byte("change") 13 | changeValOld = []byte("old") 14 | changeValNew = []byte("new") 15 | deleteKey = []byte("delete") 16 | ) 17 | db := newTestDB(t, "TestCompactionFilter", func(opts *Options) { 18 | opts.SetCompactionFilter(&mockCompactionFilter{ 19 | filter: func(level int, key, val []byte) (remove bool, newVal []byte) { 20 | if bytes.Equal(key, changeKey) { 21 | return false, changeValNew 22 | } 23 | if bytes.Equal(key, deleteKey) { 24 | return true, val 25 | } 26 | t.Errorf("key %q not expected during compaction", key) 27 | return false, nil 28 | }, 29 | }) 30 | }) 31 | defer db.Close() 32 | 33 | // insert the test keys 34 | wo := NewDefaultWriteOptions() 35 | ensure.Nil(t, db.Put(wo, changeKey, changeValOld)) 36 | ensure.Nil(t, db.Put(wo, deleteKey, changeValNew)) 37 | 38 | // trigger a compaction 39 | db.CompactRange(Range{nil, nil}) 40 | 41 | // ensure that the value is changed after compaction 42 | ro := NewDefaultReadOptions() 43 | v1, err := db.Get(ro, changeKey) 44 | defer v1.Free() 45 | ensure.Nil(t, err) 46 | ensure.DeepEqual(t, v1.Data(), changeValNew) 47 | 48 | // ensure that the key is deleted after compaction 49 | v2, err := db.Get(ro, deleteKey) 50 | ensure.Nil(t, err) 51 | ensure.True(t, v2.Data() == nil) 52 | } 53 | 54 | type mockCompactionFilter struct { 55 | filter func(level int, key, val []byte) (remove bool, newVal []byte) 56 | } 57 | 58 | func (m *mockCompactionFilter) Name() string { return "gorocksdb.test" } 59 | func (m *mockCompactionFilter) Filter(level int, key, val []byte) (bool, []byte) { 60 | return m.filter(level, key, val) 61 | } 62 | -------------------------------------------------------------------------------- /comparator.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // A Comparator object provides a total order across slices that are 7 | // used as keys in an sstable or a database. 8 | type Comparator interface { 9 | // Three-way comparison. Returns value: 10 | // < 0 iff "a" < "b", 11 | // == 0 iff "a" == "b", 12 | // > 0 iff "a" > "b" 13 | Compare(a, b []byte) int 14 | 15 | // The name of the comparator. 16 | Name() string 17 | } 18 | 19 | // NewNativeComparator creates a Comparator object. 20 | func NewNativeComparator(c *C.rocksdb_comparator_t) Comparator { 21 | return nativeComparator{c} 22 | } 23 | 24 | type nativeComparator struct { 25 | c *C.rocksdb_comparator_t 26 | } 27 | 28 | func (c nativeComparator) Compare(a, b []byte) int { return 0 } 29 | func (c nativeComparator) Name() string { return "" } 30 | 31 | // Hold references to comperators. 32 | var comperators = NewCOWList() 33 | 34 | type comperatorWrapper struct { 35 | name *C.char 36 | comparator Comparator 37 | } 38 | 39 | func registerComperator(cmp Comparator) int { 40 | return comperators.Append(comperatorWrapper{C.CString(cmp.Name()), cmp}) 41 | } 42 | 43 | //export gorocksdb_comparator_compare 44 | func gorocksdb_comparator_compare(idx int, cKeyA *C.char, cKeyALen C.size_t, cKeyB *C.char, cKeyBLen C.size_t) C.int { 45 | keyA := charToByte(cKeyA, cKeyALen) 46 | keyB := charToByte(cKeyB, cKeyBLen) 47 | return C.int(comperators.Get(idx).(comperatorWrapper).comparator.Compare(keyA, keyB)) 48 | } 49 | 50 | //export gorocksdb_comparator_name 51 | func gorocksdb_comparator_name(idx int) *C.char { 52 | return comperators.Get(idx).(comperatorWrapper).name 53 | } 54 | -------------------------------------------------------------------------------- /comparator_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/facebookgo/ensure" 8 | ) 9 | 10 | func TestComparator(t *testing.T) { 11 | db := newTestDB(t, "TestComparator", func(opts *Options) { 12 | opts.SetComparator(&bytesReverseComparator{}) 13 | }) 14 | defer db.Close() 15 | 16 | // insert keys 17 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 18 | wo := NewDefaultWriteOptions() 19 | for _, k := range givenKeys { 20 | ensure.Nil(t, db.Put(wo, k, []byte("val"))) 21 | } 22 | 23 | // create a iterator to collect the keys 24 | ro := NewDefaultReadOptions() 25 | iter := db.NewIterator(ro) 26 | defer iter.Close() 27 | 28 | // we seek to the last key and iterate in reverse order 29 | // to match given keys 30 | var actualKeys [][]byte 31 | for iter.SeekToLast(); iter.Valid(); iter.Prev() { 32 | key := make([]byte, 4) 33 | copy(key, iter.Key().Data()) 34 | actualKeys = append(actualKeys, key) 35 | } 36 | ensure.Nil(t, iter.Err()) 37 | 38 | // ensure that the order is correct 39 | ensure.DeepEqual(t, actualKeys, givenKeys) 40 | } 41 | 42 | type bytesReverseComparator struct{} 43 | 44 | func (cmp *bytesReverseComparator) Name() string { return "gorocksdb.bytes-reverse" } 45 | func (cmp *bytesReverseComparator) Compare(a, b []byte) int { 46 | return bytes.Compare(a, b) * -1 47 | } 48 | -------------------------------------------------------------------------------- /cow.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | // COWList implements a copy-on-write list. It is intended to be used by go 9 | // callback registry for CGO, which is read-heavy with occasional writes. 10 | // Reads do not block; Writes do not block reads (or vice versa), but only 11 | // one write can occur at once; 12 | type COWList struct { 13 | v *atomic.Value 14 | mu *sync.Mutex 15 | } 16 | 17 | // NewCOWList creates a new COWList. 18 | func NewCOWList() *COWList { 19 | var list []interface{} 20 | v := &atomic.Value{} 21 | v.Store(list) 22 | return &COWList{v: v, mu: new(sync.Mutex)} 23 | } 24 | 25 | // Append appends an item to the COWList and returns the index for that item. 26 | func (c *COWList) Append(i interface{}) int { 27 | c.mu.Lock() 28 | defer c.mu.Unlock() 29 | list := c.v.Load().([]interface{}) 30 | newLen := len(list) + 1 31 | newList := make([]interface{}, newLen) 32 | copy(newList, list) 33 | newList[newLen-1] = i 34 | c.v.Store(newList) 35 | return newLen - 1 36 | } 37 | 38 | // Get gets the item at index. 39 | func (c *COWList) Get(index int) interface{} { 40 | list := c.v.Load().([]interface{}) 41 | return list[index] 42 | } 43 | -------------------------------------------------------------------------------- /cow_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/facebookgo/ensure" 9 | ) 10 | 11 | func TestCOWList(t *testing.T) { 12 | cl := NewCOWList() 13 | cl.Append("hello") 14 | cl.Append("world") 15 | cl.Append("!") 16 | ensure.DeepEqual(t, cl.Get(0), "hello") 17 | ensure.DeepEqual(t, cl.Get(1), "world") 18 | ensure.DeepEqual(t, cl.Get(2), "!") 19 | } 20 | 21 | func TestCOWListMT(t *testing.T) { 22 | cl := NewCOWList() 23 | expectedRes := make([]int, 3) 24 | var wg sync.WaitGroup 25 | for i := 0; i < 3; i++ { 26 | wg.Add(1) 27 | go func(v int) { 28 | defer wg.Done() 29 | index := cl.Append(v) 30 | expectedRes[index] = v 31 | }(i) 32 | } 33 | wg.Wait() 34 | for i, v := range expectedRes { 35 | ensure.DeepEqual(t, cl.Get(i), v) 36 | } 37 | } 38 | 39 | func BenchmarkCOWList_Get(b *testing.B) { 40 | cl := NewCOWList() 41 | for i := 0; i < 10; i++ { 42 | cl.Append(fmt.Sprintf("helloworld%d", i)) 43 | } 44 | b.ResetTimer() 45 | for i := 0; i < b.N; i++ { 46 | _ = cl.Get(i % 10).(string) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "errors" 8 | "fmt" 9 | "unsafe" 10 | ) 11 | 12 | // Range is a range of keys in the database. GetApproximateSizes calls with it 13 | // begin at the key Start and end right before the key Limit. 14 | type Range struct { 15 | Start []byte 16 | Limit []byte 17 | } 18 | 19 | // DB is a reusable handle to a RocksDB database on disk, created by Open. 20 | type DB struct { 21 | c *C.rocksdb_t 22 | name string 23 | opts *Options 24 | } 25 | 26 | // OpenDb opens a database with the specified options. 27 | func OpenDb(opts *Options, name string) (*DB, error) { 28 | var ( 29 | cErr *C.char 30 | cName = C.CString(name) 31 | ) 32 | defer C.free(unsafe.Pointer(cName)) 33 | db := C.rocksdb_open(opts.c, cName, &cErr) 34 | if cErr != nil { 35 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 36 | return nil, errors.New(C.GoString(cErr)) 37 | } 38 | return &DB{ 39 | name: name, 40 | c: db, 41 | opts: opts, 42 | }, nil 43 | } 44 | 45 | // OpenDbWithTTL opens a database with TTL support with the specified options. 46 | func OpenDbWithTTL(opts *Options, name string, ttl int) (*DB, error) { 47 | var ( 48 | cErr *C.char 49 | cName = C.CString(name) 50 | ) 51 | defer C.free(unsafe.Pointer(cName)) 52 | db := C.rocksdb_open_with_ttl(opts.c, cName, C.int(ttl), &cErr) 53 | if cErr != nil { 54 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 55 | return nil, errors.New(C.GoString(cErr)) 56 | } 57 | return &DB{ 58 | name: name, 59 | c: db, 60 | opts: opts, 61 | }, nil 62 | } 63 | 64 | // OpenDbForReadOnly opens a database with the specified options for readonly usage. 65 | func OpenDbForReadOnly(opts *Options, name string, errorIfLogFileExist bool) (*DB, error) { 66 | var ( 67 | cErr *C.char 68 | cName = C.CString(name) 69 | ) 70 | defer C.free(unsafe.Pointer(cName)) 71 | db := C.rocksdb_open_for_read_only(opts.c, cName, boolToChar(errorIfLogFileExist), &cErr) 72 | if cErr != nil { 73 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 74 | return nil, errors.New(C.GoString(cErr)) 75 | } 76 | return &DB{ 77 | name: name, 78 | c: db, 79 | opts: opts, 80 | }, nil 81 | } 82 | 83 | // OpenDbColumnFamilies opens a database with the specified column families. 84 | func OpenDbColumnFamilies( 85 | opts *Options, 86 | name string, 87 | cfNames []string, 88 | cfOpts []*Options, 89 | ) (*DB, []*ColumnFamilyHandle, error) { 90 | numColumnFamilies := len(cfNames) 91 | if numColumnFamilies != len(cfOpts) { 92 | return nil, nil, errors.New("must provide the same number of column family names and options") 93 | } 94 | 95 | cName := C.CString(name) 96 | defer C.free(unsafe.Pointer(cName)) 97 | 98 | cNames := make([]*C.char, numColumnFamilies) 99 | for i, s := range cfNames { 100 | cNames[i] = C.CString(s) 101 | } 102 | defer func() { 103 | for _, s := range cNames { 104 | C.free(unsafe.Pointer(s)) 105 | } 106 | }() 107 | 108 | cOpts := make([]*C.rocksdb_options_t, numColumnFamilies) 109 | for i, o := range cfOpts { 110 | cOpts[i] = o.c 111 | } 112 | 113 | cHandles := make([]*C.rocksdb_column_family_handle_t, numColumnFamilies) 114 | 115 | var cErr *C.char 116 | db := C.rocksdb_open_column_families( 117 | opts.c, 118 | cName, 119 | C.int(numColumnFamilies), 120 | &cNames[0], 121 | &cOpts[0], 122 | &cHandles[0], 123 | &cErr, 124 | ) 125 | if cErr != nil { 126 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 127 | return nil, nil, errors.New(C.GoString(cErr)) 128 | } 129 | 130 | cfHandles := make([]*ColumnFamilyHandle, numColumnFamilies) 131 | for i, c := range cHandles { 132 | cfHandles[i] = NewNativeColumnFamilyHandle(c) 133 | } 134 | 135 | return &DB{ 136 | name: name, 137 | c: db, 138 | opts: opts, 139 | }, cfHandles, nil 140 | } 141 | 142 | // OpenDbForReadOnlyColumnFamilies opens a database with the specified column 143 | // families in read only mode. 144 | func OpenDbForReadOnlyColumnFamilies( 145 | opts *Options, 146 | name string, 147 | cfNames []string, 148 | cfOpts []*Options, 149 | errorIfLogFileExist bool, 150 | ) (*DB, []*ColumnFamilyHandle, error) { 151 | numColumnFamilies := len(cfNames) 152 | if numColumnFamilies != len(cfOpts) { 153 | return nil, nil, errors.New("must provide the same number of column family names and options") 154 | } 155 | 156 | cName := C.CString(name) 157 | defer C.free(unsafe.Pointer(cName)) 158 | 159 | cNames := make([]*C.char, numColumnFamilies) 160 | for i, s := range cfNames { 161 | cNames[i] = C.CString(s) 162 | } 163 | defer func() { 164 | for _, s := range cNames { 165 | C.free(unsafe.Pointer(s)) 166 | } 167 | }() 168 | 169 | cOpts := make([]*C.rocksdb_options_t, numColumnFamilies) 170 | for i, o := range cfOpts { 171 | cOpts[i] = o.c 172 | } 173 | 174 | cHandles := make([]*C.rocksdb_column_family_handle_t, numColumnFamilies) 175 | 176 | var cErr *C.char 177 | db := C.rocksdb_open_for_read_only_column_families( 178 | opts.c, 179 | cName, 180 | C.int(numColumnFamilies), 181 | &cNames[0], 182 | &cOpts[0], 183 | &cHandles[0], 184 | boolToChar(errorIfLogFileExist), 185 | &cErr, 186 | ) 187 | if cErr != nil { 188 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 189 | return nil, nil, errors.New(C.GoString(cErr)) 190 | } 191 | 192 | cfHandles := make([]*ColumnFamilyHandle, numColumnFamilies) 193 | for i, c := range cHandles { 194 | cfHandles[i] = NewNativeColumnFamilyHandle(c) 195 | } 196 | 197 | return &DB{ 198 | name: name, 199 | c: db, 200 | opts: opts, 201 | }, cfHandles, nil 202 | } 203 | 204 | // ListColumnFamilies lists the names of the column families in the DB. 205 | func ListColumnFamilies(opts *Options, name string) ([]string, error) { 206 | var ( 207 | cErr *C.char 208 | cLen C.size_t 209 | cName = C.CString(name) 210 | ) 211 | defer C.free(unsafe.Pointer(cName)) 212 | cNames := C.rocksdb_list_column_families(opts.c, cName, &cLen, &cErr) 213 | if cErr != nil { 214 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 215 | return nil, errors.New(C.GoString(cErr)) 216 | } 217 | namesLen := int(cLen) 218 | names := make([]string, namesLen) 219 | // The maximum capacity of the following two slices is limited to (2^29)-1 to remain compatible 220 | // with 32-bit platforms. The size of a `*C.char` (a pointer) is 4 Byte on a 32-bit system 221 | // and (2^29)*4 == math.MaxInt32 + 1. -- See issue golang/go#13656 222 | cNamesArr := (*[(1 << 29) - 1]*C.char)(unsafe.Pointer(cNames))[:namesLen:namesLen] 223 | for i, n := range cNamesArr { 224 | names[i] = C.GoString(n) 225 | } 226 | C.rocksdb_list_column_families_destroy(cNames, cLen) 227 | return names, nil 228 | } 229 | 230 | // UnsafeGetDB returns the underlying c rocksdb instance. 231 | func (db *DB) UnsafeGetDB() unsafe.Pointer { 232 | return unsafe.Pointer(db.c) 233 | } 234 | 235 | // Name returns the name of the database. 236 | func (db *DB) Name() string { 237 | return db.name 238 | } 239 | 240 | // Get returns the data associated with the key from the database. 241 | func (db *DB) Get(opts *ReadOptions, key []byte) (*Slice, error) { 242 | var ( 243 | cErr *C.char 244 | cValLen C.size_t 245 | cKey = byteToChar(key) 246 | ) 247 | cValue := C.rocksdb_get(db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr) 248 | if cErr != nil { 249 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 250 | return nil, errors.New(C.GoString(cErr)) 251 | } 252 | return NewSlice(cValue, cValLen), nil 253 | } 254 | 255 | // GetBytes is like Get but returns a copy of the data. 256 | func (db *DB) GetBytes(opts *ReadOptions, key []byte) ([]byte, error) { 257 | var ( 258 | cErr *C.char 259 | cValLen C.size_t 260 | cKey = byteToChar(key) 261 | ) 262 | cValue := C.rocksdb_get(db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr) 263 | if cErr != nil { 264 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 265 | return nil, errors.New(C.GoString(cErr)) 266 | } 267 | if cValue == nil { 268 | return nil, nil 269 | } 270 | defer C.rocksdb_free(unsafe.Pointer(cValue)) 271 | return C.GoBytes(unsafe.Pointer(cValue), C.int(cValLen)), nil 272 | } 273 | 274 | // GetCF returns the data associated with the key from the database and column family. 275 | func (db *DB) GetCF(opts *ReadOptions, cf *ColumnFamilyHandle, key []byte) (*Slice, error) { 276 | var ( 277 | cErr *C.char 278 | cValLen C.size_t 279 | cKey = byteToChar(key) 280 | ) 281 | cValue := C.rocksdb_get_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), &cValLen, &cErr) 282 | if cErr != nil { 283 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 284 | return nil, errors.New(C.GoString(cErr)) 285 | } 286 | return NewSlice(cValue, cValLen), nil 287 | } 288 | 289 | // GetPinned returns the data associated with the key from the database. 290 | func (db *DB) GetPinned(opts *ReadOptions, key []byte) (*PinnableSliceHandle, error) { 291 | var ( 292 | cErr *C.char 293 | cKey = byteToChar(key) 294 | ) 295 | cHandle := C.rocksdb_get_pinned(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) 296 | if cErr != nil { 297 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 298 | return nil, errors.New(C.GoString(cErr)) 299 | } 300 | return NewNativePinnableSliceHandle(cHandle), nil 301 | } 302 | 303 | // MultiGet returns the data associated with the passed keys from the database 304 | func (db *DB) MultiGet(opts *ReadOptions, keys ...[]byte) (Slices, error) { 305 | cKeys, cKeySizes := byteSlicesToCSlices(keys) 306 | defer cKeys.Destroy() 307 | vals := make(charsSlice, len(keys)) 308 | valSizes := make(sizeTSlice, len(keys)) 309 | rocksErrs := make(charsSlice, len(keys)) 310 | 311 | C.rocksdb_multi_get( 312 | db.c, 313 | opts.c, 314 | C.size_t(len(keys)), 315 | cKeys.c(), 316 | cKeySizes.c(), 317 | vals.c(), 318 | valSizes.c(), 319 | rocksErrs.c(), 320 | ) 321 | 322 | var errs []error 323 | 324 | for i, rocksErr := range rocksErrs { 325 | if rocksErr != nil { 326 | defer C.rocksdb_free(unsafe.Pointer(rocksErr)) 327 | err := fmt.Errorf("getting %q failed: %v", string(keys[i]), C.GoString(rocksErr)) 328 | errs = append(errs, err) 329 | } 330 | } 331 | 332 | if len(errs) > 0 { 333 | return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0]) 334 | } 335 | 336 | slices := make(Slices, len(keys)) 337 | for i, val := range vals { 338 | slices[i] = NewSlice(val, valSizes[i]) 339 | } 340 | 341 | return slices, nil 342 | } 343 | 344 | // MultiGetCF returns the data associated with the passed keys from the column family 345 | func (db *DB) MultiGetCF(opts *ReadOptions, cf *ColumnFamilyHandle, keys ...[]byte) (Slices, error) { 346 | cfs := make(ColumnFamilyHandles, len(keys)) 347 | for i := 0; i < len(keys); i++ { 348 | cfs[i] = cf 349 | } 350 | return db.MultiGetCFMultiCF(opts, cfs, keys) 351 | } 352 | 353 | // MultiGetCFMultiCF returns the data associated with the passed keys and 354 | // column families. 355 | func (db *DB) MultiGetCFMultiCF(opts *ReadOptions, cfs ColumnFamilyHandles, keys [][]byte) (Slices, error) { 356 | cKeys, cKeySizes := byteSlicesToCSlices(keys) 357 | defer cKeys.Destroy() 358 | vals := make(charsSlice, len(keys)) 359 | valSizes := make(sizeTSlice, len(keys)) 360 | rocksErrs := make(charsSlice, len(keys)) 361 | 362 | C.rocksdb_multi_get_cf( 363 | db.c, 364 | opts.c, 365 | cfs.toCSlice().c(), 366 | C.size_t(len(keys)), 367 | cKeys.c(), 368 | cKeySizes.c(), 369 | vals.c(), 370 | valSizes.c(), 371 | rocksErrs.c(), 372 | ) 373 | 374 | var errs []error 375 | 376 | for i, rocksErr := range rocksErrs { 377 | if rocksErr != nil { 378 | defer C.rocksdb_free(unsafe.Pointer(rocksErr)) 379 | err := fmt.Errorf("getting %q failed: %v", string(keys[i]), C.GoString(rocksErr)) 380 | errs = append(errs, err) 381 | } 382 | } 383 | 384 | if len(errs) > 0 { 385 | return nil, fmt.Errorf("failed to get %d keys, first error: %v", len(errs), errs[0]) 386 | } 387 | 388 | slices := make(Slices, len(keys)) 389 | for i, val := range vals { 390 | slices[i] = NewSlice(val, valSizes[i]) 391 | } 392 | 393 | return slices, nil 394 | } 395 | 396 | // Put writes data associated with a key to the database. 397 | func (db *DB) Put(opts *WriteOptions, key, value []byte) error { 398 | var ( 399 | cErr *C.char 400 | cKey = byteToChar(key) 401 | cValue = byteToChar(value) 402 | ) 403 | C.rocksdb_put(db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 404 | if cErr != nil { 405 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 406 | return errors.New(C.GoString(cErr)) 407 | } 408 | return nil 409 | } 410 | 411 | // PutCF writes data associated with a key to the database and column family. 412 | func (db *DB) PutCF(opts *WriteOptions, cf *ColumnFamilyHandle, key, value []byte) error { 413 | var ( 414 | cErr *C.char 415 | cKey = byteToChar(key) 416 | cValue = byteToChar(value) 417 | ) 418 | C.rocksdb_put_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 419 | if cErr != nil { 420 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 421 | return errors.New(C.GoString(cErr)) 422 | } 423 | return nil 424 | } 425 | 426 | // Delete removes the data associated with the key from the database. 427 | func (db *DB) Delete(opts *WriteOptions, key []byte) error { 428 | var ( 429 | cErr *C.char 430 | cKey = byteToChar(key) 431 | ) 432 | C.rocksdb_delete(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) 433 | if cErr != nil { 434 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 435 | return errors.New(C.GoString(cErr)) 436 | } 437 | return nil 438 | } 439 | 440 | // DeleteCF removes the data associated with the key from the database and column family. 441 | func (db *DB) DeleteCF(opts *WriteOptions, cf *ColumnFamilyHandle, key []byte) error { 442 | var ( 443 | cErr *C.char 444 | cKey = byteToChar(key) 445 | ) 446 | C.rocksdb_delete_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), &cErr) 447 | if cErr != nil { 448 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 449 | return errors.New(C.GoString(cErr)) 450 | } 451 | return nil 452 | } 453 | 454 | // Merge merges the data associated with the key with the actual data in the database. 455 | func (db *DB) Merge(opts *WriteOptions, key []byte, value []byte) error { 456 | var ( 457 | cErr *C.char 458 | cKey = byteToChar(key) 459 | cValue = byteToChar(value) 460 | ) 461 | C.rocksdb_merge(db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 462 | if cErr != nil { 463 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 464 | return errors.New(C.GoString(cErr)) 465 | } 466 | return nil 467 | } 468 | 469 | // MergeCF merges the data associated with the key with the actual data in the 470 | // database and column family. 471 | func (db *DB) MergeCF(opts *WriteOptions, cf *ColumnFamilyHandle, key []byte, value []byte) error { 472 | var ( 473 | cErr *C.char 474 | cKey = byteToChar(key) 475 | cValue = byteToChar(value) 476 | ) 477 | C.rocksdb_merge_cf(db.c, opts.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 478 | if cErr != nil { 479 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 480 | return errors.New(C.GoString(cErr)) 481 | } 482 | return nil 483 | } 484 | 485 | // Write writes a WriteBatch to the database 486 | func (db *DB) Write(opts *WriteOptions, batch *WriteBatch) error { 487 | var cErr *C.char 488 | C.rocksdb_write(db.c, opts.c, batch.c, &cErr) 489 | if cErr != nil { 490 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 491 | return errors.New(C.GoString(cErr)) 492 | } 493 | return nil 494 | } 495 | 496 | // NewIterator returns an Iterator over the the database that uses the 497 | // ReadOptions given. 498 | func (db *DB) NewIterator(opts *ReadOptions) *Iterator { 499 | cIter := C.rocksdb_create_iterator(db.c, opts.c) 500 | return NewNativeIterator(unsafe.Pointer(cIter)) 501 | } 502 | 503 | // NewIteratorCF returns an Iterator over the the database and column family 504 | // that uses the ReadOptions given. 505 | func (db *DB) NewIteratorCF(opts *ReadOptions, cf *ColumnFamilyHandle) *Iterator { 506 | cIter := C.rocksdb_create_iterator_cf(db.c, opts.c, cf.c) 507 | return NewNativeIterator(unsafe.Pointer(cIter)) 508 | } 509 | 510 | func (db *DB) GetUpdatesSince(seqNumber uint64) (*WalIterator, error) { 511 | var cErr *C.char 512 | cIter := C.rocksdb_get_updates_since(db.c, C.uint64_t(seqNumber), nil, &cErr) 513 | if cErr != nil { 514 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 515 | return nil, errors.New(C.GoString(cErr)) 516 | } 517 | return NewNativeWalIterator(unsafe.Pointer(cIter)), nil 518 | } 519 | 520 | func (db *DB) GetLatestSequenceNumber() uint64 { 521 | return uint64(C.rocksdb_get_latest_sequence_number(db.c)) 522 | } 523 | 524 | // NewSnapshot creates a new snapshot of the database. 525 | func (db *DB) NewSnapshot() *Snapshot { 526 | cSnap := C.rocksdb_create_snapshot(db.c) 527 | return NewNativeSnapshot(cSnap) 528 | } 529 | 530 | // ReleaseSnapshot releases the snapshot and its resources. 531 | func (db *DB) ReleaseSnapshot(snapshot *Snapshot) { 532 | C.rocksdb_release_snapshot(db.c, snapshot.c) 533 | snapshot.c = nil 534 | } 535 | 536 | // GetProperty returns the value of a database property. 537 | func (db *DB) GetProperty(propName string) string { 538 | cprop := C.CString(propName) 539 | defer C.free(unsafe.Pointer(cprop)) 540 | cValue := C.rocksdb_property_value(db.c, cprop) 541 | defer C.rocksdb_free(unsafe.Pointer(cValue)) 542 | return C.GoString(cValue) 543 | } 544 | 545 | // GetPropertyCF returns the value of a database property. 546 | func (db *DB) GetPropertyCF(propName string, cf *ColumnFamilyHandle) string { 547 | cProp := C.CString(propName) 548 | defer C.free(unsafe.Pointer(cProp)) 549 | cValue := C.rocksdb_property_value_cf(db.c, cf.c, cProp) 550 | defer C.rocksdb_free(unsafe.Pointer(cValue)) 551 | return C.GoString(cValue) 552 | } 553 | 554 | // CreateColumnFamily create a new column family. 555 | func (db *DB) CreateColumnFamily(opts *Options, name string) (*ColumnFamilyHandle, error) { 556 | var ( 557 | cErr *C.char 558 | cName = C.CString(name) 559 | ) 560 | defer C.free(unsafe.Pointer(cName)) 561 | cHandle := C.rocksdb_create_column_family(db.c, opts.c, cName, &cErr) 562 | if cErr != nil { 563 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 564 | return nil, errors.New(C.GoString(cErr)) 565 | } 566 | return NewNativeColumnFamilyHandle(cHandle), nil 567 | } 568 | 569 | // DropColumnFamily drops a column family. 570 | func (db *DB) DropColumnFamily(c *ColumnFamilyHandle) error { 571 | var cErr *C.char 572 | C.rocksdb_drop_column_family(db.c, c.c, &cErr) 573 | if cErr != nil { 574 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 575 | return errors.New(C.GoString(cErr)) 576 | } 577 | return nil 578 | } 579 | 580 | // GetApproximateSizes returns the approximate number of bytes of file system 581 | // space used by one or more key ranges. 582 | // 583 | // The keys counted will begin at Range.Start and end on the key before 584 | // Range.Limit. 585 | func (db *DB) GetApproximateSizes(ranges []Range) []uint64 { 586 | sizes := make([]uint64, len(ranges)) 587 | if len(ranges) == 0 { 588 | return sizes 589 | } 590 | 591 | cStarts := make([]*C.char, len(ranges)) 592 | cLimits := make([]*C.char, len(ranges)) 593 | cStartLens := make([]C.size_t, len(ranges)) 594 | cLimitLens := make([]C.size_t, len(ranges)) 595 | for i, r := range ranges { 596 | cStarts[i] = (*C.char)(C.CBytes(r.Start)) 597 | cStartLens[i] = C.size_t(len(r.Start)) 598 | cLimits[i] = (*C.char)(C.CBytes(r.Limit)) 599 | cLimitLens[i] = C.size_t(len(r.Limit)) 600 | } 601 | 602 | defer func() { 603 | for i := range ranges { 604 | C.free(unsafe.Pointer(cStarts[i])) 605 | C.free(unsafe.Pointer(cLimits[i])) 606 | } 607 | }() 608 | 609 | C.rocksdb_approximate_sizes( 610 | db.c, 611 | C.int(len(ranges)), 612 | &cStarts[0], 613 | &cStartLens[0], 614 | &cLimits[0], 615 | &cLimitLens[0], 616 | (*C.uint64_t)(&sizes[0])) 617 | 618 | return sizes 619 | } 620 | 621 | // GetApproximateSizesCF returns the approximate number of bytes of file system 622 | // space used by one or more key ranges in the column family. 623 | // 624 | // The keys counted will begin at Range.Start and end on the key before 625 | // Range.Limit. 626 | func (db *DB) GetApproximateSizesCF(cf *ColumnFamilyHandle, ranges []Range) []uint64 { 627 | sizes := make([]uint64, len(ranges)) 628 | if len(ranges) == 0 { 629 | return sizes 630 | } 631 | 632 | cStarts := make([]*C.char, len(ranges)) 633 | cLimits := make([]*C.char, len(ranges)) 634 | cStartLens := make([]C.size_t, len(ranges)) 635 | cLimitLens := make([]C.size_t, len(ranges)) 636 | for i, r := range ranges { 637 | cStarts[i] = (*C.char)(C.CBytes(r.Start)) 638 | cStartLens[i] = C.size_t(len(r.Start)) 639 | cLimits[i] = (*C.char)(C.CBytes(r.Limit)) 640 | cLimitLens[i] = C.size_t(len(r.Limit)) 641 | } 642 | 643 | defer func() { 644 | for i := range ranges { 645 | C.free(unsafe.Pointer(cStarts[i])) 646 | C.free(unsafe.Pointer(cLimits[i])) 647 | } 648 | }() 649 | 650 | C.rocksdb_approximate_sizes_cf( 651 | db.c, 652 | cf.c, 653 | C.int(len(ranges)), 654 | &cStarts[0], 655 | &cStartLens[0], 656 | &cLimits[0], 657 | &cLimitLens[0], 658 | (*C.uint64_t)(&sizes[0])) 659 | 660 | return sizes 661 | } 662 | 663 | // SetOptions dynamically changes options through the SetOptions API. 664 | func (db *DB) SetOptions(keys, values []string) error { 665 | num_keys := len(keys) 666 | 667 | if num_keys == 0 { 668 | return nil 669 | } 670 | 671 | cKeys := make([]*C.char, num_keys) 672 | cValues := make([]*C.char, num_keys) 673 | for i := range keys { 674 | cKeys[i] = C.CString(keys[i]) 675 | cValues[i] = C.CString(values[i]) 676 | } 677 | 678 | var cErr *C.char 679 | 680 | C.rocksdb_set_options( 681 | db.c, 682 | C.int(num_keys), 683 | &cKeys[0], 684 | &cValues[0], 685 | &cErr, 686 | ) 687 | if cErr != nil { 688 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 689 | return errors.New(C.GoString(cErr)) 690 | } 691 | return nil 692 | } 693 | 694 | // LiveFileMetadata is a metadata which is associated with each SST file. 695 | type LiveFileMetadata struct { 696 | Name string 697 | Level int 698 | Size int64 699 | SmallestKey []byte 700 | LargestKey []byte 701 | } 702 | 703 | // GetLiveFilesMetaData returns a list of all table files with their 704 | // level, start key and end key. 705 | func (db *DB) GetLiveFilesMetaData() []LiveFileMetadata { 706 | lf := C.rocksdb_livefiles(db.c) 707 | defer C.rocksdb_livefiles_destroy(lf) 708 | 709 | count := C.rocksdb_livefiles_count(lf) 710 | liveFiles := make([]LiveFileMetadata, int(count)) 711 | for i := C.int(0); i < count; i++ { 712 | var liveFile LiveFileMetadata 713 | liveFile.Name = C.GoString(C.rocksdb_livefiles_name(lf, i)) 714 | liveFile.Level = int(C.rocksdb_livefiles_level(lf, i)) 715 | liveFile.Size = int64(C.rocksdb_livefiles_size(lf, i)) 716 | 717 | var cSize C.size_t 718 | key := C.rocksdb_livefiles_smallestkey(lf, i, &cSize) 719 | liveFile.SmallestKey = C.GoBytes(unsafe.Pointer(key), C.int(cSize)) 720 | 721 | key = C.rocksdb_livefiles_largestkey(lf, i, &cSize) 722 | liveFile.LargestKey = C.GoBytes(unsafe.Pointer(key), C.int(cSize)) 723 | liveFiles[int(i)] = liveFile 724 | } 725 | return liveFiles 726 | } 727 | 728 | // CompactRange runs a manual compaction on the Range of keys given. This is 729 | // not likely to be needed for typical usage. 730 | func (db *DB) CompactRange(r Range) { 731 | cStart := byteToChar(r.Start) 732 | cLimit := byteToChar(r.Limit) 733 | C.rocksdb_compact_range(db.c, cStart, C.size_t(len(r.Start)), cLimit, C.size_t(len(r.Limit))) 734 | } 735 | 736 | // CompactRangeCF runs a manual compaction on the Range of keys given on the 737 | // given column family. This is not likely to be needed for typical usage. 738 | func (db *DB) CompactRangeCF(cf *ColumnFamilyHandle, r Range) { 739 | cStart := byteToChar(r.Start) 740 | cLimit := byteToChar(r.Limit) 741 | C.rocksdb_compact_range_cf(db.c, cf.c, cStart, C.size_t(len(r.Start)), cLimit, C.size_t(len(r.Limit))) 742 | } 743 | 744 | // Flush triggers a manuel flush for the database. 745 | func (db *DB) Flush(opts *FlushOptions) error { 746 | var cErr *C.char 747 | C.rocksdb_flush(db.c, opts.c, &cErr) 748 | if cErr != nil { 749 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 750 | return errors.New(C.GoString(cErr)) 751 | } 752 | return nil 753 | } 754 | 755 | // DisableFileDeletions disables file deletions and should be used when backup the database. 756 | func (db *DB) DisableFileDeletions() error { 757 | var cErr *C.char 758 | C.rocksdb_disable_file_deletions(db.c, &cErr) 759 | if cErr != nil { 760 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 761 | return errors.New(C.GoString(cErr)) 762 | } 763 | return nil 764 | } 765 | 766 | // EnableFileDeletions enables file deletions for the database. 767 | func (db *DB) EnableFileDeletions(force bool) error { 768 | var cErr *C.char 769 | C.rocksdb_enable_file_deletions(db.c, boolToChar(force), &cErr) 770 | if cErr != nil { 771 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 772 | return errors.New(C.GoString(cErr)) 773 | } 774 | return nil 775 | } 776 | 777 | // DeleteFile deletes the file name from the db directory and update the internal state to 778 | // reflect that. Supports deletion of sst and log files only. 'name' must be 779 | // path relative to the db directory. eg. 000001.sst, /archive/000003.log. 780 | func (db *DB) DeleteFile(name string) { 781 | cName := C.CString(name) 782 | defer C.free(unsafe.Pointer(cName)) 783 | C.rocksdb_delete_file(db.c, cName) 784 | } 785 | 786 | // DeleteFileInRange deletes SST files that contain keys between the Range, [r.Start, r.Limit] 787 | func (db *DB) DeleteFileInRange(r Range) error { 788 | cStartKey := byteToChar(r.Start) 789 | cLimitKey := byteToChar(r.Limit) 790 | 791 | var cErr *C.char 792 | 793 | C.rocksdb_delete_file_in_range( 794 | db.c, 795 | cStartKey, C.size_t(len(r.Start)), 796 | cLimitKey, C.size_t(len(r.Limit)), 797 | &cErr, 798 | ) 799 | 800 | if cErr != nil { 801 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 802 | return errors.New(C.GoString(cErr)) 803 | } 804 | return nil 805 | } 806 | 807 | // DeleteFileInRangeCF deletes SST files that contain keys between the Range, [r.Start, r.Limit], and 808 | // belong to a given column family 809 | func (db *DB) DeleteFileInRangeCF(cf *ColumnFamilyHandle, r Range) error { 810 | cStartKey := byteToChar(r.Start) 811 | cLimitKey := byteToChar(r.Limit) 812 | 813 | var cErr *C.char 814 | 815 | C.rocksdb_delete_file_in_range_cf( 816 | db.c, 817 | cf.c, 818 | cStartKey, C.size_t(len(r.Start)), 819 | cLimitKey, C.size_t(len(r.Limit)), 820 | &cErr, 821 | ) 822 | 823 | if cErr != nil { 824 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 825 | return errors.New(C.GoString(cErr)) 826 | } 827 | return nil 828 | } 829 | 830 | // IngestExternalFile loads a list of external SST files. 831 | func (db *DB) IngestExternalFile(filePaths []string, opts *IngestExternalFileOptions) error { 832 | cFilePaths := make([]*C.char, len(filePaths)) 833 | for i, s := range filePaths { 834 | cFilePaths[i] = C.CString(s) 835 | } 836 | defer func() { 837 | for _, s := range cFilePaths { 838 | C.free(unsafe.Pointer(s)) 839 | } 840 | }() 841 | 842 | var cErr *C.char 843 | 844 | C.rocksdb_ingest_external_file( 845 | db.c, 846 | &cFilePaths[0], 847 | C.size_t(len(filePaths)), 848 | opts.c, 849 | &cErr, 850 | ) 851 | 852 | if cErr != nil { 853 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 854 | return errors.New(C.GoString(cErr)) 855 | } 856 | return nil 857 | } 858 | 859 | // IngestExternalFileCF loads a list of external SST files for a column family. 860 | func (db *DB) IngestExternalFileCF(handle *ColumnFamilyHandle, filePaths []string, opts *IngestExternalFileOptions) error { 861 | cFilePaths := make([]*C.char, len(filePaths)) 862 | for i, s := range filePaths { 863 | cFilePaths[i] = C.CString(s) 864 | } 865 | defer func() { 866 | for _, s := range cFilePaths { 867 | C.free(unsafe.Pointer(s)) 868 | } 869 | }() 870 | 871 | var cErr *C.char 872 | 873 | C.rocksdb_ingest_external_file_cf( 874 | db.c, 875 | handle.c, 876 | &cFilePaths[0], 877 | C.size_t(len(filePaths)), 878 | opts.c, 879 | &cErr, 880 | ) 881 | 882 | if cErr != nil { 883 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 884 | return errors.New(C.GoString(cErr)) 885 | } 886 | return nil 887 | } 888 | 889 | // NewCheckpoint creates a new Checkpoint for this db. 890 | func (db *DB) NewCheckpoint() (*Checkpoint, error) { 891 | var ( 892 | cErr *C.char 893 | ) 894 | cCheckpoint := C.rocksdb_checkpoint_object_create( 895 | db.c, &cErr, 896 | ) 897 | if cErr != nil { 898 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 899 | return nil, errors.New(C.GoString(cErr)) 900 | } 901 | 902 | return NewNativeCheckpoint(cCheckpoint), nil 903 | } 904 | 905 | // Close closes the database. 906 | func (db *DB) Close() { 907 | C.rocksdb_close(db.c) 908 | } 909 | 910 | // DestroyDb removes a database entirely, removing everything from the 911 | // filesystem. 912 | func DestroyDb(name string, opts *Options) error { 913 | var ( 914 | cErr *C.char 915 | cName = C.CString(name) 916 | ) 917 | defer C.free(unsafe.Pointer(cName)) 918 | C.rocksdb_destroy_db(opts.c, cName, &cErr) 919 | if cErr != nil { 920 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 921 | return errors.New(C.GoString(cErr)) 922 | } 923 | return nil 924 | } 925 | 926 | // RepairDb repairs a database. 927 | func RepairDb(name string, opts *Options) error { 928 | var ( 929 | cErr *C.char 930 | cName = C.CString(name) 931 | ) 932 | defer C.free(unsafe.Pointer(cName)) 933 | C.rocksdb_repair_db(opts.c, cName, &cErr) 934 | if cErr != nil { 935 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 936 | return errors.New(C.GoString(cErr)) 937 | } 938 | return nil 939 | } 940 | -------------------------------------------------------------------------------- /db_external_file_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/facebookgo/ensure" 9 | ) 10 | 11 | func TestExternalFile(t *testing.T) { 12 | db := newTestDB(t, "TestDBExternalFile", nil) 13 | defer db.Close() 14 | 15 | envOpts := NewDefaultEnvOptions() 16 | opts := NewDefaultOptions() 17 | w := NewSSTFileWriter(envOpts, opts) 18 | defer w.Destroy() 19 | 20 | filePath, err := ioutil.TempFile("", "sst-file-test") 21 | ensure.Nil(t, err) 22 | defer os.Remove(filePath.Name()) 23 | 24 | err = w.Open(filePath.Name()) 25 | ensure.Nil(t, err) 26 | 27 | err = w.Add([]byte("aaa"), []byte("aaaValue")) 28 | ensure.Nil(t, err) 29 | err = w.Add([]byte("bbb"), []byte("bbbValue")) 30 | ensure.Nil(t, err) 31 | err = w.Add([]byte("ccc"), []byte("cccValue")) 32 | ensure.Nil(t, err) 33 | err = w.Add([]byte("ddd"), []byte("dddValue")) 34 | ensure.Nil(t, err) 35 | 36 | err = w.Finish() 37 | ensure.Nil(t, err) 38 | 39 | ingestOpts := NewDefaultIngestExternalFileOptions() 40 | err = db.IngestExternalFile([]string{filePath.Name()}, ingestOpts) 41 | ensure.Nil(t, err) 42 | 43 | readOpts := NewDefaultReadOptions() 44 | 45 | v1, err := db.Get(readOpts, []byte("aaa")) 46 | ensure.Nil(t, err) 47 | ensure.DeepEqual(t, v1.Data(), []byte("aaaValue")) 48 | v2, err := db.Get(readOpts, []byte("bbb")) 49 | ensure.Nil(t, err) 50 | ensure.DeepEqual(t, v2.Data(), []byte("bbbValue")) 51 | v3, err := db.Get(readOpts, []byte("ccc")) 52 | ensure.Nil(t, err) 53 | ensure.DeepEqual(t, v3.Data(), []byte("cccValue")) 54 | v4, err := db.Get(readOpts, []byte("ddd")) 55 | ensure.Nil(t, err) 56 | ensure.DeepEqual(t, v4.Data(), []byte("dddValue")) 57 | } 58 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/facebookgo/ensure" 9 | ) 10 | 11 | func TestOpenDb(t *testing.T) { 12 | db := newTestDB(t, "TestOpenDb", nil) 13 | defer db.Close() 14 | } 15 | 16 | func TestDBCRUD(t *testing.T) { 17 | db := newTestDB(t, "TestDBGet", nil) 18 | defer db.Close() 19 | 20 | var ( 21 | givenKey = []byte("hello") 22 | givenVal1 = []byte("") 23 | givenVal2 = []byte("world1") 24 | wo = NewDefaultWriteOptions() 25 | ro = NewDefaultReadOptions() 26 | ) 27 | 28 | // create 29 | ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) 30 | 31 | // retrieve 32 | v1, err := db.Get(ro, givenKey) 33 | defer v1.Free() 34 | ensure.Nil(t, err) 35 | ensure.DeepEqual(t, v1.Data(), givenVal1) 36 | 37 | // update 38 | ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) 39 | v2, err := db.Get(ro, givenKey) 40 | defer v2.Free() 41 | ensure.Nil(t, err) 42 | ensure.DeepEqual(t, v2.Data(), givenVal2) 43 | 44 | // retrieve pinned 45 | v3, err := db.GetPinned(ro, givenKey) 46 | defer v3.Destroy() 47 | ensure.Nil(t, err) 48 | ensure.DeepEqual(t, v3.Data(), givenVal2) 49 | 50 | // delete 51 | ensure.Nil(t, db.Delete(wo, givenKey)) 52 | v4, err := db.Get(ro, givenKey) 53 | ensure.Nil(t, err) 54 | ensure.True(t, v4.Data() == nil) 55 | 56 | // retrieve missing pinned 57 | v5, err := db.GetPinned(ro, givenKey) 58 | defer v5.Destroy() 59 | ensure.Nil(t, err) 60 | ensure.True(t, v5.Data() == nil) 61 | } 62 | 63 | func TestDBCRUDDBPaths(t *testing.T) { 64 | names := make([]string, 4) 65 | target_sizes := make([]uint64, len(names)) 66 | 67 | for i := range names { 68 | names[i] = "TestDBGet_" + strconv.FormatInt(int64(i), 10) 69 | target_sizes[i] = uint64(1024 * 1024 * (i + 1)) 70 | } 71 | 72 | db := newTestDBPathNames(t, "TestDBGet", names, target_sizes, nil) 73 | defer db.Close() 74 | 75 | var ( 76 | givenKey = []byte("hello") 77 | givenVal1 = []byte("") 78 | givenVal2 = []byte("world1") 79 | givenVal3 = []byte("world2") 80 | wo = NewDefaultWriteOptions() 81 | ro = NewDefaultReadOptions() 82 | ) 83 | 84 | // retrieve before create 85 | noexist, err := db.Get(ro, givenKey) 86 | defer noexist.Free() 87 | ensure.Nil(t, err) 88 | ensure.False(t, noexist.Exists()) 89 | ensure.DeepEqual(t, noexist.Data(), []byte(nil)) 90 | 91 | // create 92 | ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) 93 | 94 | // retrieve 95 | v1, err := db.Get(ro, givenKey) 96 | defer v1.Free() 97 | ensure.Nil(t, err) 98 | ensure.True(t, v1.Exists()) 99 | ensure.DeepEqual(t, v1.Data(), givenVal1) 100 | 101 | // update 102 | ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) 103 | v2, err := db.Get(ro, givenKey) 104 | defer v2.Free() 105 | ensure.Nil(t, err) 106 | ensure.True(t, v2.Exists()) 107 | ensure.DeepEqual(t, v2.Data(), givenVal2) 108 | 109 | // update 110 | ensure.Nil(t, db.Put(wo, givenKey, givenVal3)) 111 | v3, err := db.Get(ro, givenKey) 112 | defer v3.Free() 113 | ensure.Nil(t, err) 114 | ensure.True(t, v3.Exists()) 115 | ensure.DeepEqual(t, v3.Data(), givenVal3) 116 | 117 | // delete 118 | ensure.Nil(t, db.Delete(wo, givenKey)) 119 | v4, err := db.Get(ro, givenKey) 120 | defer v4.Free() 121 | ensure.Nil(t, err) 122 | ensure.False(t, v4.Exists()) 123 | ensure.DeepEqual(t, v4.Data(), []byte(nil)) 124 | } 125 | 126 | func newTestDB(t *testing.T, name string, applyOpts func(opts *Options)) *DB { 127 | dir, err := ioutil.TempDir("", "gorocksdb-"+name) 128 | ensure.Nil(t, err) 129 | 130 | opts := NewDefaultOptions() 131 | // test the ratelimiter 132 | rateLimiter := NewRateLimiter(1024, 100*1000, 10) 133 | opts.SetRateLimiter(rateLimiter) 134 | opts.SetCreateIfMissing(true) 135 | if applyOpts != nil { 136 | applyOpts(opts) 137 | } 138 | db, err := OpenDb(opts, dir) 139 | ensure.Nil(t, err) 140 | 141 | return db 142 | } 143 | 144 | func newTestDBPathNames(t *testing.T, name string, names []string, target_sizes []uint64, applyOpts func(opts *Options)) *DB { 145 | ensure.DeepEqual(t, len(target_sizes), len(names)) 146 | ensure.NotDeepEqual(t, len(names), 0) 147 | 148 | dir, err := ioutil.TempDir("", "gorocksdb-"+name) 149 | ensure.Nil(t, err) 150 | 151 | paths := make([]string, len(names)) 152 | for i, name := range names { 153 | dir, err := ioutil.TempDir("", "gorocksdb-"+name) 154 | ensure.Nil(t, err) 155 | paths[i] = dir 156 | } 157 | 158 | dbpaths := NewDBPathsFromData(paths, target_sizes) 159 | defer DestroyDBPaths(dbpaths) 160 | 161 | opts := NewDefaultOptions() 162 | opts.SetDBPaths(dbpaths) 163 | // test the ratelimiter 164 | rateLimiter := NewRateLimiter(1024, 100*1000, 10) 165 | opts.SetRateLimiter(rateLimiter) 166 | opts.SetCreateIfMissing(true) 167 | if applyOpts != nil { 168 | applyOpts(opts) 169 | } 170 | db, err := OpenDb(opts, dir) 171 | ensure.Nil(t, err) 172 | 173 | return db 174 | } 175 | 176 | func TestDBMultiGet(t *testing.T) { 177 | db := newTestDB(t, "TestDBMultiGet", nil) 178 | defer db.Close() 179 | 180 | var ( 181 | givenKey1 = []byte("hello1") 182 | givenKey2 = []byte("hello2") 183 | givenKey3 = []byte("hello3") 184 | givenVal1 = []byte("world1") 185 | givenVal2 = []byte("world2") 186 | givenVal3 = []byte("world3") 187 | wo = NewDefaultWriteOptions() 188 | ro = NewDefaultReadOptions() 189 | ) 190 | 191 | // create 192 | ensure.Nil(t, db.Put(wo, givenKey1, givenVal1)) 193 | ensure.Nil(t, db.Put(wo, givenKey2, givenVal2)) 194 | ensure.Nil(t, db.Put(wo, givenKey3, givenVal3)) 195 | 196 | // retrieve 197 | values, err := db.MultiGet(ro, []byte("noexist"), givenKey1, givenKey2, givenKey3) 198 | defer values.Destroy() 199 | ensure.Nil(t, err) 200 | ensure.DeepEqual(t, len(values), 4) 201 | 202 | ensure.DeepEqual(t, values[0].Data(), []byte(nil)) 203 | ensure.DeepEqual(t, values[1].Data(), givenVal1) 204 | ensure.DeepEqual(t, values[2].Data(), givenVal2) 205 | ensure.DeepEqual(t, values[3].Data(), givenVal3) 206 | } 207 | 208 | func TestDBGetApproximateSizes(t *testing.T) { 209 | db := newTestDB(t, "TestDBGetApproximateSizes", nil) 210 | defer db.Close() 211 | 212 | // no ranges 213 | sizes := db.GetApproximateSizes(nil) 214 | ensure.DeepEqual(t, len(sizes), 0) 215 | 216 | // range will nil start and limit 217 | sizes = db.GetApproximateSizes([]Range{{Start: nil, Limit: nil}}) 218 | ensure.DeepEqual(t, sizes, []uint64{0}) 219 | 220 | // valid range 221 | sizes = db.GetApproximateSizes([]Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) 222 | ensure.DeepEqual(t, sizes, []uint64{0}) 223 | } 224 | 225 | func TestDBGetApproximateSizesCF(t *testing.T) { 226 | db := newTestDB(t, "TestDBGetApproximateSizesCF", nil) 227 | defer db.Close() 228 | 229 | o := NewDefaultOptions() 230 | 231 | cf, err := db.CreateColumnFamily(o, "other") 232 | ensure.Nil(t, err) 233 | 234 | // no ranges 235 | sizes := db.GetApproximateSizesCF(cf, nil) 236 | ensure.DeepEqual(t, len(sizes), 0) 237 | 238 | // range will nil start and limit 239 | sizes = db.GetApproximateSizesCF(cf, []Range{{Start: nil, Limit: nil}}) 240 | ensure.DeepEqual(t, sizes, []uint64{0}) 241 | 242 | // valid range 243 | sizes = db.GetApproximateSizesCF(cf, []Range{{Start: []byte{0x00}, Limit: []byte{0xFF}}}) 244 | ensure.DeepEqual(t, sizes, []uint64{0}) 245 | } 246 | -------------------------------------------------------------------------------- /dbpath.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // DBPath represents options for a dbpath. 9 | type DBPath struct { 10 | c *C.rocksdb_dbpath_t 11 | } 12 | 13 | // NewDBPath creates a DBPath object 14 | // with the given path and target_size. 15 | func NewDBPath(path string, target_size uint64) *DBPath { 16 | cpath := C.CString(path) 17 | defer C.free(unsafe.Pointer(cpath)) 18 | return NewNativeDBPath(C.rocksdb_dbpath_create(cpath, C.uint64_t(target_size))) 19 | } 20 | 21 | // NewNativeDBPath creates a DBPath object. 22 | func NewNativeDBPath(c *C.rocksdb_dbpath_t) *DBPath { 23 | return &DBPath{c} 24 | } 25 | 26 | // Destroy deallocates the DBPath object. 27 | func (dbpath *DBPath) Destroy() { 28 | C.rocksdb_dbpath_destroy(dbpath.c) 29 | } 30 | 31 | // NewDBPathsFromData creates a slice with allocated DBPath objects 32 | // from paths and target_sizes. 33 | func NewDBPathsFromData(paths []string, target_sizes []uint64) []*DBPath { 34 | dbpaths := make([]*DBPath, len(paths)) 35 | for i, path := range paths { 36 | targetSize := target_sizes[i] 37 | dbpaths[i] = NewDBPath(path, targetSize) 38 | } 39 | 40 | return dbpaths 41 | } 42 | 43 | // DestroyDBPaths deallocates all DBPath objects in dbpaths. 44 | func DestroyDBPaths(dbpaths []*DBPath) { 45 | for _, dbpath := range dbpaths { 46 | dbpath.Destroy() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package gorocksdb provides the ability to create and access RocksDB databases. 3 | 4 | gorocksdb.OpenDb opens and creates databases. 5 | 6 | bbto := gorocksdb.NewDefaultBlockBasedTableOptions() 7 | bbto.SetBlockCache(gorocksdb.NewLRUCache(3 << 30)) 8 | opts := gorocksdb.NewDefaultOptions() 9 | opts.SetBlockBasedTableFactory(bbto) 10 | opts.SetCreateIfMissing(true) 11 | db, err := gorocksdb.OpenDb(opts, "/path/to/db") 12 | 13 | The DB struct returned by OpenDb provides DB.Get, DB.Put, DB.Merge and DB.Delete to modify 14 | and query the database. 15 | 16 | ro := gorocksdb.NewDefaultReadOptions() 17 | wo := gorocksdb.NewDefaultWriteOptions() 18 | // if ro and wo are not used again, be sure to Close them. 19 | err = db.Put(wo, []byte("foo"), []byte("bar")) 20 | ... 21 | value, err := db.Get(ro, []byte("foo")) 22 | defer value.Free() 23 | ... 24 | err = db.Delete(wo, []byte("foo")) 25 | 26 | For bulk reads, use an Iterator. If you want to avoid disturbing your live 27 | traffic while doing the bulk read, be sure to call SetFillCache(false) on the 28 | ReadOptions you use when creating the Iterator. 29 | 30 | ro := gorocksdb.NewDefaultReadOptions() 31 | ro.SetFillCache(false) 32 | it := db.NewIterator(ro) 33 | defer it.Close() 34 | it.Seek([]byte("foo")) 35 | for it = it; it.Valid(); it.Next() { 36 | key := it.Key() 37 | value := it.Value() 38 | fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data()) 39 | key.Free() 40 | value.Free() 41 | } 42 | if err := it.Err(); err != nil { 43 | ... 44 | } 45 | 46 | Batched, atomic writes can be performed with a WriteBatch and 47 | DB.Write. 48 | 49 | wb := gorocksdb.NewWriteBatch() 50 | // defer wb.Close or use wb.Clear and reuse. 51 | wb.Delete([]byte("foo")) 52 | wb.Put([]byte("foo"), []byte("bar")) 53 | wb.Put([]byte("bar"), []byte("foo")) 54 | err := db.Write(wo, wb) 55 | 56 | If your working dataset does not fit in memory, you'll want to add a bloom 57 | filter to your database. NewBloomFilter and 58 | BlockBasedTableOptions.SetFilterPolicy is what you want. NewBloomFilter is 59 | amount of bits in the filter to use per key in your database. 60 | 61 | filter := gorocksdb.NewBloomFilter(10) 62 | bbto := gorocksdb.NewDefaultBlockBasedTableOptions() 63 | bbto.SetFilterPolicy(filter) 64 | opts.SetBlockBasedTableFactory(bbto) 65 | db, err := gorocksdb.OpenDb(opts, "/path/to/db") 66 | 67 | If you're using a custom comparator in your code, be aware you may have to 68 | make your own filter policy object. 69 | 70 | This documentation is not a complete discussion of RocksDB. Please read the 71 | RocksDB documentation for information on its 72 | operation. You'll find lots of goodies there. 73 | */ 74 | package gorocksdb 75 | -------------------------------------------------------------------------------- /dynflag.go: -------------------------------------------------------------------------------- 1 | // +build !linux !static 2 | 3 | package gorocksdb 4 | 5 | // #cgo LDFLAGS: -lrocksdb -lstdc++ -lm -ldl 6 | import "C" 7 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // Env is a system call environment used by a database. 7 | type Env struct { 8 | c *C.rocksdb_env_t 9 | } 10 | 11 | // NewDefaultEnv creates a default environment. 12 | func NewDefaultEnv() *Env { 13 | return NewNativeEnv(C.rocksdb_create_default_env()) 14 | } 15 | 16 | // NewMemEnv creates MemEnv for in-memory testing. 17 | func NewMemEnv() *Env { 18 | return NewNativeEnv(C.rocksdb_create_mem_env()) 19 | } 20 | 21 | // NewNativeEnv creates a Environment object. 22 | func NewNativeEnv(c *C.rocksdb_env_t) *Env { 23 | return &Env{c} 24 | } 25 | 26 | // SetBackgroundThreads sets the number of background worker threads 27 | // of a specific thread pool for this environment. 28 | // 'LOW' is the default pool. 29 | // Default: 1 30 | func (env *Env) SetBackgroundThreads(n int) { 31 | C.rocksdb_env_set_background_threads(env.c, C.int(n)) 32 | } 33 | 34 | // SetHighPriorityBackgroundThreads sets the size of the high priority 35 | // thread pool that can be used to prevent compactions from stalling 36 | // memtable flushes. 37 | func (env *Env) SetHighPriorityBackgroundThreads(n int) { 38 | C.rocksdb_env_set_high_priority_background_threads(env.c, C.int(n)) 39 | } 40 | 41 | // Destroy deallocates the Env object. 42 | func (env *Env) Destroy() { 43 | C.rocksdb_env_destroy(env.c) 44 | env.c = nil 45 | } 46 | -------------------------------------------------------------------------------- /filter_policy.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // FilterPolicy is a factory type that allows the RocksDB database to create a 7 | // filter, such as a bloom filter, which will used to reduce reads. 8 | type FilterPolicy interface { 9 | // keys contains a list of keys (potentially with duplicates) 10 | // that are ordered according to the user supplied comparator. 11 | CreateFilter(keys [][]byte) []byte 12 | 13 | // "filter" contains the data appended by a preceding call to 14 | // CreateFilter(). This method must return true if 15 | // the key was in the list of keys passed to CreateFilter(). 16 | // This method may return true or false if the key was not on the 17 | // list, but it should aim to return false with a high probability. 18 | KeyMayMatch(key []byte, filter []byte) bool 19 | 20 | // Return the name of this policy. 21 | Name() string 22 | } 23 | 24 | // NewNativeFilterPolicy creates a FilterPolicy object. 25 | func NewNativeFilterPolicy(c *C.rocksdb_filterpolicy_t) FilterPolicy { 26 | return nativeFilterPolicy{c} 27 | } 28 | 29 | type nativeFilterPolicy struct { 30 | c *C.rocksdb_filterpolicy_t 31 | } 32 | 33 | func (fp nativeFilterPolicy) CreateFilter(keys [][]byte) []byte { return nil } 34 | func (fp nativeFilterPolicy) KeyMayMatch(key []byte, filter []byte) bool { return false } 35 | func (fp nativeFilterPolicy) Name() string { return "" } 36 | 37 | // NewBloomFilter returns a new filter policy that uses a bloom filter with approximately 38 | // the specified number of bits per key. A good value for bits_per_key 39 | // is 10, which yields a filter with ~1% false positive rate. 40 | // 41 | // Note: if you are using a custom comparator that ignores some parts 42 | // of the keys being compared, you must not use NewBloomFilterPolicy() 43 | // and must provide your own FilterPolicy that also ignores the 44 | // corresponding parts of the keys. For example, if the comparator 45 | // ignores trailing spaces, it would be incorrect to use a 46 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 47 | // trailing spaces in keys. 48 | func NewBloomFilter(bitsPerKey int) FilterPolicy { 49 | return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom(C.int(bitsPerKey))) 50 | } 51 | 52 | // NewBloomFilterFull returns a new filter policy created with use_block_based_builder=false 53 | // (use full or partitioned filter). 54 | func NewBloomFilterFull(bitsPerKey int) FilterPolicy { 55 | return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom_full(C.int(bitsPerKey))) 56 | } 57 | 58 | // Hold references to filter policies. 59 | var filterPolicies = NewCOWList() 60 | 61 | type filterPolicyWrapper struct { 62 | name *C.char 63 | filterPolicy FilterPolicy 64 | } 65 | 66 | func registerFilterPolicy(fp FilterPolicy) int { 67 | return filterPolicies.Append(filterPolicyWrapper{C.CString(fp.Name()), fp}) 68 | } 69 | 70 | //export gorocksdb_filterpolicy_create_filter 71 | func gorocksdb_filterpolicy_create_filter(idx int, cKeys **C.char, cKeysLen *C.size_t, cNumKeys C.int, cDstLen *C.size_t) *C.char { 72 | rawKeys := charSlice(cKeys, cNumKeys) 73 | keysLen := sizeSlice(cKeysLen, cNumKeys) 74 | keys := make([][]byte, int(cNumKeys)) 75 | for i, len := range keysLen { 76 | keys[i] = charToByte(rawKeys[i], len) 77 | } 78 | 79 | dst := filterPolicies.Get(idx).(filterPolicyWrapper).filterPolicy.CreateFilter(keys) 80 | *cDstLen = C.size_t(len(dst)) 81 | return cByteSlice(dst) 82 | } 83 | 84 | //export gorocksdb_filterpolicy_key_may_match 85 | func gorocksdb_filterpolicy_key_may_match(idx int, cKey *C.char, cKeyLen C.size_t, cFilter *C.char, cFilterLen C.size_t) C.uchar { 86 | key := charToByte(cKey, cKeyLen) 87 | filter := charToByte(cFilter, cFilterLen) 88 | return boolToChar(filterPolicies.Get(idx).(filterPolicyWrapper).filterPolicy.KeyMayMatch(key, filter)) 89 | } 90 | 91 | //export gorocksdb_filterpolicy_name 92 | func gorocksdb_filterpolicy_name(idx int) *C.char { 93 | return filterPolicies.Get(idx).(filterPolicyWrapper).name 94 | } 95 | -------------------------------------------------------------------------------- /filter_policy_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/facebookgo/ensure" 7 | ) 8 | 9 | // fatalAsError is used as a wrapper to make it possible to use ensure 10 | // also if C calls Go otherwise it will throw a internal lockOSThread error. 11 | type fatalAsError struct { 12 | t *testing.T 13 | } 14 | 15 | func (f *fatalAsError) Fatal(a ...interface{}) { 16 | f.t.Error(a...) 17 | } 18 | 19 | func TestFilterPolicy(t *testing.T) { 20 | var ( 21 | givenKeys = [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 22 | givenFilter = []byte("key") 23 | createFilterCalled = false 24 | keyMayMatchCalled = false 25 | ) 26 | policy := &mockFilterPolicy{ 27 | createFilter: func(keys [][]byte) []byte { 28 | createFilterCalled = true 29 | ensure.DeepEqual(&fatalAsError{t}, keys, givenKeys) 30 | return givenFilter 31 | }, 32 | keyMayMatch: func(key, filter []byte) bool { 33 | keyMayMatchCalled = true 34 | ensure.DeepEqual(&fatalAsError{t}, key, givenKeys[0]) 35 | ensure.DeepEqual(&fatalAsError{t}, filter, givenFilter) 36 | return true 37 | }, 38 | } 39 | 40 | db := newTestDB(t, "TestFilterPolicy", func(opts *Options) { 41 | blockOpts := NewDefaultBlockBasedTableOptions() 42 | blockOpts.SetFilterPolicy(policy) 43 | opts.SetBlockBasedTableFactory(blockOpts) 44 | }) 45 | defer db.Close() 46 | 47 | // insert keys 48 | wo := NewDefaultWriteOptions() 49 | for _, k := range givenKeys { 50 | ensure.Nil(t, db.Put(wo, k, []byte("val"))) 51 | } 52 | 53 | // flush to trigger the filter creation 54 | ensure.Nil(t, db.Flush(NewDefaultFlushOptions())) 55 | ensure.True(t, createFilterCalled) 56 | 57 | // test key may match call 58 | ro := NewDefaultReadOptions() 59 | v1, err := db.Get(ro, givenKeys[0]) 60 | defer v1.Free() 61 | ensure.Nil(t, err) 62 | ensure.True(t, keyMayMatchCalled) 63 | } 64 | 65 | type mockFilterPolicy struct { 66 | createFilter func(keys [][]byte) []byte 67 | keyMayMatch func(key, filter []byte) bool 68 | } 69 | 70 | func (m *mockFilterPolicy) Name() string { return "gorocksdb.test" } 71 | func (m *mockFilterPolicy) CreateFilter(keys [][]byte) []byte { 72 | return m.createFilter(keys) 73 | } 74 | func (m *mockFilterPolicy) KeyMayMatch(key, filter []byte) bool { 75 | return m.keyMayMatch(key, filter) 76 | } 77 | -------------------------------------------------------------------------------- /gorocksdb.c: -------------------------------------------------------------------------------- 1 | #include "gorocksdb.h" 2 | #include "_cgo_export.h" 3 | 4 | /* Base */ 5 | 6 | void gorocksdb_destruct_handler(void* state) { } 7 | 8 | /* Comparator */ 9 | 10 | rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx) { 11 | return rocksdb_comparator_create( 12 | (void*)idx, 13 | gorocksdb_destruct_handler, 14 | (int (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_comparator_compare), 15 | (const char *(*)(void*))(gorocksdb_comparator_name)); 16 | } 17 | 18 | /* CompactionFilter */ 19 | 20 | rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx) { 21 | return rocksdb_compactionfilter_create( 22 | (void*)idx, 23 | gorocksdb_destruct_handler, 24 | (unsigned char (*)(void*, int, const char*, size_t, const char*, size_t, char**, size_t*, unsigned char*))(gorocksdb_compactionfilter_filter), 25 | (const char *(*)(void*))(gorocksdb_compactionfilter_name)); 26 | } 27 | 28 | /* Filter Policy */ 29 | 30 | rocksdb_filterpolicy_t* gorocksdb_filterpolicy_create(uintptr_t idx) { 31 | return rocksdb_filterpolicy_create( 32 | (void*)idx, 33 | gorocksdb_destruct_handler, 34 | (char* (*)(void*, const char* const*, const size_t*, int, size_t*))(gorocksdb_filterpolicy_create_filter), 35 | (unsigned char (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_filterpolicy_key_may_match), 36 | gorocksdb_filterpolicy_delete_filter, 37 | (const char *(*)(void*))(gorocksdb_filterpolicy_name)); 38 | } 39 | 40 | void gorocksdb_filterpolicy_delete_filter(void* state, const char* v, size_t s) { 41 | free((char*)v); 42 | } 43 | 44 | /* Merge Operator */ 45 | 46 | rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx) { 47 | return rocksdb_mergeoperator_create( 48 | (void*)idx, 49 | gorocksdb_destruct_handler, 50 | (char* (*)(void*, const char*, size_t, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_full_merge), 51 | (char* (*)(void*, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_partial_merge_multi), 52 | gorocksdb_mergeoperator_delete_value, 53 | (const char* (*)(void*))(gorocksdb_mergeoperator_name)); 54 | } 55 | 56 | void gorocksdb_mergeoperator_delete_value(void* id, const char* v, size_t s) { 57 | free((char*)v); 58 | } 59 | 60 | /* Slice Transform */ 61 | 62 | rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx) { 63 | return rocksdb_slicetransform_create( 64 | (void*)idx, 65 | gorocksdb_destruct_handler, 66 | (char* (*)(void*, const char*, size_t, size_t*))(gorocksdb_slicetransform_transform), 67 | (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_domain), 68 | (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_range), 69 | (const char* (*)(void*))(gorocksdb_slicetransform_name)); 70 | } 71 | -------------------------------------------------------------------------------- /gorocksdb.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "rocksdb/c.h" 3 | 4 | // This API provides convenient C wrapper functions for rocksdb client. 5 | 6 | /* Base */ 7 | 8 | extern void gorocksdb_destruct_handler(void* state); 9 | 10 | /* CompactionFilter */ 11 | 12 | extern rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx); 13 | 14 | /* Comparator */ 15 | 16 | extern rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx); 17 | 18 | /* Filter Policy */ 19 | 20 | extern rocksdb_filterpolicy_t* gorocksdb_filterpolicy_create(uintptr_t idx); 21 | extern void gorocksdb_filterpolicy_delete_filter(void* state, const char* v, size_t s); 22 | 23 | /* Merge Operator */ 24 | 25 | extern rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx); 26 | extern void gorocksdb_mergeoperator_delete_value(void* state, const char* v, size_t s); 27 | 28 | /* Slice Transform */ 29 | 30 | extern rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx); 31 | -------------------------------------------------------------------------------- /iterator.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "bytes" 8 | "errors" 9 | "unsafe" 10 | ) 11 | 12 | // Iterator provides a way to seek to specific keys and iterate through 13 | // the keyspace from that point, as well as access the values of those keys. 14 | // 15 | // For example: 16 | // 17 | // it := db.NewIterator(readOpts) 18 | // defer it.Close() 19 | // 20 | // it.Seek([]byte("foo")) 21 | // for ; it.Valid(); it.Next() { 22 | // fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data()) 23 | // } 24 | // 25 | // if err := it.Err(); err != nil { 26 | // return err 27 | // } 28 | // 29 | type Iterator struct { 30 | c *C.rocksdb_iterator_t 31 | } 32 | 33 | // NewNativeIterator creates a Iterator object. 34 | func NewNativeIterator(c unsafe.Pointer) *Iterator { 35 | return &Iterator{(*C.rocksdb_iterator_t)(c)} 36 | } 37 | 38 | // Valid returns false only when an Iterator has iterated past either the 39 | // first or the last key in the database. 40 | func (iter *Iterator) Valid() bool { 41 | return C.rocksdb_iter_valid(iter.c) != 0 42 | } 43 | 44 | // ValidForPrefix returns false only when an Iterator has iterated past the 45 | // first or the last key in the database or the specified prefix. 46 | func (iter *Iterator) ValidForPrefix(prefix []byte) bool { 47 | if C.rocksdb_iter_valid(iter.c) == 0 { 48 | return false 49 | } 50 | 51 | key := iter.Key() 52 | result := bytes.HasPrefix(key.Data(), prefix) 53 | key.Free() 54 | return result 55 | } 56 | 57 | // Key returns the key the iterator currently holds. 58 | func (iter *Iterator) Key() *Slice { 59 | var cLen C.size_t 60 | cKey := C.rocksdb_iter_key(iter.c, &cLen) 61 | if cKey == nil { 62 | return nil 63 | } 64 | return &Slice{cKey, cLen, true} 65 | } 66 | 67 | // Value returns the value in the database the iterator currently holds. 68 | func (iter *Iterator) Value() *Slice { 69 | var cLen C.size_t 70 | cVal := C.rocksdb_iter_value(iter.c, &cLen) 71 | if cVal == nil { 72 | return nil 73 | } 74 | return &Slice{cVal, cLen, true} 75 | } 76 | 77 | // Next moves the iterator to the next sequential key in the database. 78 | func (iter *Iterator) Next() { 79 | C.rocksdb_iter_next(iter.c) 80 | } 81 | 82 | // Prev moves the iterator to the previous sequential key in the database. 83 | func (iter *Iterator) Prev() { 84 | C.rocksdb_iter_prev(iter.c) 85 | } 86 | 87 | // SeekToFirst moves the iterator to the first key in the database. 88 | func (iter *Iterator) SeekToFirst() { 89 | C.rocksdb_iter_seek_to_first(iter.c) 90 | } 91 | 92 | // SeekToLast moves the iterator to the last key in the database. 93 | func (iter *Iterator) SeekToLast() { 94 | C.rocksdb_iter_seek_to_last(iter.c) 95 | } 96 | 97 | // Seek moves the iterator to the position greater than or equal to the key. 98 | func (iter *Iterator) Seek(key []byte) { 99 | cKey := byteToChar(key) 100 | C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key))) 101 | } 102 | 103 | // SeekForPrev moves the iterator to the last key that less than or equal 104 | // to the target key, in contrast with Seek. 105 | func (iter *Iterator) SeekForPrev(key []byte) { 106 | cKey := byteToChar(key) 107 | C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key))) 108 | } 109 | 110 | // Err returns nil if no errors happened during iteration, or the actual 111 | // error otherwise. 112 | func (iter *Iterator) Err() error { 113 | var cErr *C.char 114 | C.rocksdb_iter_get_error(iter.c, &cErr) 115 | if cErr != nil { 116 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 117 | return errors.New(C.GoString(cErr)) 118 | } 119 | return nil 120 | } 121 | 122 | // Close closes the iterator. 123 | func (iter *Iterator) Close() { 124 | C.rocksdb_iter_destroy(iter.c) 125 | iter.c = nil 126 | } 127 | -------------------------------------------------------------------------------- /iterator_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/facebookgo/ensure" 7 | ) 8 | 9 | func TestIterator(t *testing.T) { 10 | db := newTestDB(t, "TestIterator", nil) 11 | defer db.Close() 12 | 13 | // insert keys 14 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 15 | wo := NewDefaultWriteOptions() 16 | for _, k := range givenKeys { 17 | ensure.Nil(t, db.Put(wo, k, []byte("val"))) 18 | } 19 | 20 | ro := NewDefaultReadOptions() 21 | iter := db.NewIterator(ro) 22 | defer iter.Close() 23 | var actualKeys [][]byte 24 | for iter.SeekToFirst(); iter.Valid(); iter.Next() { 25 | key := make([]byte, 4) 26 | copy(key, iter.Key().Data()) 27 | actualKeys = append(actualKeys, key) 28 | } 29 | ensure.Nil(t, iter.Err()) 30 | ensure.DeepEqual(t, actualKeys, givenKeys) 31 | } 32 | -------------------------------------------------------------------------------- /memory_usage.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "errors" 8 | "unsafe" 9 | ) 10 | 11 | // MemoryUsage contains memory usage statistics provided by RocksDB 12 | type MemoryUsage struct { 13 | // MemTableTotal estimates memory usage of all mem-tables 14 | MemTableTotal uint64 15 | // MemTableUnflushed estimates memory usage of unflushed mem-tables 16 | MemTableUnflushed uint64 17 | // MemTableReadersTotal memory usage of table readers (indexes and bloom filters) 18 | MemTableReadersTotal uint64 19 | // CacheTotal memory usage of cache 20 | CacheTotal uint64 21 | } 22 | 23 | // GetApproximateMemoryUsageByType returns summary 24 | // memory usage stats for given databases and caches. 25 | func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (*MemoryUsage, error) { 26 | // register memory consumers 27 | consumers := C.rocksdb_memory_consumers_create() 28 | defer C.rocksdb_memory_consumers_destroy(consumers) 29 | 30 | for _, db := range dbs { 31 | if db != nil { 32 | C.rocksdb_memory_consumers_add_db(consumers, db.c) 33 | } 34 | } 35 | for _, cache := range caches { 36 | if cache != nil { 37 | C.rocksdb_memory_consumers_add_cache(consumers, cache.c) 38 | } 39 | } 40 | 41 | // obtain memory usage stats 42 | var cErr *C.char 43 | memoryUsage := C.rocksdb_approximate_memory_usage_create(consumers, &cErr) 44 | if cErr != nil { 45 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 46 | return nil, errors.New(C.GoString(cErr)) 47 | } 48 | 49 | defer C.rocksdb_approximate_memory_usage_destroy(memoryUsage) 50 | 51 | result := &MemoryUsage{ 52 | MemTableTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_total(memoryUsage)), 53 | MemTableUnflushed: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_unflushed(memoryUsage)), 54 | MemTableReadersTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_readers_total(memoryUsage)), 55 | CacheTotal: uint64(C.rocksdb_approximate_memory_usage_get_cache_total(memoryUsage)), 56 | } 57 | return result, nil 58 | } 59 | -------------------------------------------------------------------------------- /memory_usage_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/facebookgo/ensure" 10 | ) 11 | 12 | func TestMemoryUsage(t *testing.T) { 13 | // create database with cache 14 | cache := NewLRUCache(8 * 1024 * 1024) 15 | bbto := NewDefaultBlockBasedTableOptions() 16 | bbto.SetBlockCache(cache) 17 | defer cache.Destroy() 18 | 19 | applyOpts := func(opts *Options) { 20 | opts.SetBlockBasedTableFactory(bbto) 21 | } 22 | 23 | db := newTestDB(t, "TestMemoryUsage", applyOpts) 24 | defer db.Close() 25 | 26 | // take first memory usage snapshot 27 | mu1, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) 28 | ensure.Nil(t, err) 29 | 30 | // perform IO operations that will affect in-memory tables (and maybe cache as well) 31 | wo := NewDefaultWriteOptions() 32 | defer wo.Destroy() 33 | ro := NewDefaultReadOptions() 34 | defer ro.Destroy() 35 | 36 | key := []byte("key") 37 | value := make([]byte, 1024) 38 | _, err = rand.Read(value) 39 | ensure.Nil(t, err) 40 | 41 | err = db.Put(wo, key, value) 42 | ensure.Nil(t, err) 43 | _, err = db.Get(ro, key) 44 | ensure.Nil(t, err) 45 | 46 | // take second memory usage snapshot 47 | mu2, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) 48 | ensure.Nil(t, err) 49 | 50 | // the amount of memory used by memtables should increase after write/read; 51 | // cache memory usage is not likely to be changed, perhaps because requested key is kept by memtable 52 | assert.True(t, mu2.MemTableTotal > mu1.MemTableTotal) 53 | assert.True(t, mu2.MemTableUnflushed > mu1.MemTableUnflushed) 54 | assert.True(t, mu2.CacheTotal >= mu1.CacheTotal) 55 | assert.True(t, mu2.MemTableReadersTotal >= mu1.MemTableReadersTotal) 56 | } 57 | -------------------------------------------------------------------------------- /merge_operator.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // A MergeOperator specifies the SEMANTICS of a merge, which only 7 | // client knows. It could be numeric addition, list append, string 8 | // concatenation, edit data structure, ... , anything. 9 | // The library, on the other hand, is concerned with the exercise of this 10 | // interface, at the right time (during get, iteration, compaction...) 11 | // 12 | // Please read the RocksDB documentation for 13 | // more details and example implementations. 14 | type MergeOperator interface { 15 | // Gives the client a way to express the read -> modify -> write semantics 16 | // key: The key that's associated with this merge operation. 17 | // Client could multiplex the merge operator based on it 18 | // if the key space is partitioned and different subspaces 19 | // refer to different types of data which have different 20 | // merge operation semantics. 21 | // existingValue: null indicates that the key does not exist before this op. 22 | // operands: the sequence of merge operations to apply, front() first. 23 | // 24 | // Return true on success. 25 | // 26 | // All values passed in will be client-specific values. So if this method 27 | // returns false, it is because client specified bad data or there was 28 | // internal corruption. This will be treated as an error by the library. 29 | FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) 30 | 31 | // The name of the MergeOperator. 32 | Name() string 33 | } 34 | 35 | // PartialMerger implements PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, err) 36 | // When a MergeOperator implements this interface, PartialMerge will be called in addition 37 | // to FullMerge for compactions across levels 38 | type PartialMerger interface { 39 | // This function performs merge(left_op, right_op) 40 | // when both the operands are themselves merge operation types 41 | // that you would have passed to a db.Merge() call in the same order 42 | // (i.e.: db.Merge(key,left_op), followed by db.Merge(key,right_op)). 43 | // 44 | // PartialMerge should combine them into a single merge operation. 45 | // The return value should be constructed such that a call to 46 | // db.Merge(key, new_value) would yield the same result as a call 47 | // to db.Merge(key, left_op) followed by db.Merge(key, right_op). 48 | // 49 | // If it is impossible or infeasible to combine the two operations, return false. 50 | // The library will internally keep track of the operations, and apply them in the 51 | // correct order once a base-value (a Put/Delete/End-of-Database) is seen. 52 | PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) 53 | } 54 | 55 | // MultiMerger implements PartialMergeMulti(key []byte, operands [][]byte) ([]byte, err) 56 | // When a MergeOperator implements this interface, PartialMergeMulti will be called in addition 57 | // to FullMerge for compactions across levels 58 | type MultiMerger interface { 59 | // PartialMerge performs merge on multiple operands 60 | // when all of the operands are themselves merge operation types 61 | // that you would have passed to a db.Merge() call in the same order 62 | // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), 63 | // ... db.Merge(key, operand[n])). 64 | // 65 | // PartialMerge should combine them into a single merge operation. 66 | // The return value should be constructed such that a call to 67 | // db.Merge(key, new_value) would yield the same result as a call 68 | // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), 69 | // ... db.Merge(key, operand[n])). 70 | // 71 | // If it is impossible or infeasible to combine the operations, return false. 72 | // The library will internally keep track of the operations, and apply them in the 73 | // correct order once a base-value (a Put/Delete/End-of-Database) is seen. 74 | PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) 75 | } 76 | 77 | // NewNativeMergeOperator creates a MergeOperator object. 78 | func NewNativeMergeOperator(c *C.rocksdb_mergeoperator_t) MergeOperator { 79 | return nativeMergeOperator{c} 80 | } 81 | 82 | type nativeMergeOperator struct { 83 | c *C.rocksdb_mergeoperator_t 84 | } 85 | 86 | func (mo nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 87 | return nil, false 88 | } 89 | func (mo nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { 90 | return nil, false 91 | } 92 | func (mo nativeMergeOperator) Name() string { return "" } 93 | 94 | // Hold references to merge operators. 95 | var mergeOperators = NewCOWList() 96 | 97 | type mergeOperatorWrapper struct { 98 | name *C.char 99 | mergeOperator MergeOperator 100 | } 101 | 102 | func registerMergeOperator(merger MergeOperator) int { 103 | return mergeOperators.Append(mergeOperatorWrapper{C.CString(merger.Name()), merger}) 104 | } 105 | 106 | //export gorocksdb_mergeoperator_full_merge 107 | func gorocksdb_mergeoperator_full_merge(idx int, cKey *C.char, cKeyLen C.size_t, cExistingValue *C.char, cExistingValueLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { 108 | key := charToByte(cKey, cKeyLen) 109 | rawOperands := charSlice(cOperands, cNumOperands) 110 | operandsLen := sizeSlice(cOperandsLen, cNumOperands) 111 | existingValue := charToByte(cExistingValue, cExistingValueLen) 112 | operands := make([][]byte, int(cNumOperands)) 113 | for i, len := range operandsLen { 114 | operands[i] = charToByte(rawOperands[i], len) 115 | } 116 | 117 | newValue, success := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.FullMerge(key, existingValue, operands) 118 | newValueLen := len(newValue) 119 | 120 | *cNewValueLen = C.size_t(newValueLen) 121 | *cSuccess = boolToChar(success) 122 | 123 | return cByteSlice(newValue) 124 | } 125 | 126 | //export gorocksdb_mergeoperator_partial_merge_multi 127 | func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { 128 | key := charToByte(cKey, cKeyLen) 129 | rawOperands := charSlice(cOperands, cNumOperands) 130 | operandsLen := sizeSlice(cOperandsLen, cNumOperands) 131 | operands := make([][]byte, int(cNumOperands)) 132 | for i, len := range operandsLen { 133 | operands[i] = charToByte(rawOperands[i], len) 134 | } 135 | 136 | var newValue []byte 137 | success := true 138 | 139 | merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator 140 | 141 | // check if this MergeOperator supports partial or multi merges 142 | switch v := merger.(type) { 143 | case MultiMerger: 144 | newValue, success = v.PartialMergeMulti(key, operands) 145 | case PartialMerger: 146 | leftOperand := operands[0] 147 | for i := 1; i < int(cNumOperands); i++ { 148 | newValue, success = v.PartialMerge(key, leftOperand, operands[i]) 149 | if !success { 150 | break 151 | } 152 | leftOperand = newValue 153 | } 154 | default: 155 | success = false 156 | } 157 | 158 | newValueLen := len(newValue) 159 | *cNewValueLen = C.size_t(newValueLen) 160 | *cSuccess = boolToChar(success) 161 | 162 | return cByteSlice(newValue) 163 | } 164 | 165 | //export gorocksdb_mergeoperator_name 166 | func gorocksdb_mergeoperator_name(idx int) *C.char { 167 | return mergeOperators.Get(idx).(mergeOperatorWrapper).name 168 | } 169 | -------------------------------------------------------------------------------- /merge_operator_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/facebookgo/ensure" 7 | ) 8 | 9 | func TestMergeOperator(t *testing.T) { 10 | var ( 11 | givenKey = []byte("hello") 12 | givenVal1 = []byte("foo") 13 | givenVal2 = []byte("bar") 14 | givenMerged = []byte("foobar") 15 | ) 16 | merger := &mockMergeOperator{ 17 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 18 | ensure.DeepEqual(&fatalAsError{t}, key, givenKey) 19 | ensure.DeepEqual(&fatalAsError{t}, existingValue, givenVal1) 20 | ensure.DeepEqual(&fatalAsError{t}, operands, [][]byte{givenVal2}) 21 | return givenMerged, true 22 | }, 23 | } 24 | db := newTestDB(t, "TestMergeOperator", func(opts *Options) { 25 | opts.SetMergeOperator(merger) 26 | }) 27 | defer db.Close() 28 | 29 | wo := NewDefaultWriteOptions() 30 | ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) 31 | ensure.Nil(t, db.Merge(wo, givenKey, givenVal2)) 32 | 33 | // trigger a compaction to ensure that a merge is performed 34 | db.CompactRange(Range{nil, nil}) 35 | 36 | ro := NewDefaultReadOptions() 37 | v1, err := db.Get(ro, givenKey) 38 | defer v1.Free() 39 | ensure.Nil(t, err) 40 | ensure.DeepEqual(t, v1.Data(), givenMerged) 41 | } 42 | 43 | func TestPartialMergeOperator(t *testing.T) { 44 | var ( 45 | givenKey = []byte("hello") 46 | startingVal = []byte("foo") 47 | mergeVal1 = []byte("bar") 48 | mergeVal2 = []byte("baz") 49 | fMergeResult = []byte("foobarbaz") 50 | pMergeResult = []byte("barbaz") 51 | ) 52 | 53 | merger := &mockMergePartialOperator{ 54 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 55 | ensure.DeepEqual(&fatalAsError{t}, key, givenKey) 56 | ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) 57 | ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) 58 | return fMergeResult, true 59 | }, 60 | partialMerge: func(key, leftOperand, rightOperand []byte) ([]byte, bool) { 61 | ensure.DeepEqual(&fatalAsError{t}, key, givenKey) 62 | ensure.DeepEqual(&fatalAsError{t}, leftOperand, mergeVal1) 63 | ensure.DeepEqual(&fatalAsError{t}, rightOperand, mergeVal2) 64 | return pMergeResult, true 65 | }, 66 | } 67 | db := newTestDB(t, "TestMergeOperator", func(opts *Options) { 68 | opts.SetMergeOperator(merger) 69 | }) 70 | defer db.Close() 71 | 72 | wo := NewDefaultWriteOptions() 73 | defer wo.Destroy() 74 | 75 | // insert a starting value and compact to trigger merges 76 | ensure.Nil(t, db.Put(wo, givenKey, startingVal)) 77 | 78 | // trigger a compaction to ensure that a merge is performed 79 | db.CompactRange(Range{nil, nil}) 80 | 81 | // we expect these two operands to be passed to merge partial 82 | ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) 83 | ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) 84 | 85 | // trigger a compaction to ensure that a 86 | // partial and full merge are performed 87 | db.CompactRange(Range{nil, nil}) 88 | 89 | ro := NewDefaultReadOptions() 90 | v1, err := db.Get(ro, givenKey) 91 | defer v1.Free() 92 | ensure.Nil(t, err) 93 | ensure.DeepEqual(t, v1.Data(), fMergeResult) 94 | 95 | } 96 | 97 | func TestMergeMultiOperator(t *testing.T) { 98 | var ( 99 | givenKey = []byte("hello") 100 | startingVal = []byte("foo") 101 | mergeVal1 = []byte("bar") 102 | mergeVal2 = []byte("baz") 103 | fMergeResult = []byte("foobarbaz") 104 | pMergeResult = []byte("barbaz") 105 | ) 106 | 107 | merger := &mockMergeMultiOperator{ 108 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 109 | ensure.DeepEqual(&fatalAsError{t}, key, givenKey) 110 | ensure.DeepEqual(&fatalAsError{t}, existingValue, startingVal) 111 | ensure.DeepEqual(&fatalAsError{t}, operands[0], pMergeResult) 112 | return fMergeResult, true 113 | }, 114 | partialMergeMulti: func(key []byte, operands [][]byte) ([]byte, bool) { 115 | ensure.DeepEqual(&fatalAsError{t}, key, givenKey) 116 | ensure.DeepEqual(&fatalAsError{t}, operands[0], mergeVal1) 117 | ensure.DeepEqual(&fatalAsError{t}, operands[1], mergeVal2) 118 | return pMergeResult, true 119 | }, 120 | } 121 | db := newTestDB(t, "TestMergeOperator", func(opts *Options) { 122 | opts.SetMergeOperator(merger) 123 | }) 124 | defer db.Close() 125 | 126 | wo := NewDefaultWriteOptions() 127 | defer wo.Destroy() 128 | 129 | // insert a starting value and compact to trigger merges 130 | ensure.Nil(t, db.Put(wo, givenKey, startingVal)) 131 | 132 | // trigger a compaction to ensure that a merge is performed 133 | db.CompactRange(Range{nil, nil}) 134 | 135 | // we expect these two operands to be passed to merge multi 136 | ensure.Nil(t, db.Merge(wo, givenKey, mergeVal1)) 137 | ensure.Nil(t, db.Merge(wo, givenKey, mergeVal2)) 138 | 139 | // trigger a compaction to ensure that a 140 | // partial and full merge are performed 141 | db.CompactRange(Range{nil, nil}) 142 | 143 | ro := NewDefaultReadOptions() 144 | v1, err := db.Get(ro, givenKey) 145 | defer v1.Free() 146 | ensure.Nil(t, err) 147 | ensure.DeepEqual(t, v1.Data(), fMergeResult) 148 | 149 | } 150 | 151 | // Mock Objects 152 | type mockMergeOperator struct { 153 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 154 | } 155 | 156 | func (m *mockMergeOperator) Name() string { return "gorocksdb.test" } 157 | func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 158 | return m.fullMerge(key, existingValue, operands) 159 | } 160 | 161 | type mockMergeMultiOperator struct { 162 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 163 | partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) 164 | } 165 | 166 | func (m *mockMergeMultiOperator) Name() string { return "gorocksdb.multi" } 167 | func (m *mockMergeMultiOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 168 | return m.fullMerge(key, existingValue, operands) 169 | } 170 | func (m *mockMergeMultiOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { 171 | return m.partialMergeMulti(key, operands) 172 | } 173 | 174 | type mockMergePartialOperator struct { 175 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 176 | partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) 177 | } 178 | 179 | func (m *mockMergePartialOperator) Name() string { return "gorocksdb.partial" } 180 | func (m *mockMergePartialOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 181 | return m.fullMerge(key, existingValue, operands) 182 | } 183 | func (m *mockMergePartialOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { 184 | return m.partialMerge(key, leftOperand, rightOperand) 185 | } 186 | -------------------------------------------------------------------------------- /options_block_based_table.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | // #include "gorocksdb.h" 5 | import "C" 6 | 7 | // IndexType specifies the index type that will be used for this table. 8 | type IndexType uint 9 | 10 | const ( 11 | // A space efficient index block that is optimized for 12 | // binary-search-based index. 13 | KBinarySearchIndexType = 0 14 | // The hash index, if enabled, will do the hash lookup when 15 | // `Options.prefix_extractor` is provided. 16 | KHashSearchIndexType = 1 17 | // A two-level index implementation. Both levels are binary search indexes. 18 | KTwoLevelIndexSearchIndexType = 2 19 | ) 20 | 21 | // BlockBasedTableOptions represents block-based table options. 22 | type BlockBasedTableOptions struct { 23 | c *C.rocksdb_block_based_table_options_t 24 | 25 | // Hold references for GC. 26 | cache *Cache 27 | compCache *Cache 28 | 29 | // We keep these so we can free their memory in Destroy. 30 | cFp *C.rocksdb_filterpolicy_t 31 | } 32 | 33 | // NewDefaultBlockBasedTableOptions creates a default BlockBasedTableOptions object. 34 | func NewDefaultBlockBasedTableOptions() *BlockBasedTableOptions { 35 | return NewNativeBlockBasedTableOptions(C.rocksdb_block_based_options_create()) 36 | } 37 | 38 | // NewNativeBlockBasedTableOptions creates a BlockBasedTableOptions object. 39 | func NewNativeBlockBasedTableOptions(c *C.rocksdb_block_based_table_options_t) *BlockBasedTableOptions { 40 | return &BlockBasedTableOptions{c: c} 41 | } 42 | 43 | // Destroy deallocates the BlockBasedTableOptions object. 44 | func (opts *BlockBasedTableOptions) Destroy() { 45 | C.rocksdb_block_based_options_destroy(opts.c) 46 | opts.c = nil 47 | opts.cache = nil 48 | opts.compCache = nil 49 | } 50 | 51 | // SetCacheIndexAndFilterBlocks is indicating if we'd put index/filter blocks to the block cache. 52 | // If not specified, each "table reader" object will pre-load index/filter 53 | // block during table initialization. 54 | // Default: false 55 | func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocks(value bool) { 56 | C.rocksdb_block_based_options_set_cache_index_and_filter_blocks(opts.c, boolToChar(value)) 57 | } 58 | 59 | // SetCacheIndexAndFilterBlocksWithHighPriority sets cache index and filter 60 | // blocks with high priority (if cache_index_and_filter_blocks is enabled). 61 | // If set to true, depending on implementation of block cache, 62 | // index and filter blocks may be less likely to be evicted than data blocks. 63 | func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocksWithHighPriority(value bool) { 64 | C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(opts.c, boolToChar(value)) 65 | } 66 | 67 | // SetPinL0FilterAndIndexBlocksInCache sets cache_index_and_filter_blocks. 68 | // If is true and the below is true (hash_index_allow_collision), then 69 | // filter and index blocks are stored in the cache, but a reference is 70 | // held in the "table reader" object so the blocks are pinned and only 71 | // evicted from cache when the table reader is freed. 72 | func (opts *BlockBasedTableOptions) SetPinL0FilterAndIndexBlocksInCache(value bool) { 73 | C.rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(opts.c, boolToChar(value)) 74 | } 75 | 76 | // SetPinTopLevelIndexAndFilter set that if cache_index_and_filter_blocks is true, then 77 | // the top-level index of partitioned filter and index blocks are stored in 78 | // the cache, but a reference is held in the "table reader" object so the 79 | // blocks are pinned and only evicted from cache when the table reader is 80 | // freed. This is not limited to l0 in LSM tree. 81 | func (opts *BlockBasedTableOptions) SetPinTopLevelIndexAndFilter(value bool) { 82 | C.rocksdb_block_based_options_set_pin_top_level_index_and_filter(opts.c, boolToChar(value)) 83 | } 84 | 85 | // SetBlockSize sets the approximate size of user data packed per block. 86 | // Note that the block size specified here corresponds opts uncompressed data. 87 | // The actual size of the unit read from disk may be smaller if 88 | // compression is enabled. This parameter can be changed dynamically. 89 | // Default: 4K 90 | func (opts *BlockBasedTableOptions) SetBlockSize(blockSize int) { 91 | C.rocksdb_block_based_options_set_block_size(opts.c, C.size_t(blockSize)) 92 | } 93 | 94 | // SetBlockSizeDeviation sets the block size deviation. 95 | // This is used opts close a block before it reaches the configured 96 | // 'block_size'. If the percentage of free space in the current block is less 97 | // than this specified number and adding a new record opts the block will 98 | // exceed the configured block size, then this block will be closed and the 99 | // new record will be written opts the next block. 100 | // Default: 10 101 | func (opts *BlockBasedTableOptions) SetBlockSizeDeviation(blockSizeDeviation int) { 102 | C.rocksdb_block_based_options_set_block_size_deviation(opts.c, C.int(blockSizeDeviation)) 103 | } 104 | 105 | // SetBlockRestartInterval sets the number of keys between 106 | // restart points for delta encoding of keys. 107 | // This parameter can be changed dynamically. Most clients should 108 | // leave this parameter alone. 109 | // Default: 16 110 | func (opts *BlockBasedTableOptions) SetBlockRestartInterval(blockRestartInterval int) { 111 | C.rocksdb_block_based_options_set_block_restart_interval(opts.c, C.int(blockRestartInterval)) 112 | } 113 | 114 | // SetIndexBlockRestartInterval is the same as SetBlockRestartInterval but used for the index block. 115 | // Default: 1 116 | func (opts *BlockBasedTableOptions) SetIndexBlockRestartInterval(indexBlockRestartInterval int) { 117 | C.rocksdb_block_based_options_set_index_block_restart_interval(opts.c, C.int(indexBlockRestartInterval)) 118 | } 119 | 120 | // SetMetadataBlockSize sets the block size for partitioned metadata. 121 | // Currently applied to indexes when 122 | // kTwoLevelIndexSearch is used and to filters when partition_filters is used. 123 | // Note: Since in the current implementation the filters and index partitions 124 | // are aligned, an index/filter block is created when either index or filter 125 | // block size reaches the specified limit. 126 | // Note: this limit is currently applied to only index blocks; a filter 127 | // partition is cut right after an index block is cut 128 | // Default: 4096 129 | func (opts *BlockBasedTableOptions) SetMetadataBlockSize(metadataBlockSize uint64) { 130 | C.rocksdb_block_based_options_set_metadata_block_size(opts.c, C.uint64_t(metadataBlockSize)) 131 | } 132 | 133 | // SetPartitionFilters sets using partitioned full filters for each SST file. 134 | // This option is incompatible with block-based filters. 135 | // Note: currently this option requires kTwoLevelIndexSearch to be set as well. 136 | // Default: false 137 | func (opts *BlockBasedTableOptions) SetPartitionFilters(value bool) { 138 | C.rocksdb_block_based_options_set_partition_filters(opts.c, boolToChar(value)) 139 | } 140 | 141 | // SetUseDeltaEncoding sets using delta encoding to compress keys in blocks. 142 | // ReadOptions::pin_data requires this option to be disabled. 143 | func (opts *BlockBasedTableOptions) SetUseDeltaEncoding(value bool) { 144 | C.rocksdb_block_based_options_set_use_delta_encoding(opts.c, boolToChar(value)) 145 | } 146 | 147 | // SetFilterPolicy sets the filter policy opts reduce disk reads. 148 | // Many applications will benefit from passing the result of 149 | // NewBloomFilterPolicy() here. 150 | // Default: nil 151 | func (opts *BlockBasedTableOptions) SetFilterPolicy(fp FilterPolicy) { 152 | if nfp, ok := fp.(nativeFilterPolicy); ok { 153 | opts.cFp = nfp.c 154 | } else { 155 | idx := registerFilterPolicy(fp) 156 | opts.cFp = C.gorocksdb_filterpolicy_create(C.uintptr_t(idx)) 157 | } 158 | C.rocksdb_block_based_options_set_filter_policy(opts.c, opts.cFp) 159 | } 160 | 161 | // SetNoBlockCache specify whether block cache should be used or not. 162 | // Default: false 163 | func (opts *BlockBasedTableOptions) SetNoBlockCache(value bool) { 164 | C.rocksdb_block_based_options_set_no_block_cache(opts.c, boolToChar(value)) 165 | } 166 | 167 | // SetBlockCache sets the control over blocks (user data is stored in a set of blocks, and 168 | // a block is the unit of reading from disk). 169 | // 170 | // If set, use the specified cache for blocks. 171 | // If nil, rocksdb will auoptsmatically create and use an 8MB internal cache. 172 | // Default: nil 173 | func (opts *BlockBasedTableOptions) SetBlockCache(cache *Cache) { 174 | opts.cache = cache 175 | C.rocksdb_block_based_options_set_block_cache(opts.c, cache.c) 176 | } 177 | 178 | // SetBlockCacheCompressed sets the cache for compressed blocks. 179 | // If nil, rocksdb will not use a compressed block cache. 180 | // Default: nil 181 | func (opts *BlockBasedTableOptions) SetBlockCacheCompressed(cache *Cache) { 182 | opts.compCache = cache 183 | C.rocksdb_block_based_options_set_block_cache_compressed(opts.c, cache.c) 184 | } 185 | 186 | // SetWholeKeyFiltering specify if whole keys in the filter (not just prefixes) 187 | // should be placed. 188 | // This must generally be true for gets opts be efficient. 189 | // Default: true 190 | func (opts *BlockBasedTableOptions) SetWholeKeyFiltering(value bool) { 191 | C.rocksdb_block_based_options_set_whole_key_filtering(opts.c, boolToChar(value)) 192 | } 193 | 194 | // SetFormatVersion sets the format version. 195 | // We currently have five versions: 196 | // 0 -- This version is currently written out by all RocksDB's versions by 197 | // default. Can be read by really old RocksDB's. Doesn't support changing 198 | // checksum (default is CRC32). 199 | // 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default 200 | // checksum, like xxHash. It is written by RocksDB when 201 | // BlockBasedTableOptions::checksum is something other than kCRC32c. (version 202 | // 0 is silently upconverted) 203 | // 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we 204 | // encode compressed blocks with LZ4, BZip2 and Zlib compression. If you 205 | // don't plan to run RocksDB before version 3.10, you should probably use 206 | // this. 207 | // 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we 208 | // encode the keys in index blocks. If you don't plan to run RocksDB before 209 | // version 5.15, you should probably use this. 210 | // This option only affects newly written tables. When reading existing 211 | // tables, the information about version is read from the footer. 212 | // 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we 213 | // encode the values in index blocks. If you don't plan to run RocksDB before 214 | // version 5.16 and you are using index_block_restart_interval > 1, you should 215 | // probably use this as it would reduce the index size. 216 | // This option only affects newly written tables. When reading existing 217 | // tables, the information about version is read from the footer. 218 | // Default: 2 219 | func (opts *BlockBasedTableOptions) SetFormatVersion(version int) { 220 | C.rocksdb_block_based_options_set_format_version(opts.c, C.int(version)) 221 | } 222 | 223 | // SetIndexType sets the index type used for this table. 224 | // kBinarySearch: 225 | // A space efficient index block that is optimized for 226 | // binary-search-based index. 227 | // 228 | // kHashSearch: 229 | // The hash index, if enabled, will do the hash lookup when 230 | // `Options.prefix_extractor` is provided. 231 | // 232 | // kTwoLevelIndexSearch: 233 | // A two-level index implementation. Both levels are binary search indexes. 234 | // Default: kBinarySearch 235 | func (opts *BlockBasedTableOptions) SetIndexType(value IndexType) { 236 | C.rocksdb_block_based_options_set_index_type(opts.c, C.int(value)) 237 | } 238 | -------------------------------------------------------------------------------- /options_compaction.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // UniversalCompactionStopStyle describes a algorithm used to make a 7 | // compaction request stop picking new files into a single compaction run. 8 | type UniversalCompactionStopStyle uint 9 | 10 | // Compaction stop style types. 11 | const ( 12 | CompactionStopStyleSimilarSize = UniversalCompactionStopStyle(C.rocksdb_similar_size_compaction_stop_style) 13 | CompactionStopStyleTotalSize = UniversalCompactionStopStyle(C.rocksdb_total_size_compaction_stop_style) 14 | ) 15 | 16 | // FIFOCompactionOptions represent all of the available options for 17 | // FIFO compaction. 18 | type FIFOCompactionOptions struct { 19 | c *C.rocksdb_fifo_compaction_options_t 20 | } 21 | 22 | // NewDefaultFIFOCompactionOptions creates a default FIFOCompactionOptions object. 23 | func NewDefaultFIFOCompactionOptions() *FIFOCompactionOptions { 24 | return NewNativeFIFOCompactionOptions(C.rocksdb_fifo_compaction_options_create()) 25 | } 26 | 27 | // NewNativeFIFOCompactionOptions creates a native FIFOCompactionOptions object. 28 | func NewNativeFIFOCompactionOptions(c *C.rocksdb_fifo_compaction_options_t) *FIFOCompactionOptions { 29 | return &FIFOCompactionOptions{c} 30 | } 31 | 32 | // SetMaxTableFilesSize sets the max table file size. 33 | // Once the total sum of table files reaches this, we will delete the oldest 34 | // table file 35 | // Default: 1GB 36 | func (opts *FIFOCompactionOptions) SetMaxTableFilesSize(value uint64) { 37 | C.rocksdb_fifo_compaction_options_set_max_table_files_size(opts.c, C.uint64_t(value)) 38 | } 39 | 40 | // Destroy deallocates the FIFOCompactionOptions object. 41 | func (opts *FIFOCompactionOptions) Destroy() { 42 | C.rocksdb_fifo_compaction_options_destroy(opts.c) 43 | } 44 | 45 | // UniversalCompactionOptions represent all of the available options for 46 | // universal compaction. 47 | type UniversalCompactionOptions struct { 48 | c *C.rocksdb_universal_compaction_options_t 49 | } 50 | 51 | // NewDefaultUniversalCompactionOptions creates a default UniversalCompactionOptions 52 | // object. 53 | func NewDefaultUniversalCompactionOptions() *UniversalCompactionOptions { 54 | return NewNativeUniversalCompactionOptions(C.rocksdb_universal_compaction_options_create()) 55 | } 56 | 57 | // NewNativeUniversalCompactionOptions creates a UniversalCompactionOptions 58 | // object. 59 | func NewNativeUniversalCompactionOptions(c *C.rocksdb_universal_compaction_options_t) *UniversalCompactionOptions { 60 | return &UniversalCompactionOptions{c} 61 | } 62 | 63 | // SetSizeRatio sets the percentage flexibilty while comparing file size. 64 | // If the candidate file(s) size is 1% smaller than the next file's size, 65 | // then include next file into this candidate set. 66 | // Default: 1 67 | func (opts *UniversalCompactionOptions) SetSizeRatio(value uint) { 68 | C.rocksdb_universal_compaction_options_set_size_ratio(opts.c, C.int(value)) 69 | } 70 | 71 | // SetMinMergeWidth sets the minimum number of files in a single compaction run. 72 | // Default: 2 73 | func (opts *UniversalCompactionOptions) SetMinMergeWidth(value uint) { 74 | C.rocksdb_universal_compaction_options_set_min_merge_width(opts.c, C.int(value)) 75 | } 76 | 77 | // SetMaxMergeWidth sets the maximum number of files in a single compaction run. 78 | // Default: UINT_MAX 79 | func (opts *UniversalCompactionOptions) SetMaxMergeWidth(value uint) { 80 | C.rocksdb_universal_compaction_options_set_max_merge_width(opts.c, C.int(value)) 81 | } 82 | 83 | // SetMaxSizeAmplificationPercent sets the size amplification. 84 | // It is defined as the amount (in percentage) of 85 | // additional storage needed to store a single byte of data in the database. 86 | // For example, a size amplification of 2% means that a database that 87 | // contains 100 bytes of user-data may occupy upto 102 bytes of 88 | // physical storage. By this definition, a fully compacted database has 89 | // a size amplification of 0%. Rocksdb uses the following heuristic 90 | // to calculate size amplification: it assumes that all files excluding 91 | // the earliest file contribute to the size amplification. 92 | // Default: 200, which means that a 100 byte database could require upto 93 | // 300 bytes of storage. 94 | func (opts *UniversalCompactionOptions) SetMaxSizeAmplificationPercent(value uint) { 95 | C.rocksdb_universal_compaction_options_set_max_size_amplification_percent(opts.c, C.int(value)) 96 | } 97 | 98 | // SetCompressionSizePercent sets the percentage of compression size. 99 | // 100 | // If this option is set to be -1, all the output files 101 | // will follow compression type specified. 102 | // 103 | // If this option is not negative, we will try to make sure compressed 104 | // size is just above this value. In normal cases, at least this percentage 105 | // of data will be compressed. 106 | // When we are compacting to a new file, here is the criteria whether 107 | // it needs to be compressed: assuming here are the list of files sorted 108 | // by generation time: 109 | // A1...An B1...Bm C1...Ct 110 | // where A1 is the newest and Ct is the oldest, and we are going to compact 111 | // B1...Bm, we calculate the total size of all the files as total_size, as 112 | // well as the total size of C1...Ct as total_C, the compaction output file 113 | // will be compressed iff 114 | // total_C / total_size < this percentage 115 | // Default: -1 116 | func (opts *UniversalCompactionOptions) SetCompressionSizePercent(value int) { 117 | C.rocksdb_universal_compaction_options_set_compression_size_percent(opts.c, C.int(value)) 118 | } 119 | 120 | // SetStopStyle sets the algorithm used to stop picking files into a single compaction run. 121 | // Default: CompactionStopStyleTotalSize 122 | func (opts *UniversalCompactionOptions) SetStopStyle(value UniversalCompactionStopStyle) { 123 | C.rocksdb_universal_compaction_options_set_stop_style(opts.c, C.int(value)) 124 | } 125 | 126 | // Destroy deallocates the UniversalCompactionOptions object. 127 | func (opts *UniversalCompactionOptions) Destroy() { 128 | C.rocksdb_universal_compaction_options_destroy(opts.c) 129 | opts.c = nil 130 | } 131 | -------------------------------------------------------------------------------- /options_compression.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // CompressionOptions represents options for different compression algorithms like Zlib. 4 | type CompressionOptions struct { 5 | WindowBits int 6 | Level int 7 | Strategy int 8 | MaxDictBytes int 9 | } 10 | 11 | // NewDefaultCompressionOptions creates a default CompressionOptions object. 12 | func NewDefaultCompressionOptions() *CompressionOptions { 13 | return NewCompressionOptions(-14, -1, 0, 0) 14 | } 15 | 16 | // NewCompressionOptions creates a CompressionOptions object. 17 | func NewCompressionOptions(windowBits, level, strategy, maxDictBytes int) *CompressionOptions { 18 | return &CompressionOptions{ 19 | WindowBits: windowBits, 20 | Level: level, 21 | Strategy: strategy, 22 | MaxDictBytes: maxDictBytes, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /options_env.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // EnvOptions represents options for env. 7 | type EnvOptions struct { 8 | c *C.rocksdb_envoptions_t 9 | } 10 | 11 | // NewDefaultEnvOptions creates a default EnvOptions object. 12 | func NewDefaultEnvOptions() *EnvOptions { 13 | return NewNativeEnvOptions(C.rocksdb_envoptions_create()) 14 | } 15 | 16 | // NewNativeEnvOptions creates a EnvOptions object. 17 | func NewNativeEnvOptions(c *C.rocksdb_envoptions_t) *EnvOptions { 18 | return &EnvOptions{c: c} 19 | } 20 | 21 | // Destroy deallocates the EnvOptions object. 22 | func (opts *EnvOptions) Destroy() { 23 | C.rocksdb_envoptions_destroy(opts.c) 24 | opts.c = nil 25 | } 26 | -------------------------------------------------------------------------------- /options_flush.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // FlushOptions represent all of the available options when manual flushing the 7 | // database. 8 | type FlushOptions struct { 9 | c *C.rocksdb_flushoptions_t 10 | } 11 | 12 | // NewDefaultFlushOptions creates a default FlushOptions object. 13 | func NewDefaultFlushOptions() *FlushOptions { 14 | return NewNativeFlushOptions(C.rocksdb_flushoptions_create()) 15 | } 16 | 17 | // NewNativeFlushOptions creates a FlushOptions object. 18 | func NewNativeFlushOptions(c *C.rocksdb_flushoptions_t) *FlushOptions { 19 | return &FlushOptions{c} 20 | } 21 | 22 | // SetWait specify if the flush will wait until the flush is done. 23 | // Default: true 24 | func (opts *FlushOptions) SetWait(value bool) { 25 | C.rocksdb_flushoptions_set_wait(opts.c, boolToChar(value)) 26 | } 27 | 28 | // Destroy deallocates the FlushOptions object. 29 | func (opts *FlushOptions) Destroy() { 30 | C.rocksdb_flushoptions_destroy(opts.c) 31 | opts.c = nil 32 | } 33 | -------------------------------------------------------------------------------- /options_ingest.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // IngestExternalFileOptions represents available options when ingesting external files. 7 | type IngestExternalFileOptions struct { 8 | c *C.rocksdb_ingestexternalfileoptions_t 9 | } 10 | 11 | // NewDefaultIngestExternalFileOptions creates a default IngestExternalFileOptions object. 12 | func NewDefaultIngestExternalFileOptions() *IngestExternalFileOptions { 13 | return NewNativeIngestExternalFileOptions(C.rocksdb_ingestexternalfileoptions_create()) 14 | } 15 | 16 | // NewNativeIngestExternalFileOptions creates a IngestExternalFileOptions object. 17 | func NewNativeIngestExternalFileOptions(c *C.rocksdb_ingestexternalfileoptions_t) *IngestExternalFileOptions { 18 | return &IngestExternalFileOptions{c: c} 19 | } 20 | 21 | // SetMoveFiles specifies if it should move the files instead of copying them. 22 | // Default to false. 23 | func (opts *IngestExternalFileOptions) SetMoveFiles(flag bool) { 24 | C.rocksdb_ingestexternalfileoptions_set_move_files(opts.c, boolToChar(flag)) 25 | } 26 | 27 | // SetSnapshotConsistency if specifies the consistency. 28 | // If set to false, an ingested file key could appear in existing snapshots that were created before the 29 | // file was ingested. 30 | // Default to true. 31 | func (opts *IngestExternalFileOptions) SetSnapshotConsistency(flag bool) { 32 | C.rocksdb_ingestexternalfileoptions_set_snapshot_consistency(opts.c, boolToChar(flag)) 33 | } 34 | 35 | // SetAllowGlobalSeqNo sets allow_global_seqno. If set to false,IngestExternalFile() will fail if the file key 36 | // range overlaps with existing keys or tombstones in the DB. 37 | // Default true. 38 | func (opts *IngestExternalFileOptions) SetAllowGlobalSeqNo(flag bool) { 39 | C.rocksdb_ingestexternalfileoptions_set_allow_global_seqno(opts.c, boolToChar(flag)) 40 | } 41 | 42 | // SetAllowBlockingFlush sets allow_blocking_flush. If set to false and the file key range overlaps with 43 | // the memtable key range (memtable flush required), IngestExternalFile will fail. 44 | // Default to true. 45 | func (opts *IngestExternalFileOptions) SetAllowBlockingFlush(flag bool) { 46 | C.rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(opts.c, boolToChar(flag)) 47 | } 48 | 49 | // SetIngestionBehind sets ingest_behind 50 | // Set to true if you would like duplicate keys in the file being ingested 51 | // to be skipped rather than overwriting existing data under that key. 52 | // Usecase: back-fill of some historical data in the database without 53 | // over-writing existing newer version of data. 54 | // This option could only be used if the DB has been running 55 | // with allow_ingest_behind=true since the dawn of time. 56 | // All files will be ingested at the bottommost level with seqno=0. 57 | func (opts *IngestExternalFileOptions) SetIngestionBehind(flag bool) { 58 | C.rocksdb_ingestexternalfileoptions_set_ingest_behind(opts.c, boolToChar(flag)) 59 | } 60 | 61 | // Destroy deallocates the IngestExternalFileOptions object. 62 | func (opts *IngestExternalFileOptions) Destroy() { 63 | C.rocksdb_ingestexternalfileoptions_destroy(opts.c) 64 | opts.c = nil 65 | } 66 | -------------------------------------------------------------------------------- /options_read.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // ReadTier controls fetching of data during a read request. 8 | // An application can issue a read request (via Get/Iterators) and specify 9 | // if that read should process data that ALREADY resides on a specified cache 10 | // level. For example, if an application specifies BlockCacheTier then the 11 | // Get call will process data that is already processed in the memtable or 12 | // the block cache. It will not page in data from the OS cache or data that 13 | // resides in storage. 14 | type ReadTier uint 15 | 16 | const ( 17 | // ReadAllTier reads data in memtable, block cache, OS cache or storage. 18 | ReadAllTier = ReadTier(0) 19 | // BlockCacheTier reads data in memtable or block cache. 20 | BlockCacheTier = ReadTier(1) 21 | ) 22 | 23 | // ReadOptions represent all of the available options when reading from a 24 | // database. 25 | type ReadOptions struct { 26 | c *C.rocksdb_readoptions_t 27 | } 28 | 29 | // NewDefaultReadOptions creates a default ReadOptions object. 30 | func NewDefaultReadOptions() *ReadOptions { 31 | return NewNativeReadOptions(C.rocksdb_readoptions_create()) 32 | } 33 | 34 | // NewNativeReadOptions creates a ReadOptions object. 35 | func NewNativeReadOptions(c *C.rocksdb_readoptions_t) *ReadOptions { 36 | return &ReadOptions{c} 37 | } 38 | 39 | // UnsafeGetReadOptions returns the underlying c read options object. 40 | func (opts *ReadOptions) UnsafeGetReadOptions() unsafe.Pointer { 41 | return unsafe.Pointer(opts.c) 42 | } 43 | 44 | // SetVerifyChecksums speciy if all data read from underlying storage will be 45 | // verified against corresponding checksums. 46 | // Default: false 47 | func (opts *ReadOptions) SetVerifyChecksums(value bool) { 48 | C.rocksdb_readoptions_set_verify_checksums(opts.c, boolToChar(value)) 49 | } 50 | 51 | // SetPrefixSameAsStart Enforce that the iterator only iterates over the same 52 | // prefix as the seek. 53 | // This option is effective only for prefix seeks, i.e. prefix_extractor is 54 | // non-null for the column family and total_order_seek is false. Unlike 55 | // iterate_upper_bound, prefix_same_as_start only works within a prefix 56 | // but in both directions. 57 | // Default: false 58 | func (opts *ReadOptions) SetPrefixSameAsStart(value bool) { 59 | C.rocksdb_readoptions_set_prefix_same_as_start(opts.c, boolToChar(value)) 60 | } 61 | 62 | // SetFillCache specify whether the "data block"/"index block"/"filter block" 63 | // read for this iteration should be cached in memory? 64 | // Callers may wish to set this field to false for bulk scans. 65 | // Default: true 66 | func (opts *ReadOptions) SetFillCache(value bool) { 67 | C.rocksdb_readoptions_set_fill_cache(opts.c, boolToChar(value)) 68 | } 69 | 70 | // SetSnapshot sets the snapshot which should be used for the read. 71 | // The snapshot must belong to the DB that is being read and must 72 | // not have been released. 73 | // Default: nil 74 | func (opts *ReadOptions) SetSnapshot(snap *Snapshot) { 75 | C.rocksdb_readoptions_set_snapshot(opts.c, snap.c) 76 | } 77 | 78 | // SetReadTier specify if this read request should process data that ALREADY 79 | // resides on a particular cache. If the required data is not 80 | // found at the specified cache, then Status::Incomplete is returned. 81 | // Default: ReadAllTier 82 | func (opts *ReadOptions) SetReadTier(value ReadTier) { 83 | C.rocksdb_readoptions_set_read_tier(opts.c, C.int(value)) 84 | } 85 | 86 | // SetTailing specify if to create a tailing iterator. 87 | // A special iterator that has a view of the complete database 88 | // (i.e. it can also be used to read newly added data) and 89 | // is optimized for sequential reads. It will return records 90 | // that were inserted into the database after the creation of the iterator. 91 | // Default: false 92 | func (opts *ReadOptions) SetTailing(value bool) { 93 | C.rocksdb_readoptions_set_tailing(opts.c, boolToChar(value)) 94 | } 95 | 96 | // SetIterateUpperBound specifies "iterate_upper_bound", which defines 97 | // the extent upto which the forward iterator can returns entries. 98 | // Once the bound is reached, Valid() will be false. 99 | // "iterate_upper_bound" is exclusive ie the bound value is 100 | // not a valid entry. If iterator_extractor is not null, the Seek target 101 | // and iterator_upper_bound need to have the same prefix. 102 | // This is because ordering is not guaranteed outside of prefix domain. 103 | // There is no lower bound on the iterator. If needed, that can be easily 104 | // implemented. 105 | // Default: nullptr 106 | func (opts *ReadOptions) SetIterateUpperBound(key []byte) { 107 | cKey := byteToChar(key) 108 | cKeyLen := C.size_t(len(key)) 109 | C.rocksdb_readoptions_set_iterate_upper_bound(opts.c, cKey, cKeyLen) 110 | } 111 | 112 | // SetPinData specifies the value of "pin_data". If true, it keeps the blocks 113 | // loaded by the iterator pinned in memory as long as the iterator is not deleted, 114 | // If used when reading from tables created with 115 | // BlockBasedTableOptions::use_delta_encoding = false, 116 | // Iterator's property "rocksdb.iterator.is-key-pinned" is guaranteed to 117 | // return 1. 118 | // Default: false 119 | func (opts *ReadOptions) SetPinData(value bool) { 120 | C.rocksdb_readoptions_set_pin_data(opts.c, boolToChar(value)) 121 | } 122 | 123 | // SetReadaheadSize specifies the value of "readahead_size". 124 | // If non-zero, NewIterator will create a new table reader which 125 | // performs reads of the given size. Using a large size (> 2MB) can 126 | // improve the performance of forward iteration on spinning disks. 127 | // Default: 0 128 | func (opts *ReadOptions) SetReadaheadSize(value uint64) { 129 | C.rocksdb_readoptions_set_readahead_size(opts.c, C.size_t(value)) 130 | } 131 | 132 | // Destroy deallocates the ReadOptions object. 133 | func (opts *ReadOptions) Destroy() { 134 | C.rocksdb_readoptions_destroy(opts.c) 135 | opts.c = nil 136 | } 137 | -------------------------------------------------------------------------------- /options_transaction.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // TransactionOptions represent all of the available options options for 7 | // a transaction on the database. 8 | type TransactionOptions struct { 9 | c *C.rocksdb_transaction_options_t 10 | } 11 | 12 | // NewDefaultTransactionOptions creates a default TransactionOptions object. 13 | func NewDefaultTransactionOptions() *TransactionOptions { 14 | return NewNativeTransactionOptions(C.rocksdb_transaction_options_create()) 15 | } 16 | 17 | // NewNativeTransactionOptions creates a TransactionOptions object. 18 | func NewNativeTransactionOptions(c *C.rocksdb_transaction_options_t) *TransactionOptions { 19 | return &TransactionOptions{c} 20 | } 21 | 22 | // SetSetSnapshot to true is the same as calling 23 | // Transaction::SetSnapshot(). 24 | func (opts *TransactionOptions) SetSetSnapshot(value bool) { 25 | C.rocksdb_transaction_options_set_set_snapshot(opts.c, boolToChar(value)) 26 | } 27 | 28 | // SetDeadlockDetect to true means that before acquiring locks, this transaction will 29 | // check if doing so will cause a deadlock. If so, it will return with 30 | // Status::Busy. The user should retry their transaction. 31 | func (opts *TransactionOptions) SetDeadlockDetect(value bool) { 32 | C.rocksdb_transaction_options_set_deadlock_detect(opts.c, boolToChar(value)) 33 | } 34 | 35 | // SetLockTimeout positive, specifies the wait timeout in milliseconds when 36 | // a transaction attempts to lock a key. 37 | // If 0, no waiting is done if a lock cannot instantly be acquired. 38 | // If negative, TransactionDBOptions::transaction_lock_timeout will be used 39 | func (opts *TransactionOptions) SetLockTimeout(lock_timeout int64) { 40 | C.rocksdb_transaction_options_set_lock_timeout(opts.c, C.int64_t(lock_timeout)) 41 | } 42 | 43 | // SetExpiration sets the Expiration duration in milliseconds. 44 | // If non-negative, transactions that last longer than this many milliseconds will fail to commit. 45 | // If not set, a forgotten transaction that is never committed, rolled back, or deleted 46 | // will never relinquish any locks it holds. This could prevent keys from 47 | // being written by other writers. 48 | func (opts *TransactionOptions) SetExpiration(expiration int64) { 49 | C.rocksdb_transaction_options_set_expiration(opts.c, C.int64_t(expiration)) 50 | } 51 | 52 | // SetDeadlockDetectDepth sets the number of traversals to make during deadlock detection. 53 | func (opts *TransactionOptions) SetDeadlockDetectDepth(depth int64) { 54 | C.rocksdb_transaction_options_set_deadlock_detect_depth(opts.c, C.int64_t(depth)) 55 | } 56 | 57 | // SetMaxWriteBatchSize sets the maximum number of bytes used for the write batch. 0 means no limit. 58 | func (opts *TransactionOptions) SetMaxWriteBatchSize(size uint64) { 59 | C.rocksdb_transaction_options_set_max_write_batch_size(opts.c, C.size_t(size)) 60 | } 61 | 62 | // Destroy deallocates the TransactionOptions object. 63 | func (opts *TransactionOptions) Destroy() { 64 | C.rocksdb_transaction_options_destroy(opts.c) 65 | opts.c = nil 66 | } 67 | -------------------------------------------------------------------------------- /options_transactiondb.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // TransactionDBOptions represent all of the available options when opening a transactional database 7 | // with OpenTransactionDb. 8 | type TransactionDBOptions struct { 9 | c *C.rocksdb_transactiondb_options_t 10 | } 11 | 12 | // NewDefaultTransactionDBOptions creates a default TransactionDBOptions object. 13 | func NewDefaultTransactionDBOptions() *TransactionDBOptions { 14 | return NewNativeTransactionDBOptions(C.rocksdb_transactiondb_options_create()) 15 | } 16 | 17 | // NewDefaultTransactionDBOptions creates a TransactionDBOptions object. 18 | func NewNativeTransactionDBOptions(c *C.rocksdb_transactiondb_options_t) *TransactionDBOptions { 19 | return &TransactionDBOptions{c} 20 | } 21 | 22 | // SetMaxNumLocks sets the maximum number of keys that can be locked at the same time 23 | // per column family. 24 | // If the number of locked keys is greater than max_num_locks, transaction 25 | // writes (or GetForUpdate) will return an error. 26 | // If this value is not positive, no limit will be enforced. 27 | func (opts *TransactionDBOptions) SetMaxNumLocks(max_num_locks int64) { 28 | C.rocksdb_transactiondb_options_set_max_num_locks(opts.c, C.int64_t(max_num_locks)) 29 | } 30 | 31 | // SetNumStripes sets the concurrency level. 32 | // Increasing this value will increase the concurrency by dividing the lock 33 | // table (per column family) into more sub-tables, each with their own 34 | // separate 35 | // mutex. 36 | func (opts *TransactionDBOptions) SetNumStripes(num_stripes uint64) { 37 | C.rocksdb_transactiondb_options_set_num_stripes(opts.c, C.size_t(num_stripes)) 38 | } 39 | 40 | // SetTransactionLockTimeout if positive, specifies the default wait timeout in milliseconds when 41 | // a transaction attempts to lock a key if not specified by 42 | // TransactionOptions::lock_timeout. 43 | // 44 | // If 0, no waiting is done if a lock cannot instantly be acquired. 45 | // If negative, there is no timeout. Not using a timeout is not recommended 46 | // as it can lead to deadlocks. Currently, there is no deadlock-detection to 47 | // recover from a deadlock. 48 | func (opts *TransactionDBOptions) SetTransactionLockTimeout(txn_lock_timeout int64) { 49 | C.rocksdb_transactiondb_options_set_transaction_lock_timeout(opts.c, C.int64_t(txn_lock_timeout)) 50 | } 51 | 52 | // SetDefaultLockTimeout if posititve, specifies the wait timeout in milliseconds when writing a key 53 | // OUTSIDE of a transaction (ie by calling DB::Put(),Merge(),Delete(),Write() 54 | // directly). 55 | // If 0, no waiting is done if a lock cannot instantly be acquired. 56 | // If negative, there is no timeout and will block indefinitely when acquiring 57 | // a lock. 58 | // 59 | // Not using a timeout can lead to deadlocks. Currently, there 60 | // is no deadlock-detection to recover from a deadlock. While DB writes 61 | // cannot deadlock with other DB writes, they can deadlock with a transaction. 62 | // A negative timeout should only be used if all transactions have a small 63 | // expiration set. 64 | func (opts *TransactionDBOptions) SetDefaultLockTimeout(default_lock_timeout int64) { 65 | C.rocksdb_transactiondb_options_set_default_lock_timeout(opts.c, C.int64_t(default_lock_timeout)) 66 | } 67 | 68 | // Destroy deallocates the TransactionDBOptions object. 69 | func (opts *TransactionDBOptions) Destroy() { 70 | C.rocksdb_transactiondb_options_destroy(opts.c) 71 | opts.c = nil 72 | } 73 | -------------------------------------------------------------------------------- /options_write.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // WriteOptions represent all of the available options when writing to a 7 | // database. 8 | type WriteOptions struct { 9 | c *C.rocksdb_writeoptions_t 10 | } 11 | 12 | // NewDefaultWriteOptions creates a default WriteOptions object. 13 | func NewDefaultWriteOptions() *WriteOptions { 14 | return NewNativeWriteOptions(C.rocksdb_writeoptions_create()) 15 | } 16 | 17 | // NewNativeWriteOptions creates a WriteOptions object. 18 | func NewNativeWriteOptions(c *C.rocksdb_writeoptions_t) *WriteOptions { 19 | return &WriteOptions{c} 20 | } 21 | 22 | // SetSync sets the sync mode. If true, the write will be flushed 23 | // from the operating system buffer cache before the write is considered complete. 24 | // If this flag is true, writes will be slower. 25 | // Default: false 26 | func (opts *WriteOptions) SetSync(value bool) { 27 | C.rocksdb_writeoptions_set_sync(opts.c, boolToChar(value)) 28 | } 29 | 30 | // DisableWAL sets whether WAL should be active or not. 31 | // If true, writes will not first go to the write ahead log, 32 | // and the write may got lost after a crash. 33 | // Default: false 34 | func (opts *WriteOptions) DisableWAL(value bool) { 35 | C.rocksdb_writeoptions_disable_WAL(opts.c, C.int(btoi(value))) 36 | } 37 | 38 | // Destroy deallocates the WriteOptions object. 39 | func (opts *WriteOptions) Destroy() { 40 | C.rocksdb_writeoptions_destroy(opts.c) 41 | opts.c = nil 42 | } 43 | -------------------------------------------------------------------------------- /ratelimiter.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | // RateLimiter, is used to control write rate of flush and 8 | // compaction. 9 | type RateLimiter struct { 10 | c *C.rocksdb_ratelimiter_t 11 | } 12 | 13 | // NewDefaultRateLimiter creates a default RateLimiter object. 14 | func NewRateLimiter(rate_bytes_per_sec, refill_period_us int64, fairness int32) *RateLimiter { 15 | return NewNativeRateLimiter(C.rocksdb_ratelimiter_create( 16 | C.int64_t(rate_bytes_per_sec), 17 | C.int64_t(refill_period_us), 18 | C.int32_t(fairness), 19 | )) 20 | } 21 | 22 | // NewNativeRateLimiter creates a native RateLimiter object. 23 | func NewNativeRateLimiter(c *C.rocksdb_ratelimiter_t) *RateLimiter { 24 | return &RateLimiter{c} 25 | } 26 | 27 | // Destroy deallocates the RateLimiter object. 28 | func (self *RateLimiter) Destroy() { 29 | C.rocksdb_ratelimiter_destroy(self.c) 30 | self.c = nil 31 | } 32 | -------------------------------------------------------------------------------- /slice.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // Slice is used as a wrapper for non-copy values 9 | type Slice struct { 10 | data *C.char 11 | size C.size_t 12 | freed bool 13 | } 14 | 15 | type Slices []*Slice 16 | 17 | func (slices Slices) Destroy() { 18 | for _, s := range slices { 19 | s.Free() 20 | } 21 | } 22 | 23 | // NewSlice returns a slice with the given data. 24 | func NewSlice(data *C.char, size C.size_t) *Slice { 25 | return &Slice{data, size, false} 26 | } 27 | 28 | // StringToSlice is similar to NewSlice, but can be called with 29 | // a Go string type. This exists to make testing integration 30 | // with Gorocksdb easier. 31 | func StringToSlice(data string) *Slice { 32 | return NewSlice(C.CString(data), C.size_t(len(data))) 33 | } 34 | 35 | // Data returns the data of the slice. If the key doesn't exist this will be a 36 | // nil slice. 37 | func (s *Slice) Data() []byte { 38 | return charToByte(s.data, s.size) 39 | } 40 | 41 | // Size returns the size of the data. 42 | func (s *Slice) Size() int { 43 | return int(s.size) 44 | } 45 | 46 | // Exists returns if the key exists 47 | func (s *Slice) Exists() bool { 48 | return s.data != nil 49 | } 50 | 51 | // Free frees the slice data. 52 | func (s *Slice) Free() { 53 | if !s.freed { 54 | C.rocksdb_free(unsafe.Pointer(s.data)) 55 | s.freed = true 56 | } 57 | } 58 | 59 | // PinnableSliceHandle represents a handle to a PinnableSlice. 60 | type PinnableSliceHandle struct { 61 | c *C.rocksdb_pinnableslice_t 62 | } 63 | 64 | // NewNativePinnableSliceHandle creates a PinnableSliceHandle object. 65 | func NewNativePinnableSliceHandle(c *C.rocksdb_pinnableslice_t) *PinnableSliceHandle { 66 | return &PinnableSliceHandle{c} 67 | } 68 | 69 | // Data returns the data of the slice. 70 | func (h *PinnableSliceHandle) Data() []byte { 71 | if h.c == nil { 72 | return nil 73 | } 74 | 75 | var cValLen C.size_t 76 | cValue := C.rocksdb_pinnableslice_value(h.c, &cValLen) 77 | 78 | return charToByte(cValue, cValLen) 79 | } 80 | 81 | // Destroy calls the destructor of the underlying pinnable slice handle. 82 | func (h *PinnableSliceHandle) Destroy() { 83 | C.rocksdb_pinnableslice_destroy(h.c) 84 | } 85 | -------------------------------------------------------------------------------- /slice_transform.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // A SliceTransform can be used as a prefix extractor. 7 | type SliceTransform interface { 8 | // Transform a src in domain to a dst in the range. 9 | Transform(src []byte) []byte 10 | 11 | // Determine whether this is a valid src upon the function applies. 12 | InDomain(src []byte) bool 13 | 14 | // Determine whether dst=Transform(src) for some src. 15 | InRange(src []byte) bool 16 | 17 | // Return the name of this transformation. 18 | Name() string 19 | } 20 | 21 | // NewFixedPrefixTransform creates a new fixed prefix transform. 22 | func NewFixedPrefixTransform(prefixLen int) SliceTransform { 23 | return NewNativeSliceTransform(C.rocksdb_slicetransform_create_fixed_prefix(C.size_t(prefixLen))) 24 | } 25 | 26 | // NewNoopPrefixTransform creates a new no-op prefix transform. 27 | func NewNoopPrefixTransform() SliceTransform { 28 | return NewNativeSliceTransform(C.rocksdb_slicetransform_create_noop()) 29 | } 30 | 31 | // NewNativeSliceTransform creates a SliceTransform object. 32 | func NewNativeSliceTransform(c *C.rocksdb_slicetransform_t) SliceTransform { 33 | return nativeSliceTransform{c} 34 | } 35 | 36 | type nativeSliceTransform struct { 37 | c *C.rocksdb_slicetransform_t 38 | } 39 | 40 | func (st nativeSliceTransform) Transform(src []byte) []byte { return nil } 41 | func (st nativeSliceTransform) InDomain(src []byte) bool { return false } 42 | func (st nativeSliceTransform) InRange(src []byte) bool { return false } 43 | func (st nativeSliceTransform) Name() string { return "" } 44 | 45 | // Hold references to slice transforms. 46 | var sliceTransforms = NewCOWList() 47 | 48 | type sliceTransformWrapper struct { 49 | name *C.char 50 | sliceTransform SliceTransform 51 | } 52 | 53 | func registerSliceTransform(st SliceTransform) int { 54 | return sliceTransforms.Append(sliceTransformWrapper{C.CString(st.Name()), st}) 55 | } 56 | 57 | //export gorocksdb_slicetransform_transform 58 | func gorocksdb_slicetransform_transform(idx int, cKey *C.char, cKeyLen C.size_t, cDstLen *C.size_t) *C.char { 59 | key := charToByte(cKey, cKeyLen) 60 | dst := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.Transform(key) 61 | *cDstLen = C.size_t(len(dst)) 62 | return cByteSlice(dst) 63 | } 64 | 65 | //export gorocksdb_slicetransform_in_domain 66 | func gorocksdb_slicetransform_in_domain(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { 67 | key := charToByte(cKey, cKeyLen) 68 | inDomain := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InDomain(key) 69 | return boolToChar(inDomain) 70 | } 71 | 72 | //export gorocksdb_slicetransform_in_range 73 | func gorocksdb_slicetransform_in_range(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { 74 | key := charToByte(cKey, cKeyLen) 75 | inRange := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InRange(key) 76 | return boolToChar(inRange) 77 | } 78 | 79 | //export gorocksdb_slicetransform_name 80 | func gorocksdb_slicetransform_name(idx int) *C.char { 81 | return sliceTransforms.Get(idx).(sliceTransformWrapper).name 82 | } 83 | -------------------------------------------------------------------------------- /slice_transform_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/facebookgo/ensure" 7 | ) 8 | 9 | func TestSliceTransform(t *testing.T) { 10 | db := newTestDB(t, "TestSliceTransform", func(opts *Options) { 11 | opts.SetPrefixExtractor(&testSliceTransform{}) 12 | }) 13 | defer db.Close() 14 | 15 | wo := NewDefaultWriteOptions() 16 | ensure.Nil(t, db.Put(wo, []byte("foo1"), []byte("foo"))) 17 | ensure.Nil(t, db.Put(wo, []byte("foo2"), []byte("foo"))) 18 | ensure.Nil(t, db.Put(wo, []byte("bar1"), []byte("bar"))) 19 | 20 | iter := db.NewIterator(NewDefaultReadOptions()) 21 | defer iter.Close() 22 | prefix := []byte("foo") 23 | numFound := 0 24 | for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { 25 | numFound++ 26 | } 27 | ensure.Nil(t, iter.Err()) 28 | ensure.DeepEqual(t, numFound, 2) 29 | } 30 | 31 | func TestFixedPrefixTransformOpen(t *testing.T) { 32 | db := newTestDB(t, "TestFixedPrefixTransformOpen", func(opts *Options) { 33 | opts.SetPrefixExtractor(NewFixedPrefixTransform(3)) 34 | }) 35 | defer db.Close() 36 | } 37 | 38 | func TestNewNoopPrefixTransform(t *testing.T) { 39 | db := newTestDB(t, "TestNewNoopPrefixTransform", func(opts *Options) { 40 | opts.SetPrefixExtractor(NewNoopPrefixTransform()) 41 | }) 42 | defer db.Close() 43 | } 44 | 45 | type testSliceTransform struct { 46 | initiated bool 47 | } 48 | 49 | func (st *testSliceTransform) Name() string { return "gorocksdb.test" } 50 | func (st *testSliceTransform) Transform(src []byte) []byte { return src[0:3] } 51 | func (st *testSliceTransform) InDomain(src []byte) bool { return len(src) >= 3 } 52 | func (st *testSliceTransform) InRange(src []byte) bool { return len(src) == 3 } 53 | -------------------------------------------------------------------------------- /snapshot.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // Snapshot provides a consistent view of read operations in a DB. 7 | type Snapshot struct { 8 | c *C.rocksdb_snapshot_t 9 | } 10 | 11 | // NewNativeSnapshot creates a Snapshot object. 12 | func NewNativeSnapshot(c *C.rocksdb_snapshot_t) *Snapshot { 13 | return &Snapshot{c} 14 | } 15 | -------------------------------------------------------------------------------- /sst_file_writer.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "errors" 9 | "unsafe" 10 | ) 11 | 12 | // SSTFileWriter is used to create sst files that can be added to database later. 13 | // All keys in files generated by SstFileWriter will have sequence number = 0. 14 | type SSTFileWriter struct { 15 | c *C.rocksdb_sstfilewriter_t 16 | } 17 | 18 | // NewSSTFileWriter creates an SSTFileWriter object. 19 | func NewSSTFileWriter(opts *EnvOptions, dbOpts *Options) *SSTFileWriter { 20 | c := C.rocksdb_sstfilewriter_create(opts.c, dbOpts.c) 21 | return &SSTFileWriter{c: c} 22 | } 23 | 24 | // Open prepares SstFileWriter to write into file located at "path". 25 | func (w *SSTFileWriter) Open(path string) error { 26 | var ( 27 | cErr *C.char 28 | cPath = C.CString(path) 29 | ) 30 | defer C.free(unsafe.Pointer(cPath)) 31 | C.rocksdb_sstfilewriter_open(w.c, cPath, &cErr) 32 | if cErr != nil { 33 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 34 | return errors.New(C.GoString(cErr)) 35 | } 36 | return nil 37 | } 38 | 39 | // Add adds key, value to currently opened file. 40 | // REQUIRES: key is after any previously added key according to comparator. 41 | func (w *SSTFileWriter) Add(key, value []byte) error { 42 | cKey := byteToChar(key) 43 | cValue := byteToChar(value) 44 | var cErr *C.char 45 | C.rocksdb_sstfilewriter_add(w.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 46 | if cErr != nil { 47 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 48 | return errors.New(C.GoString(cErr)) 49 | } 50 | return nil 51 | } 52 | 53 | // Finish finishes writing to sst file and close file. 54 | func (w *SSTFileWriter) Finish() error { 55 | var cErr *C.char 56 | C.rocksdb_sstfilewriter_finish(w.c, &cErr) 57 | if cErr != nil { 58 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 59 | return errors.New(C.GoString(cErr)) 60 | } 61 | return nil 62 | } 63 | 64 | // Destroy destroys the SSTFileWriter object. 65 | func (w *SSTFileWriter) Destroy() { 66 | C.rocksdb_sstfilewriter_destroy(w.c) 67 | } 68 | -------------------------------------------------------------------------------- /staticflag_linux.go: -------------------------------------------------------------------------------- 1 | // +build static 2 | 3 | package gorocksdb 4 | 5 | // #cgo LDFLAGS: -l:librocksdb.a -l:libstdc++.a -lm -ldl 6 | import "C" 7 | -------------------------------------------------------------------------------- /transaction.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "errors" 9 | "unsafe" 10 | ) 11 | 12 | // Transaction is used with TransactionDB for transaction support. 13 | type Transaction struct { 14 | c *C.rocksdb_transaction_t 15 | } 16 | 17 | // NewNativeTransaction creates a Transaction object. 18 | func NewNativeTransaction(c *C.rocksdb_transaction_t) *Transaction { 19 | return &Transaction{c} 20 | } 21 | 22 | // Commit commits the transaction to the database. 23 | func (transaction *Transaction) Commit() error { 24 | var ( 25 | cErr *C.char 26 | ) 27 | C.rocksdb_transaction_commit(transaction.c, &cErr) 28 | if cErr != nil { 29 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 30 | return errors.New(C.GoString(cErr)) 31 | } 32 | return nil 33 | } 34 | 35 | // Rollback performs a rollback on the transaction. 36 | func (transaction *Transaction) Rollback() error { 37 | var ( 38 | cErr *C.char 39 | ) 40 | C.rocksdb_transaction_rollback(transaction.c, &cErr) 41 | 42 | if cErr != nil { 43 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 44 | return errors.New(C.GoString(cErr)) 45 | } 46 | return nil 47 | } 48 | 49 | // Get returns the data associated with the key from the database given this transaction. 50 | func (transaction *Transaction) Get(opts *ReadOptions, key []byte) (*Slice, error) { 51 | var ( 52 | cErr *C.char 53 | cValLen C.size_t 54 | cKey = byteToChar(key) 55 | ) 56 | cValue := C.rocksdb_transaction_get( 57 | transaction.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr, 58 | ) 59 | if cErr != nil { 60 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 61 | return nil, errors.New(C.GoString(cErr)) 62 | } 63 | return NewSlice(cValue, cValLen), nil 64 | } 65 | 66 | // GetForUpdate queries the data associated with the key and puts an exclusive lock on the key from the database given this transaction. 67 | func (transaction *Transaction) GetForUpdate(opts *ReadOptions, key []byte) (*Slice, error) { 68 | var ( 69 | cErr *C.char 70 | cValLen C.size_t 71 | cKey = byteToChar(key) 72 | ) 73 | cValue := C.rocksdb_transaction_get_for_update( 74 | transaction.c, opts.c, cKey, C.size_t(len(key)), &cValLen, C.uchar(byte(1)) /*exclusive*/, &cErr, 75 | ) 76 | if cErr != nil { 77 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 78 | return nil, errors.New(C.GoString(cErr)) 79 | } 80 | return NewSlice(cValue, cValLen), nil 81 | } 82 | 83 | // Put writes data associated with a key to the transaction. 84 | func (transaction *Transaction) Put(key, value []byte) error { 85 | var ( 86 | cErr *C.char 87 | cKey = byteToChar(key) 88 | cValue = byteToChar(value) 89 | ) 90 | C.rocksdb_transaction_put( 91 | transaction.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr, 92 | ) 93 | if cErr != nil { 94 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 95 | return errors.New(C.GoString(cErr)) 96 | } 97 | return nil 98 | } 99 | 100 | // Delete removes the data associated with the key from the transaction. 101 | func (transaction *Transaction) Delete(key []byte) error { 102 | var ( 103 | cErr *C.char 104 | cKey = byteToChar(key) 105 | ) 106 | C.rocksdb_transaction_delete(transaction.c, cKey, C.size_t(len(key)), &cErr) 107 | if cErr != nil { 108 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 109 | return errors.New(C.GoString(cErr)) 110 | } 111 | return nil 112 | } 113 | 114 | // NewIterator returns an Iterator over the database that uses the 115 | // ReadOptions given. 116 | func (transaction *Transaction) NewIterator(opts *ReadOptions) *Iterator { 117 | return NewNativeIterator( 118 | unsafe.Pointer(C.rocksdb_transaction_create_iterator(transaction.c, opts.c))) 119 | } 120 | 121 | // Destroy deallocates the transaction object. 122 | func (transaction *Transaction) Destroy() { 123 | C.rocksdb_transaction_destroy(transaction.c) 124 | transaction.c = nil 125 | } 126 | -------------------------------------------------------------------------------- /transactiondb.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "errors" 8 | "unsafe" 9 | ) 10 | 11 | // TransactionDB is a reusable handle to a RocksDB transactional database on disk, created by OpenTransactionDb. 12 | type TransactionDB struct { 13 | c *C.rocksdb_transactiondb_t 14 | name string 15 | opts *Options 16 | transactionDBOpts *TransactionDBOptions 17 | } 18 | 19 | // OpenTransactionDb opens a database with the specified options. 20 | func OpenTransactionDb( 21 | opts *Options, 22 | transactionDBOpts *TransactionDBOptions, 23 | name string, 24 | ) (*TransactionDB, error) { 25 | var ( 26 | cErr *C.char 27 | cName = C.CString(name) 28 | ) 29 | defer C.free(unsafe.Pointer(cName)) 30 | db := C.rocksdb_transactiondb_open( 31 | opts.c, transactionDBOpts.c, cName, &cErr) 32 | if cErr != nil { 33 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 34 | return nil, errors.New(C.GoString(cErr)) 35 | } 36 | return &TransactionDB{ 37 | name: name, 38 | c: db, 39 | opts: opts, 40 | transactionDBOpts: transactionDBOpts, 41 | }, nil 42 | } 43 | 44 | // NewSnapshot creates a new snapshot of the database. 45 | func (db *TransactionDB) NewSnapshot() *Snapshot { 46 | return NewNativeSnapshot(C.rocksdb_transactiondb_create_snapshot(db.c)) 47 | } 48 | 49 | // ReleaseSnapshot releases the snapshot and its resources. 50 | func (db *TransactionDB) ReleaseSnapshot(snapshot *Snapshot) { 51 | C.rocksdb_transactiondb_release_snapshot(db.c, snapshot.c) 52 | snapshot.c = nil 53 | } 54 | 55 | // TransactionBegin begins a new transaction 56 | // with the WriteOptions and TransactionOptions given. 57 | func (db *TransactionDB) TransactionBegin( 58 | opts *WriteOptions, 59 | transactionOpts *TransactionOptions, 60 | oldTransaction *Transaction, 61 | ) *Transaction { 62 | if oldTransaction != nil { 63 | return NewNativeTransaction(C.rocksdb_transaction_begin( 64 | db.c, 65 | opts.c, 66 | transactionOpts.c, 67 | oldTransaction.c, 68 | )) 69 | } 70 | 71 | return NewNativeTransaction(C.rocksdb_transaction_begin( 72 | db.c, opts.c, transactionOpts.c, nil)) 73 | } 74 | 75 | // Get returns the data associated with the key from the database. 76 | func (db *TransactionDB) Get(opts *ReadOptions, key []byte) (*Slice, error) { 77 | var ( 78 | cErr *C.char 79 | cValLen C.size_t 80 | cKey = byteToChar(key) 81 | ) 82 | cValue := C.rocksdb_transactiondb_get( 83 | db.c, opts.c, cKey, C.size_t(len(key)), &cValLen, &cErr, 84 | ) 85 | if cErr != nil { 86 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 87 | return nil, errors.New(C.GoString(cErr)) 88 | } 89 | return NewSlice(cValue, cValLen), nil 90 | } 91 | 92 | // Put writes data associated with a key to the database. 93 | func (db *TransactionDB) Put(opts *WriteOptions, key, value []byte) error { 94 | var ( 95 | cErr *C.char 96 | cKey = byteToChar(key) 97 | cValue = byteToChar(value) 98 | ) 99 | C.rocksdb_transactiondb_put( 100 | db.c, opts.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr, 101 | ) 102 | if cErr != nil { 103 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 104 | return errors.New(C.GoString(cErr)) 105 | } 106 | return nil 107 | } 108 | 109 | // Delete removes the data associated with the key from the database. 110 | func (db *TransactionDB) Delete(opts *WriteOptions, key []byte) error { 111 | var ( 112 | cErr *C.char 113 | cKey = byteToChar(key) 114 | ) 115 | C.rocksdb_transactiondb_delete(db.c, opts.c, cKey, C.size_t(len(key)), &cErr) 116 | if cErr != nil { 117 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 118 | return errors.New(C.GoString(cErr)) 119 | } 120 | return nil 121 | } 122 | 123 | // NewCheckpoint creates a new Checkpoint for this db. 124 | func (db *TransactionDB) NewCheckpoint() (*Checkpoint, error) { 125 | var ( 126 | cErr *C.char 127 | ) 128 | cCheckpoint := C.rocksdb_transactiondb_checkpoint_object_create( 129 | db.c, &cErr, 130 | ) 131 | if cErr != nil { 132 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 133 | return nil, errors.New(C.GoString(cErr)) 134 | } 135 | 136 | return NewNativeCheckpoint(cCheckpoint), nil 137 | } 138 | 139 | // Close closes the database. 140 | func (transactionDB *TransactionDB) Close() { 141 | C.rocksdb_transactiondb_close(transactionDB.c) 142 | transactionDB.c = nil 143 | } 144 | -------------------------------------------------------------------------------- /transactiondb_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/facebookgo/ensure" 8 | ) 9 | 10 | func TestOpenTransactionDb(t *testing.T) { 11 | db := newTestTransactionDB(t, "TestOpenTransactionDb", nil) 12 | defer db.Close() 13 | } 14 | 15 | func TestTransactionDBCRUD(t *testing.T) { 16 | db := newTestTransactionDB(t, "TestTransactionDBGet", nil) 17 | defer db.Close() 18 | 19 | var ( 20 | givenKey = []byte("hello") 21 | givenVal1 = []byte("world1") 22 | givenVal2 = []byte("world2") 23 | givenTxnKey = []byte("hello2") 24 | givenTxnKey2 = []byte("hello3") 25 | givenTxnVal1 = []byte("whatawonderful") 26 | wo = NewDefaultWriteOptions() 27 | ro = NewDefaultReadOptions() 28 | to = NewDefaultTransactionOptions() 29 | ) 30 | 31 | // create 32 | ensure.Nil(t, db.Put(wo, givenKey, givenVal1)) 33 | 34 | // retrieve 35 | v1, err := db.Get(ro, givenKey) 36 | defer v1.Free() 37 | ensure.Nil(t, err) 38 | ensure.DeepEqual(t, v1.Data(), givenVal1) 39 | 40 | // update 41 | ensure.Nil(t, db.Put(wo, givenKey, givenVal2)) 42 | v2, err := db.Get(ro, givenKey) 43 | defer v2.Free() 44 | ensure.Nil(t, err) 45 | ensure.DeepEqual(t, v2.Data(), givenVal2) 46 | 47 | // delete 48 | ensure.Nil(t, db.Delete(wo, givenKey)) 49 | v3, err := db.Get(ro, givenKey) 50 | defer v3.Free() 51 | ensure.Nil(t, err) 52 | ensure.True(t, v3.Data() == nil) 53 | 54 | // transaction 55 | txn := db.TransactionBegin(wo, to, nil) 56 | defer txn.Destroy() 57 | // create 58 | ensure.Nil(t, txn.Put(givenTxnKey, givenTxnVal1)) 59 | v4, err := txn.Get(ro, givenTxnKey) 60 | defer v4.Free() 61 | ensure.Nil(t, err) 62 | ensure.DeepEqual(t, v4.Data(), givenTxnVal1) 63 | 64 | ensure.Nil(t, txn.Commit()) 65 | v5, err := db.Get(ro, givenTxnKey) 66 | defer v5.Free() 67 | ensure.Nil(t, err) 68 | ensure.DeepEqual(t, v5.Data(), givenTxnVal1) 69 | 70 | // transaction 71 | txn2 := db.TransactionBegin(wo, to, nil) 72 | defer txn2.Destroy() 73 | // create 74 | ensure.Nil(t, txn2.Put(givenTxnKey2, givenTxnVal1)) 75 | // rollback 76 | ensure.Nil(t, txn2.Rollback()) 77 | 78 | v6, err := txn2.Get(ro, givenTxnKey2) 79 | defer v6.Free() 80 | ensure.Nil(t, err) 81 | ensure.True(t, v6.Data() == nil) 82 | // transaction 83 | txn3 := db.TransactionBegin(wo, to, nil) 84 | defer txn3.Destroy() 85 | // delete 86 | ensure.Nil(t, txn3.Delete(givenTxnKey)) 87 | ensure.Nil(t, txn3.Commit()) 88 | 89 | v7, err := db.Get(ro, givenTxnKey) 90 | defer v7.Free() 91 | ensure.Nil(t, err) 92 | ensure.True(t, v7.Data() == nil) 93 | 94 | } 95 | 96 | func TestTransactionDBGetForUpdate(t *testing.T) { 97 | lockTimeoutMilliSec := int64(50) 98 | applyOpts := func(opts *Options, transactionDBOpts *TransactionDBOptions) { 99 | transactionDBOpts.SetTransactionLockTimeout(lockTimeoutMilliSec) 100 | } 101 | db := newTestTransactionDB(t, "TestOpenTransactionDb", applyOpts) 102 | defer db.Close() 103 | 104 | var ( 105 | givenKey = []byte("hello") 106 | givenVal = []byte("world") 107 | wo = NewDefaultWriteOptions() 108 | ro = NewDefaultReadOptions() 109 | to = NewDefaultTransactionOptions() 110 | ) 111 | 112 | txn := db.TransactionBegin(wo, to, nil) 113 | defer txn.Destroy() 114 | 115 | v, err := txn.GetForUpdate(ro, givenKey) 116 | defer v.Free() 117 | ensure.Nil(t, err) 118 | 119 | // expect lock timeout error to be thrown 120 | if err := db.Put(wo, givenKey, givenVal); err == nil { 121 | t.Error("expect locktime out error, got nil error") 122 | } 123 | } 124 | 125 | func newTestTransactionDB(t *testing.T, name string, applyOpts func(opts *Options, transactionDBOpts *TransactionDBOptions)) *TransactionDB { 126 | dir, err := ioutil.TempDir("", "gorockstransactiondb-"+name) 127 | ensure.Nil(t, err) 128 | 129 | opts := NewDefaultOptions() 130 | opts.SetCreateIfMissing(true) 131 | transactionDBOpts := NewDefaultTransactionDBOptions() 132 | if applyOpts != nil { 133 | applyOpts(opts, transactionDBOpts) 134 | } 135 | db, err := OpenTransactionDb(opts, transactionDBOpts, dir) 136 | ensure.Nil(t, err) 137 | 138 | return db 139 | } 140 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | 5 | import "C" 6 | import ( 7 | "reflect" 8 | "unsafe" 9 | ) 10 | 11 | // btoi converts a bool value to int. 12 | func btoi(b bool) int { 13 | if b { 14 | return 1 15 | } 16 | return 0 17 | } 18 | 19 | // boolToChar converts a bool value to C.uchar. 20 | func boolToChar(b bool) C.uchar { 21 | if b { 22 | return 1 23 | } 24 | return 0 25 | } 26 | 27 | // charToByte converts a *C.char to a byte slice. 28 | func charToByte(data *C.char, len C.size_t) []byte { 29 | var value []byte 30 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) 31 | sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) 32 | return value 33 | } 34 | 35 | // byteToChar returns *C.char from byte slice. 36 | func byteToChar(b []byte) *C.char { 37 | var c *C.char 38 | if len(b) > 0 { 39 | c = (*C.char)(unsafe.Pointer(&b[0])) 40 | } 41 | return c 42 | } 43 | 44 | // Go []byte to C string 45 | // The C string is allocated in the C heap using malloc. 46 | func cByteSlice(b []byte) *C.char { 47 | var c *C.char 48 | if len(b) > 0 { 49 | c = (*C.char)(C.CBytes(b)) 50 | } 51 | return c 52 | } 53 | 54 | // stringToChar returns *C.char from string. 55 | func stringToChar(s string) *C.char { 56 | ptrStr := (*reflect.StringHeader)(unsafe.Pointer(&s)) 57 | return (*C.char)(unsafe.Pointer(ptrStr.Data)) 58 | } 59 | 60 | // charSlice converts a C array of *char to a []*C.char. 61 | func charSlice(data **C.char, len C.int) []*C.char { 62 | var value []*C.char 63 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) 64 | sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) 65 | return value 66 | } 67 | 68 | // sizeSlice converts a C array of size_t to a []C.size_t. 69 | func sizeSlice(data *C.size_t, len C.int) []C.size_t { 70 | var value []C.size_t 71 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) 72 | sH.Cap, sH.Len, sH.Data = int(len), int(len), uintptr(unsafe.Pointer(data)) 73 | return value 74 | } 75 | -------------------------------------------------------------------------------- /wal_iterator.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "errors" 8 | "unsafe" 9 | ) 10 | 11 | type WalIterator struct { 12 | c *C.rocksdb_wal_iterator_t 13 | } 14 | 15 | func NewNativeWalIterator(c unsafe.Pointer) *WalIterator { 16 | return &WalIterator{(*C.rocksdb_wal_iterator_t)(c)} 17 | } 18 | 19 | func (iter *WalIterator) Valid() bool { 20 | return C.rocksdb_wal_iter_valid(iter.c) != 0 21 | } 22 | 23 | func (iter *WalIterator) Next() { 24 | C.rocksdb_wal_iter_next(iter.c) 25 | } 26 | 27 | func (iter *WalIterator) Err() error { 28 | var cErr *C.char 29 | C.rocksdb_wal_iter_status(iter.c, &cErr) 30 | if cErr != nil { 31 | defer C.rocksdb_free(unsafe.Pointer(cErr)) 32 | return errors.New(C.GoString(cErr)) 33 | } 34 | return nil 35 | } 36 | 37 | func (iter *WalIterator) Destroy() { 38 | C.rocksdb_wal_iter_destroy(iter.c) 39 | iter.c = nil 40 | } 41 | 42 | // C.rocksdb_wal_iter_get_batch in the official rocksdb c wrapper has memory leak 43 | // see https://github.com/facebook/rocksdb/pull/5515 44 | // https://github.com/facebook/rocksdb/issues/5536 45 | func (iter *WalIterator) GetBatch() (*WriteBatch, uint64) { 46 | var cSeq C.uint64_t 47 | cB := C.rocksdb_wal_iter_get_batch(iter.c, &cSeq) 48 | return NewNativeWriteBatch(cB), uint64(cSeq) 49 | } 50 | -------------------------------------------------------------------------------- /write_batch.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import ( 6 | "errors" 7 | "io" 8 | ) 9 | 10 | // WriteBatch is a batching of Puts, Merges and Deletes. 11 | type WriteBatch struct { 12 | c *C.rocksdb_writebatch_t 13 | } 14 | 15 | // NewWriteBatch create a WriteBatch object. 16 | func NewWriteBatch() *WriteBatch { 17 | return NewNativeWriteBatch(C.rocksdb_writebatch_create()) 18 | } 19 | 20 | // NewNativeWriteBatch create a WriteBatch object. 21 | func NewNativeWriteBatch(c *C.rocksdb_writebatch_t) *WriteBatch { 22 | return &WriteBatch{c} 23 | } 24 | 25 | // WriteBatchFrom creates a write batch from a serialized WriteBatch. 26 | func WriteBatchFrom(data []byte) *WriteBatch { 27 | return NewNativeWriteBatch(C.rocksdb_writebatch_create_from(byteToChar(data), C.size_t(len(data)))) 28 | } 29 | 30 | // Put queues a key-value pair. 31 | func (wb *WriteBatch) Put(key, value []byte) { 32 | cKey := byteToChar(key) 33 | cValue := byteToChar(value) 34 | C.rocksdb_writebatch_put(wb.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) 35 | } 36 | 37 | // PutCF queues a key-value pair in a column family. 38 | func (wb *WriteBatch) PutCF(cf *ColumnFamilyHandle, key, value []byte) { 39 | cKey := byteToChar(key) 40 | cValue := byteToChar(value) 41 | C.rocksdb_writebatch_put_cf(wb.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) 42 | } 43 | 44 | // Append a blob of arbitrary size to the records in this batch. 45 | func (wb *WriteBatch) PutLogData(blob []byte) { 46 | cBlob := byteToChar(blob) 47 | C.rocksdb_writebatch_put_log_data(wb.c, cBlob, C.size_t(len(blob))) 48 | } 49 | 50 | // Merge queues a merge of "value" with the existing value of "key". 51 | func (wb *WriteBatch) Merge(key, value []byte) { 52 | cKey := byteToChar(key) 53 | cValue := byteToChar(value) 54 | C.rocksdb_writebatch_merge(wb.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) 55 | } 56 | 57 | // MergeCF queues a merge of "value" with the existing value of "key" in a 58 | // column family. 59 | func (wb *WriteBatch) MergeCF(cf *ColumnFamilyHandle, key, value []byte) { 60 | cKey := byteToChar(key) 61 | cValue := byteToChar(value) 62 | C.rocksdb_writebatch_merge_cf(wb.c, cf.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value))) 63 | } 64 | 65 | // Delete queues a deletion of the data at key. 66 | func (wb *WriteBatch) Delete(key []byte) { 67 | cKey := byteToChar(key) 68 | C.rocksdb_writebatch_delete(wb.c, cKey, C.size_t(len(key))) 69 | } 70 | 71 | // DeleteCF queues a deletion of the data at key in a column family. 72 | func (wb *WriteBatch) DeleteCF(cf *ColumnFamilyHandle, key []byte) { 73 | cKey := byteToChar(key) 74 | C.rocksdb_writebatch_delete_cf(wb.c, cf.c, cKey, C.size_t(len(key))) 75 | } 76 | 77 | // DeleteRange deletes keys that are between [startKey, endKey) 78 | func (wb *WriteBatch) DeleteRange(startKey []byte, endKey []byte) { 79 | cStartKey := byteToChar(startKey) 80 | cEndKey := byteToChar(endKey) 81 | C.rocksdb_writebatch_delete_range(wb.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey))) 82 | } 83 | 84 | // DeleteRangeCF deletes keys that are between [startKey, endKey) and 85 | // belong to a given column family 86 | func (wb *WriteBatch) DeleteRangeCF(cf *ColumnFamilyHandle, startKey []byte, endKey []byte) { 87 | cStartKey := byteToChar(startKey) 88 | cEndKey := byteToChar(endKey) 89 | C.rocksdb_writebatch_delete_range_cf(wb.c, cf.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey))) 90 | } 91 | 92 | // Data returns the serialized version of this batch. 93 | func (wb *WriteBatch) Data() []byte { 94 | var cSize C.size_t 95 | cValue := C.rocksdb_writebatch_data(wb.c, &cSize) 96 | return charToByte(cValue, cSize) 97 | } 98 | 99 | // Count returns the number of updates in the batch. 100 | func (wb *WriteBatch) Count() int { 101 | return int(C.rocksdb_writebatch_count(wb.c)) 102 | } 103 | 104 | // NewIterator returns a iterator to iterate over the records in the batch. 105 | func (wb *WriteBatch) NewIterator() *WriteBatchIterator { 106 | data := wb.Data() 107 | if len(data) < 8+4 { 108 | return &WriteBatchIterator{} 109 | } 110 | return &WriteBatchIterator{data: data[12:]} 111 | } 112 | 113 | // Clear removes all the enqueued Put and Deletes. 114 | func (wb *WriteBatch) Clear() { 115 | C.rocksdb_writebatch_clear(wb.c) 116 | } 117 | 118 | // Destroy deallocates the WriteBatch object. 119 | func (wb *WriteBatch) Destroy() { 120 | C.rocksdb_writebatch_destroy(wb.c) 121 | wb.c = nil 122 | } 123 | 124 | // WriteBatchRecordType describes the type of a batch record. 125 | type WriteBatchRecordType byte 126 | 127 | // Types of batch records. 128 | const ( 129 | WriteBatchDeletionRecord WriteBatchRecordType = 0x0 130 | WriteBatchValueRecord WriteBatchRecordType = 0x1 131 | WriteBatchMergeRecord WriteBatchRecordType = 0x2 132 | WriteBatchLogDataRecord WriteBatchRecordType = 0x3 133 | WriteBatchCFDeletionRecord WriteBatchRecordType = 0x4 134 | WriteBatchCFValueRecord WriteBatchRecordType = 0x5 135 | WriteBatchCFMergeRecord WriteBatchRecordType = 0x6 136 | WriteBatchSingleDeletionRecord WriteBatchRecordType = 0x7 137 | WriteBatchCFSingleDeletionRecord WriteBatchRecordType = 0x8 138 | WriteBatchBeginPrepareXIDRecord WriteBatchRecordType = 0x9 139 | WriteBatchEndPrepareXIDRecord WriteBatchRecordType = 0xA 140 | WriteBatchCommitXIDRecord WriteBatchRecordType = 0xB 141 | WriteBatchRollbackXIDRecord WriteBatchRecordType = 0xC 142 | WriteBatchNoopRecord WriteBatchRecordType = 0xD 143 | WriteBatchRangeDeletion WriteBatchRecordType = 0xF 144 | WriteBatchCFRangeDeletion WriteBatchRecordType = 0xE 145 | WriteBatchCFBlobIndex WriteBatchRecordType = 0x10 146 | WriteBatchBlobIndex WriteBatchRecordType = 0x11 147 | WriteBatchBeginPersistedPrepareXIDRecord WriteBatchRecordType = 0x12 148 | WriteBatchNotUsedRecord WriteBatchRecordType = 0x7F 149 | ) 150 | 151 | // WriteBatchRecord represents a record inside a WriteBatch. 152 | type WriteBatchRecord struct { 153 | CF int 154 | Key []byte 155 | Value []byte 156 | Type WriteBatchRecordType 157 | } 158 | 159 | // WriteBatchIterator represents a iterator to iterator over records. 160 | type WriteBatchIterator struct { 161 | data []byte 162 | record WriteBatchRecord 163 | err error 164 | } 165 | 166 | // Next returns the next record. 167 | // Returns false if no further record exists. 168 | func (iter *WriteBatchIterator) Next() bool { 169 | if iter.err != nil || len(iter.data) == 0 { 170 | return false 171 | } 172 | // reset the current record 173 | iter.record.CF = 0 174 | iter.record.Key = nil 175 | iter.record.Value = nil 176 | 177 | // parse the record type 178 | iter.record.Type = iter.decodeRecType() 179 | 180 | switch iter.record.Type { 181 | case 182 | WriteBatchDeletionRecord, 183 | WriteBatchSingleDeletionRecord: 184 | iter.record.Key = iter.decodeSlice() 185 | case 186 | WriteBatchCFDeletionRecord, 187 | WriteBatchCFSingleDeletionRecord: 188 | iter.record.CF = int(iter.decodeVarint()) 189 | if iter.err == nil { 190 | iter.record.Key = iter.decodeSlice() 191 | } 192 | case 193 | WriteBatchValueRecord, 194 | WriteBatchMergeRecord, 195 | WriteBatchRangeDeletion, 196 | WriteBatchBlobIndex: 197 | iter.record.Key = iter.decodeSlice() 198 | if iter.err == nil { 199 | iter.record.Value = iter.decodeSlice() 200 | } 201 | case 202 | WriteBatchCFValueRecord, 203 | WriteBatchCFRangeDeletion, 204 | WriteBatchCFMergeRecord, 205 | WriteBatchCFBlobIndex: 206 | iter.record.CF = int(iter.decodeVarint()) 207 | if iter.err == nil { 208 | iter.record.Key = iter.decodeSlice() 209 | } 210 | if iter.err == nil { 211 | iter.record.Value = iter.decodeSlice() 212 | } 213 | case WriteBatchLogDataRecord: 214 | iter.record.Value = iter.decodeSlice() 215 | case 216 | WriteBatchNoopRecord, 217 | WriteBatchBeginPrepareXIDRecord, 218 | WriteBatchBeginPersistedPrepareXIDRecord: 219 | case 220 | WriteBatchEndPrepareXIDRecord, 221 | WriteBatchCommitXIDRecord, 222 | WriteBatchRollbackXIDRecord: 223 | iter.record.Value = iter.decodeSlice() 224 | default: 225 | iter.err = errors.New("unsupported wal record type") 226 | } 227 | 228 | return iter.err == nil 229 | 230 | } 231 | 232 | // Record returns the current record. 233 | func (iter *WriteBatchIterator) Record() *WriteBatchRecord { 234 | return &iter.record 235 | } 236 | 237 | // Error returns the error if the iteration is failed. 238 | func (iter *WriteBatchIterator) Error() error { 239 | return iter.err 240 | } 241 | 242 | func (iter *WriteBatchIterator) decodeSlice() []byte { 243 | l := int(iter.decodeVarint()) 244 | if l > len(iter.data) { 245 | iter.err = io.ErrShortBuffer 246 | } 247 | if iter.err != nil { 248 | return []byte{} 249 | } 250 | ret := iter.data[:l] 251 | iter.data = iter.data[l:] 252 | return ret 253 | } 254 | 255 | func (iter *WriteBatchIterator) decodeRecType() WriteBatchRecordType { 256 | if len(iter.data) == 0 { 257 | iter.err = io.ErrShortBuffer 258 | return WriteBatchNotUsedRecord 259 | } 260 | t := iter.data[0] 261 | iter.data = iter.data[1:] 262 | return WriteBatchRecordType(t) 263 | } 264 | 265 | func (iter *WriteBatchIterator) decodeVarint() uint64 { 266 | var n int 267 | var x uint64 268 | for shift := uint(0); shift < 64 && n < len(iter.data); shift += 7 { 269 | b := uint64(iter.data[n]) 270 | n++ 271 | x |= (b & 0x7F) << shift 272 | if (b & 0x80) == 0 { 273 | iter.data = iter.data[n:] 274 | return x 275 | } 276 | } 277 | if n == len(iter.data) { 278 | iter.err = io.ErrShortBuffer 279 | } else { 280 | iter.err = errors.New("malformed varint") 281 | } 282 | return 0 283 | } 284 | -------------------------------------------------------------------------------- /write_batch_test.go: -------------------------------------------------------------------------------- 1 | package gorocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/facebookgo/ensure" 7 | ) 8 | 9 | func TestWriteBatch(t *testing.T) { 10 | db := newTestDB(t, "TestWriteBatch", nil) 11 | defer db.Close() 12 | 13 | var ( 14 | givenKey1 = []byte("key1") 15 | givenVal1 = []byte("val1") 16 | givenKey2 = []byte("key2") 17 | ) 18 | wo := NewDefaultWriteOptions() 19 | ensure.Nil(t, db.Put(wo, givenKey2, []byte("foo"))) 20 | 21 | // create and fill the write batch 22 | wb := NewWriteBatch() 23 | defer wb.Destroy() 24 | wb.Put(givenKey1, givenVal1) 25 | wb.Delete(givenKey2) 26 | ensure.DeepEqual(t, wb.Count(), 2) 27 | 28 | // perform the batch 29 | ensure.Nil(t, db.Write(wo, wb)) 30 | 31 | // check changes 32 | ro := NewDefaultReadOptions() 33 | v1, err := db.Get(ro, givenKey1) 34 | defer v1.Free() 35 | ensure.Nil(t, err) 36 | ensure.DeepEqual(t, v1.Data(), givenVal1) 37 | 38 | v2, err := db.Get(ro, givenKey2) 39 | defer v2.Free() 40 | ensure.Nil(t, err) 41 | ensure.True(t, v2.Data() == nil) 42 | 43 | // DeleteRange test 44 | wb.Clear() 45 | wb.DeleteRange(givenKey1, givenKey2) 46 | 47 | // perform the batch 48 | ensure.Nil(t, db.Write(wo, wb)) 49 | 50 | v1, err = db.Get(ro, givenKey1) 51 | defer v1.Free() 52 | ensure.Nil(t, err) 53 | ensure.True(t, v1.Data() == nil) 54 | } 55 | 56 | func TestWriteBatchIterator(t *testing.T) { 57 | db := newTestDB(t, "TestWriteBatchIterator", nil) 58 | defer db.Close() 59 | 60 | var ( 61 | givenKey1 = []byte("key1") 62 | givenVal1 = []byte("val1") 63 | givenKey2 = []byte("key2") 64 | ) 65 | // create and fill the write batch 66 | wb := NewWriteBatch() 67 | defer wb.Destroy() 68 | wb.Put(givenKey1, givenVal1) 69 | wb.Delete(givenKey2) 70 | ensure.DeepEqual(t, wb.Count(), 2) 71 | 72 | // iterate over the batch 73 | iter := wb.NewIterator() 74 | ensure.True(t, iter.Next()) 75 | record := iter.Record() 76 | ensure.DeepEqual(t, record.Type, WriteBatchValueRecord) 77 | ensure.DeepEqual(t, record.Key, givenKey1) 78 | ensure.DeepEqual(t, record.Value, givenVal1) 79 | 80 | ensure.True(t, iter.Next()) 81 | record = iter.Record() 82 | ensure.DeepEqual(t, record.Type, WriteBatchDeletionRecord) 83 | ensure.DeepEqual(t, record.Key, givenKey2) 84 | 85 | // there shouldn't be any left 86 | ensure.False(t, iter.Next()) 87 | } 88 | --------------------------------------------------------------------------------