├── .github ├── dependabot.yml └── workflows │ └── go.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── array.go ├── array_test.go ├── backup.go ├── backup_test.go ├── build.sh ├── c.h ├── cache.go ├── cache_test.go ├── cf_handle.go ├── cf_metadata.go ├── cf_test.go ├── cf_ts_test.go ├── checkpoint.go ├── checkpoint_test.go ├── compaction_filter.go ├── compaction_filter_test.go ├── comparator.go ├── comparator_test.go ├── comparator_ts_test.go ├── cow.go ├── cow_test.go ├── cuckoo_table.go ├── db.go ├── db_external_file_test.go ├── db_test.go ├── db_ts_test.go ├── dbpath.go ├── doc.go ├── env.go ├── env_test.go ├── filter_policy.go ├── filter_policy_test.go ├── go.mod ├── go.sum ├── grocksdb.c ├── grocksdb.h ├── iterator.go ├── iterator_test.go ├── iterator_ts_test.go ├── jemalloc.go ├── jemalloc_test.go ├── logger.go ├── mem_alloc.go ├── memory_usage.go ├── memory_usage_test.go ├── merge_operator.go ├── merge_operator_test.go ├── non_builtin.go ├── non_builtin_clean_link.go ├── optimistic_transaction_db.go ├── options.go ├── options_backup.go ├── options_backup_test.go ├── options_blob_test.go ├── options_block_based_table.go ├── options_block_based_table_test.go ├── options_compaction.go ├── options_compaction_test.go ├── options_compression.go ├── options_env.go ├── options_env_test.go ├── options_flush.go ├── options_flush_test.go ├── options_ingest.go ├── options_read.go ├── options_read_test.go ├── options_restore.go ├── options_restore_test.go ├── options_test.go ├── options_transaction.go ├── options_transactiondb.go ├── options_wait_for_compact.go ├── options_wait_for_compact_test.go ├── options_write.go ├── options_write_test.go ├── perf_context.go ├── perf_level.go ├── ratelimiter.go ├── slice.go ├── slice_transform.go ├── slice_transform_test.go ├── snapshot.go ├── sst_file_writer.go ├── stats.go ├── testing_darwin_arm64.go ├── testing_linux_amd64.go ├── testing_linux_arm64.go ├── transaction.go ├── transactiondb.go ├── transactiondb_test.go ├── util.go ├── wal_iterator.go ├── write_batch.go ├── write_batch_test.go ├── write_batch_ts_test.go ├── write_batch_wi.go ├── write_batch_wi_test.go ├── write_buffer_manager.go └── write_buffer_manager_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | name: CI 12 | 13 | runs-on: ${{ matrix.os }} 14 | 15 | strategy: 16 | matrix: 17 | os: [ubuntu-latest] 18 | 19 | steps: 20 | - name: Set up Go 21 | uses: actions/setup-go@v4 22 | with: 23 | go-version: ^1.17 24 | id: go 25 | 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@v3 28 | 29 | - name: Cache prebuilt static libs 30 | uses: actions/cache@v3 31 | id: cache-prebuilt-static-libs 32 | continue-on-error: false 33 | with: 34 | path: | 35 | dist/ 36 | key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/build.sh') }} 37 | restore-keys: ${{ runner.os }}-${{ runner.arch }}- 38 | 39 | - name: Build static libs 40 | if: steps.cache-prebuilt-static-libs.outputs.cache-hit != 'true' 41 | run: make libs 42 | 43 | - name: Get dependencies 44 | run: | 45 | go mod download 46 | 47 | - name: Test Coverage 48 | run: go test -v -tags testing -count=1 -coverprofile=coverage.out 49 | 50 | - name: Convert coverage to lcov 51 | uses: jandelgado/gcov2lcov-action@v1.0.9 52 | with: 53 | infile: coverage.out 54 | outfile: coverage.lcov 55 | 56 | - name: Coveralls 57 | uses: coverallsapp/github-action@master 58 | with: 59 | github-token: ${{ secrets.github_token }} 60 | path-to-lcov: coverage.lcov 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | *.o 14 | *.DS_Store* 15 | libs/bzip2/bzip2 16 | libs/bzip2/bzip2.a 17 | libs/bzip2/libbz2.a 18 | build/* 19 | .vscode/c_cpp_properties.json 20 | dist 21 | .vscode/settings.json 22 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linxGnu/grocksdb/5af746e8a90a800a9affa890ba6c4ae3a7020932/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2016 Thomas Adam 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOOS ?= $(shell go env GOOS) 2 | GOARCH ?= $(shell go env GOARCH) 3 | GOOS_GOARCH := $(GOOS)_$(GOARCH) 4 | GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH) 5 | 6 | ROOT_DIR=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 7 | DEST=$(ROOT_DIR)/dist/$(GOOS_GOARCH) 8 | DEST_LIB=$(DEST)/lib 9 | DEST_INCLUDE=$(DEST)/include 10 | 11 | default: prepare libs 12 | 13 | .PHONY: prepare 14 | prepare: 15 | rm -rf $(DEST) 16 | mkdir -p $(DEST_LIB) $(DEST_INCLUDE) 17 | 18 | .PHONY: libs 19 | libs: 20 | ./build.sh $(DEST) 21 | 22 | .PHONY: test 23 | test: 24 | go test -race -v -count=1 -tags testing,grocksdb_no_link 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # grocksdb, RocksDB wrapper for Go 2 | 3 | [![](https://github.com/linxGnu/grocksdb/workflows/CI/badge.svg)]() 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/linxGnu/grocksdb)](https://goreportcard.com/report/github.com/linxGnu/grocksdb) 5 | [![Coverage Status](https://coveralls.io/repos/github/linxGnu/grocksdb/badge.svg?branch=master)](https://coveralls.io/github/linxGnu/grocksdb?branch=master) 6 | [![godoc](https://img.shields.io/badge/docs-GoDoc-green.svg)](https://godoc.org/github.com/linxGnu/grocksdb) 7 | 8 | This is a `Fork` from [tecbot/gorocksdb](https://github.com/tecbot/gorocksdb). I respect the author work and community contribution. 9 | The `LICENSE` still remains as upstream. 10 | 11 | Why I made a patched clone instead of PR: 12 | - Supports almost C API (unlike upstream). Catching up with latest version of Rocksdb as promise. 13 | - This fork contains `no defer` in codebase (my side project requires as less overhead as possible). This introduces loose 14 | convention of how/when to free c-mem, thus break the rule of [tecbot/gorocksdb](https://github.com/tecbot/gorocksdb). 15 | 16 | ## Install 17 | 18 | ### Prerequisite 19 | 20 | - librocksdb 21 | - libsnappy 22 | - libz 23 | - liblz4 24 | - libzstd 25 | - libbz2 (optional) 26 | 27 | Please follow this guide: https://github.com/facebook/rocksdb/blob/master/INSTALL.md to build above libs. 28 | 29 | ### Build 30 | 31 | After installing both `rocksdb` and `grocksdb`, you can build your app using the following commands: 32 | 33 | ```bash 34 | CGO_CFLAGS="-I/path/to/rocksdb/include" \ 35 | CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd" \ 36 | go build 37 | ``` 38 | 39 | Or just: 40 | ```bash 41 | go build // if prerequisites are in linker paths 42 | ``` 43 | 44 | If your rocksdb was linked with bz2: 45 | ```bash 46 | CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lm -lz -lsnappy -llz4 -lzstd -lbz2" \ 47 | go build 48 | ``` 49 | 50 | #### Customize the build flags 51 | Currently, the default build flags without specifying `CGO_LDFLAGS` or the corresponding environment variables are `-lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy` 52 | 53 | If you want to customize the build flags: 54 | 55 | 1. Use `-tags grocksdb_clean_link` to create a cleaner set of flags and build it based on the cleaner flag. The base build flags after using the tag are `-lrocksdb -pthread -lstdc++ -ldl`. 56 | ```bash 57 | CGO_LDFLAGS="-L/path/to/rocksdb -lzstd" go build -tags grocksdb_clean_link 58 | ``` 59 | 2. Use `-tags grocksdb_no_link` to ignore the build flags provided by the library and build it fully based on the custom flags. 60 | ```bash 61 | CGO_LDFLAGS="-L/path/to/rocksdb -lrocksdb -lstdc++ -lzstd -llz4" go build -tags grocksdb_clean_link 62 | ``` 63 | 64 | ## Usage 65 | 66 | See also: [doc](https://godoc.org/github.com/linxGnu/grocksdb) 67 | 68 | ## API Support 69 | 70 | Almost C API, excepts: 71 | - [ ] get_db_identity 72 | - [ ] putv/mergev/deletev/delete_rangev 73 | - [ ] compaction_filter/compaction_filter_factory/compaction_filter_context 74 | - [ ] transactiondb_property_value/transactiondb_property_int 75 | - [ ] optimistictransactiondb_property_value/optimistictransactiondb_property_int 76 | - [ ] writebatch_update_timestamps/writebatch_wi_update_timestamps/writebatch_iterate_cf 77 | - [ ] approximate_sizes_cf_with_flags 78 | - [ ] logger_create_callback_logger 79 | 80 | -------------------------------------------------------------------------------- /array.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "stdlib.h" 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "reflect" 9 | "unsafe" 10 | ) 11 | 12 | type ( 13 | charsSlice []*C.char 14 | sizeTSlice []C.size_t 15 | columnFamilySlice []*C.rocksdb_column_family_handle_t 16 | pinnableSliceSlice []*C.rocksdb_pinnableslice_t 17 | ) 18 | 19 | func (s charsSlice) c() **C.char { 20 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 21 | return (**C.char)(unsafe.Pointer(sH.Data)) 22 | } 23 | 24 | func (s sizeTSlice) c() *C.size_t { 25 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 26 | return (*C.size_t)(unsafe.Pointer(sH.Data)) 27 | } 28 | 29 | func (s columnFamilySlice) c() **C.rocksdb_column_family_handle_t { 30 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 31 | return (**C.rocksdb_column_family_handle_t)(unsafe.Pointer(sH.Data)) 32 | } 33 | 34 | func (s pinnableSliceSlice) c() **C.rocksdb_pinnableslice_t { 35 | sH := (*reflect.SliceHeader)(unsafe.Pointer(&s)) 36 | return (**C.rocksdb_pinnableslice_t)(unsafe.Pointer(sH.Data)) 37 | } 38 | 39 | // bytesSliceToCSlices converts a slice of byte slices to two slices with C 40 | // datatypes. One containing pointers to copies of the byte slices and one 41 | // containing their sizes. 42 | // IMPORTANT: All the contents of the charsSlice array are malloced and 43 | // should be freed using the Destroy method of charsSlice. 44 | func byteSlicesToCSlices(vals [][]byte) (charsSlice, sizeTSlice) { 45 | if len(vals) == 0 { 46 | return nil, nil 47 | } 48 | 49 | chars := make(charsSlice, len(vals)) 50 | sizes := make(sizeTSlice, len(vals)) 51 | for i, val := range vals { 52 | chars[i] = (*C.char)(C.CBytes(val)) 53 | sizes[i] = C.size_t(len(val)) 54 | } 55 | 56 | return chars, sizes 57 | } 58 | 59 | func (s charsSlice) Destroy() { 60 | for _, chars := range s { 61 | C.free(unsafe.Pointer(chars)) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /array_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestBytesToCSlice(t *testing.T) { 10 | t.Parallel() 11 | 12 | v, err := byteSlicesToCSlices(nil) 13 | require.Nil(t, v) 14 | require.Nil(t, err) 15 | } 16 | -------------------------------------------------------------------------------- /backup.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "unsafe" 8 | ) 9 | 10 | // BackupInfo represents the information about a backup. 11 | type BackupInfo struct { 12 | ID uint32 13 | Timestamp int64 14 | Size uint64 15 | NumFiles uint32 16 | } 17 | 18 | // BackupEngine is a reusable handle to a RocksDB Backup, created by 19 | // OpenBackupEngine. 20 | type BackupEngine struct { 21 | c *C.rocksdb_backup_engine_t 22 | db *DB 23 | } 24 | 25 | // OpenBackupEngine opens a backup engine with specified options. 26 | func OpenBackupEngine(opts *Options, path string) (be *BackupEngine, err error) { 27 | cpath := C.CString(path) 28 | 29 | var cErr *C.char 30 | bEngine := C.rocksdb_backup_engine_open(opts.c, cpath, &cErr) 31 | if err = fromCError(cErr); err == nil { 32 | be = &BackupEngine{ 33 | c: bEngine, 34 | } 35 | } 36 | 37 | C.free(unsafe.Pointer(cpath)) 38 | return 39 | } 40 | 41 | // OpenBackupEngineWithOpt opens a backup engine with specified options. 42 | func OpenBackupEngineWithOpt(opts *BackupEngineOptions, env *Env) (be *BackupEngine, err error) { 43 | var cErr *C.char 44 | bEngine := C.rocksdb_backup_engine_open_opts(opts.c, env.c, &cErr) 45 | if err = fromCError(cErr); err == nil { 46 | be = &BackupEngine{ 47 | c: bEngine, 48 | } 49 | } 50 | 51 | return 52 | } 53 | 54 | // CreateBackupEngine opens a backup engine from DB. 55 | func CreateBackupEngine(db *DB) (be *BackupEngine, err error) { 56 | if be, err = OpenBackupEngine(db.opts, db.Name()); err == nil { 57 | be.db = db 58 | } 59 | return 60 | } 61 | 62 | // CreateBackupEngineWithPath opens a backup engine from DB and path 63 | func CreateBackupEngineWithPath(db *DB, path string) (be *BackupEngine, err error) { 64 | if be, err = OpenBackupEngine(db.opts, path); err == nil { 65 | be.db = db 66 | } 67 | return 68 | } 69 | 70 | // CreateNewBackup takes a new backup from db. 71 | func (b *BackupEngine) CreateNewBackup() (err error) { 72 | var cErr *C.char 73 | C.rocksdb_backup_engine_create_new_backup(b.c, b.db.c, &cErr) 74 | err = fromCError(cErr) 75 | return 76 | } 77 | 78 | // CreateNewBackupFlush takes a new backup from db. 79 | // Backup would be created after flushing. 80 | func (b *BackupEngine) CreateNewBackupFlush(flushBeforeBackup bool) (err error) { 81 | var cErr *C.char 82 | C.rocksdb_backup_engine_create_new_backup_flush(b.c, b.db.c, boolToChar(flushBeforeBackup), &cErr) 83 | err = fromCError(cErr) 84 | return 85 | } 86 | 87 | // PurgeOldBackups deletes old backups, where `numBackupsToKeep` is how many backups you’d like to keep. 88 | func (b *BackupEngine) PurgeOldBackups(numBackupsToKeep uint32) (err error) { 89 | var cErr *C.char 90 | C.rocksdb_backup_engine_purge_old_backups(b.c, C.uint32_t(numBackupsToKeep), &cErr) 91 | err = fromCError(cErr) 92 | return 93 | } 94 | 95 | // VerifyBackup verifies a backup by its id. 96 | func (b *BackupEngine) VerifyBackup(backupID uint32) (err error) { 97 | var cErr *C.char 98 | C.rocksdb_backup_engine_verify_backup(b.c, C.uint32_t(backupID), &cErr) 99 | err = fromCError(cErr) 100 | return 101 | } 102 | 103 | // GetInfo gets an object that gives information about 104 | // the backups that have already been taken 105 | func (b *BackupEngine) GetInfo() (infos []BackupInfo) { 106 | info := C.rocksdb_backup_engine_get_backup_info(b.c) 107 | 108 | n := int(C.rocksdb_backup_engine_info_count(info)) 109 | infos = make([]BackupInfo, n) 110 | for i := range infos { 111 | index := C.int(i) 112 | infos[i].ID = uint32(C.rocksdb_backup_engine_info_backup_id(info, index)) 113 | infos[i].Timestamp = int64(C.rocksdb_backup_engine_info_timestamp(info, index)) 114 | infos[i].Size = uint64(C.rocksdb_backup_engine_info_size(info, index)) 115 | infos[i].NumFiles = uint32(C.rocksdb_backup_engine_info_number_files(info, index)) 116 | } 117 | 118 | C.rocksdb_backup_engine_info_destroy(info) 119 | return 120 | } 121 | 122 | // RestoreDBFromLatestBackup restores the latest backup to dbDir. walDir 123 | // is where the write ahead logs are restored to and usually the same as dbDir. 124 | func (b *BackupEngine) RestoreDBFromLatestBackup(dbDir, walDir string, ro *RestoreOptions) (err error) { 125 | cDbDir := C.CString(dbDir) 126 | cWalDir := C.CString(walDir) 127 | 128 | var cErr *C.char 129 | C.rocksdb_backup_engine_restore_db_from_latest_backup(b.c, cDbDir, cWalDir, ro.c, &cErr) 130 | err = fromCError(cErr) 131 | 132 | C.free(unsafe.Pointer(cDbDir)) 133 | C.free(unsafe.Pointer(cWalDir)) 134 | return 135 | } 136 | 137 | // RestoreDBFromBackup restores the backup (identified by its id) to dbDir. walDir 138 | // is where the write ahead logs are restored to and usually the same as dbDir. 139 | func (b *BackupEngine) RestoreDBFromBackup(dbDir, walDir string, ro *RestoreOptions, backupID uint32) (err error) { 140 | cDbDir := C.CString(dbDir) 141 | cWalDir := C.CString(walDir) 142 | 143 | var cErr *C.char 144 | C.rocksdb_backup_engine_restore_db_from_backup(b.c, cDbDir, cWalDir, ro.c, C.uint32_t(backupID), &cErr) 145 | err = fromCError(cErr) 146 | 147 | C.free(unsafe.Pointer(cDbDir)) 148 | C.free(unsafe.Pointer(cWalDir)) 149 | return 150 | } 151 | 152 | // Close close the backup engine and cleans up state 153 | // The backups already taken remain on storage. 154 | func (b *BackupEngine) Close() { 155 | C.rocksdb_backup_engine_close(b.c) 156 | b.c = nil 157 | b.db = nil 158 | } 159 | -------------------------------------------------------------------------------- /backup_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestBackupEngine(t *testing.T) { 10 | t.Parallel() 11 | 12 | db := newTestDB(t, nil) 13 | defer db.Close() 14 | 15 | var ( 16 | givenKey = []byte("hello") 17 | givenVal1 = []byte("") 18 | givenVal2 = []byte("world1") 19 | wo = NewDefaultWriteOptions() 20 | ro = NewDefaultReadOptions() 21 | ) 22 | defer wo.Destroy() 23 | defer ro.Destroy() 24 | 25 | // create 26 | require.Nil(t, db.Put(wo, givenKey, givenVal1)) 27 | 28 | // retrieve 29 | v1, err := db.Get(ro, givenKey) 30 | require.Nil(t, err) 31 | require.EqualValues(t, v1.Data(), givenVal1) 32 | v1.Free() 33 | 34 | // retrieve bytes 35 | _v1, err := db.GetBytes(ro, givenKey) 36 | require.Nil(t, err) 37 | require.EqualValues(t, _v1, givenVal1) 38 | 39 | // update 40 | require.Nil(t, db.Put(wo, givenKey, givenVal2)) 41 | v2, err := db.Get(ro, givenKey) 42 | require.Nil(t, err) 43 | require.EqualValues(t, v2.Data(), givenVal2) 44 | v2.Free() 45 | 46 | // retrieve pinned 47 | v3, err := db.GetPinned(ro, givenKey) 48 | require.Nil(t, err) 49 | require.EqualValues(t, v3.Data(), givenVal2) 50 | v3.Destroy() 51 | 52 | engine, err := CreateBackupEngine(db) 53 | require.Nil(t, err) 54 | defer func() { 55 | engine.Close() 56 | 57 | // re-open with opts 58 | opts := NewBackupableDBOptions(db.name) 59 | env := NewDefaultEnv() 60 | 61 | _, err = OpenBackupEngineWithOpt(opts, env) 62 | require.Nil(t, err) 63 | 64 | env.Destroy() 65 | opts.Destroy() 66 | }() 67 | 68 | { 69 | infos := engine.GetInfo() 70 | require.Empty(t, infos) 71 | 72 | // create first backup 73 | require.Nil(t, engine.CreateNewBackup()) 74 | 75 | // create second backup 76 | require.Nil(t, engine.CreateNewBackupFlush(true)) 77 | 78 | infos = engine.GetInfo() 79 | require.Equal(t, 2, len(infos)) 80 | for i := range infos { 81 | require.Nil(t, engine.VerifyBackup(infos[i].ID)) 82 | require.True(t, infos[i].Size > 0) 83 | require.True(t, infos[i].NumFiles > 0) 84 | } 85 | } 86 | 87 | { 88 | require.Nil(t, engine.PurgeOldBackups(1)) 89 | 90 | infos := engine.GetInfo() 91 | require.Equal(t, 1, len(infos)) 92 | } 93 | 94 | { 95 | dir := t.TempDir() 96 | 97 | ro := NewRestoreOptions() 98 | defer ro.Destroy() 99 | require.Nil(t, engine.RestoreDBFromLatestBackup(dir, dir, ro)) 100 | require.Nil(t, engine.RestoreDBFromLatestBackup(dir, dir, ro)) 101 | } 102 | 103 | { 104 | infos := engine.GetInfo() 105 | require.Equal(t, 1, len(infos)) 106 | 107 | dir := t.TempDir() 108 | 109 | ro := NewRestoreOptions() 110 | defer ro.Destroy() 111 | require.Nil(t, engine.RestoreDBFromBackup(dir, dir, ro, infos[0].ID)) 112 | 113 | // try to reopen restored db 114 | backupDB, err := OpenDb(db.opts, dir) 115 | require.Nil(t, err) 116 | defer backupDB.Close() 117 | 118 | r := NewDefaultReadOptions() 119 | defer r.Destroy() 120 | 121 | for i := 0; i < 1000; i++ { 122 | v3, err := backupDB.GetPinned(r, givenKey) 123 | require.Nil(t, err) 124 | require.EqualValues(t, v3.Data(), givenVal2) 125 | v3.Destroy() 126 | 127 | v4, err := backupDB.GetPinned(r, []byte("justFake")) 128 | require.Nil(t, err) 129 | require.False(t, v4.Exists()) 130 | v4.Destroy() 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 3 | 4 | INSTALL_PREFIX=$1 5 | 6 | BUILD_PATH=/tmp/build 7 | mkdir -p $BUILD_PATH 8 | 9 | CMAKE_REQUIRED_PARAMS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" 10 | 11 | snappy_version="1.2.2" 12 | cd $BUILD_PATH && wget https://github.com/google/snappy/archive/${snappy_version}.tar.gz && tar xzf ${snappy_version}.tar.gz && cd snappy-${snappy_version} && \ 13 | mkdir -p build_place && cd build_place && \ 14 | cmake $CMAKE_REQUIRED_PARAMS -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF .. && \ 15 | make install/strip -j16 && \ 16 | cd $BUILD_PATH && rm -rf * 17 | 18 | export CFLAGS='-fPIC -O3 -pipe' 19 | export CXXFLAGS='-fPIC -O3 -pipe -Wno-uninitialized -Wno-array-bounds' 20 | 21 | zlib_version="1.3.1" 22 | cd $BUILD_PATH && wget https://github.com/madler/zlib/archive/v${zlib_version}.tar.gz && tar xzf v${zlib_version}.tar.gz && cd zlib-${zlib_version} && \ 23 | ./configure --prefix=$INSTALL_PREFIX --static && make -j16 install && \ 24 | cd $BUILD_PATH && rm -rf * 25 | 26 | lz4_version="1.10.0" 27 | cd $BUILD_PATH && wget https://github.com/lz4/lz4/archive/v${lz4_version}.tar.gz && tar xzf v${lz4_version}.tar.gz && cd lz4-${lz4_version}/build/cmake && \ 28 | cmake $CMAKE_REQUIRED_PARAMS -DLZ4_BUILD_LEGACY_LZ4C=OFF -DBUILD_SHARED_LIBS=OFF -DLZ4_POSITION_INDEPENDENT_LIB=ON && make -j16 install && \ 29 | cd $BUILD_PATH && rm -rf * 30 | 31 | zstd_version="1.5.7" 32 | cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz && tar xzf v${zstd_version}.tar.gz && \ 33 | cd zstd-${zstd_version}/build/cmake && mkdir -p build_place && cd build_place && \ 34 | cmake $CMAKE_REQUIRED_PARAMS -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_CONTRIB=OFF -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_TESTS=OFF \ 35 | -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j16 install && \ 36 | cd $BUILD_PATH && rm -rf * && ldconfig 37 | 38 | # Note: if you don't have a good reason, please do not set -DPORTABLE=ON 39 | # This one is set here on purpose of compatibility with github action runtime processor 40 | rocksdb_version="10.2.1" 41 | cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \ 42 | mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \ 43 | -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \ 44 | -DWITH_ZSTD=ON -DWITH_BZ2=OFF -WITH_GFLAGS=OFF -DPORTABLE=1 .. && make -j16 install/strip && \ 45 | cd $BUILD_PATH && rm -rf * 46 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // Cache is a cache used to store data read from data in memory. 7 | type Cache struct { 8 | c *C.rocksdb_cache_t 9 | } 10 | 11 | // NewLRUCache creates a new LRU Cache object with the capacity given. 12 | func NewLRUCache(capacity uint64) *Cache { 13 | cCache := C.rocksdb_cache_create_lru(C.size_t(capacity)) 14 | return newNativeCache(cCache) 15 | } 16 | 17 | // NewLRUCacheWithOptions creates a new LRU Cache from options. 18 | func NewLRUCacheWithOptions(opt *LRUCacheOptions) *Cache { 19 | cCache := C.rocksdb_cache_create_lru_opts(opt.c) 20 | return newNativeCache(cCache) 21 | } 22 | 23 | // NewHyperClockCache creates a new hyper clock cache. 24 | func NewHyperClockCache(capacity, estimatedEntryCharge int) *Cache { 25 | cCache := C.rocksdb_cache_create_hyper_clock(C.size_t(capacity), C.size_t(estimatedEntryCharge)) 26 | return newNativeCache(cCache) 27 | } 28 | 29 | // NewHyperClockCacheWithOpts creates a hyper clock cache with predefined options. 30 | func NewHyperClockCacheWithOpts(opt *HyperClockCacheOptions) *Cache { 31 | cCache := C.rocksdb_cache_create_hyper_clock_opts(opt.c) 32 | return newNativeCache(cCache) 33 | } 34 | 35 | // NewNativeCache creates a Cache object. 36 | func newNativeCache(c *C.rocksdb_cache_t) *Cache { 37 | return &Cache{c: c} 38 | } 39 | 40 | // GetUsage returns the Cache memory usage. 41 | func (c *Cache) GetUsage() uint64 { 42 | return uint64(C.rocksdb_cache_get_usage(c.c)) 43 | } 44 | 45 | // GetPinnedUsage returns the Cache pinned memory usage. 46 | func (c *Cache) GetPinnedUsage() uint64 { 47 | return uint64(C.rocksdb_cache_get_pinned_usage(c.c)) 48 | } 49 | 50 | // TODO: try to re-enable later, along with next release of RocksDB 51 | 52 | // // GetTableAddressCount returns the number of ways the hash function is divided for addressing 53 | // // entries. Zero means "not supported." This is used for inspecting the load 54 | // // factor, along with GetOccupancyCount(). 55 | // func (c *Cache) GetTableAddressCount() int { 56 | // return int(rocksdb_cache_get_table_address_count(c.c)) 57 | // } 58 | 59 | // // GetOccupancyCount returns the number of entries currently tracked in the table. SIZE_MAX 60 | // // means "not supported." This is used for inspecting the load factor, along 61 | // // with GetTableAddressCount(). 62 | // func (c *Cache) GetOccupancyCount() int { 63 | // return int(rocksdb_cache_get_occupancy_count(c.c)) 64 | // } 65 | 66 | // SetCapacity sets capacity of the cache. 67 | func (c *Cache) SetCapacity(value uint64) { 68 | C.rocksdb_cache_set_capacity(c.c, C.size_t(value)) 69 | } 70 | 71 | // GetCapacity returns capacity of the cache. 72 | func (c *Cache) GetCapacity() uint64 { 73 | return uint64(C.rocksdb_cache_get_capacity(c.c)) 74 | } 75 | 76 | // Disowndata call this on shutdown if you want to speed it up. Cache will disown 77 | // any underlying data and will not free it on delete. This call will leak 78 | // memory - call this only if you're shutting down the process. 79 | // Any attempts of using cache after this call will fail terribly. 80 | // Always delete the DB object before calling this method! 81 | func (c *Cache) DisownData() { 82 | C.rocksdb_cache_disown_data(c.c) 83 | } 84 | 85 | // Destroy deallocates the Cache object. 86 | func (c *Cache) Destroy() { 87 | C.rocksdb_cache_destroy(c.c) 88 | c.c = nil 89 | } 90 | 91 | // LRUCacheOptions are options for LRU Cache. 92 | type LRUCacheOptions struct { 93 | c *C.rocksdb_lru_cache_options_t 94 | } 95 | 96 | // NewLRUCacheOptions creates lru cache options. 97 | func NewLRUCacheOptions() *LRUCacheOptions { 98 | return &LRUCacheOptions{c: C.rocksdb_lru_cache_options_create()} 99 | } 100 | 101 | // Destroy lru cache options. 102 | func (l *LRUCacheOptions) Destroy() { 103 | C.rocksdb_lru_cache_options_destroy(l.c) 104 | l.c = nil 105 | } 106 | 107 | // SetCapacity sets capacity for this lru cache. 108 | func (l *LRUCacheOptions) SetCapacity(s uint) { 109 | C.rocksdb_lru_cache_options_set_capacity(l.c, C.size_t(s)) 110 | } 111 | 112 | // SetCapacity sets number of shards used for this lru cache. 113 | func (l *LRUCacheOptions) SetNumShardBits(n int) { 114 | C.rocksdb_lru_cache_options_set_num_shard_bits(l.c, C.int(n)) 115 | } 116 | 117 | // SetMemoryAllocator for this lru cache. 118 | func (l *LRUCacheOptions) SetMemoryAllocator(m *MemoryAllocator) { 119 | C.rocksdb_lru_cache_options_set_memory_allocator(l.c, m.c) 120 | } 121 | 122 | // HyperClockCacheOptions are options for HyperClockCache. 123 | // 124 | // HyperClockCache is a lock-free Cache alternative for RocksDB block cache 125 | // that offers much improved CPU efficiency vs. LRUCache under high parallel 126 | // load or high contention, with some caveats: 127 | // * Not a general Cache implementation: can only be used for 128 | // BlockBasedTableOptions::block_cache, which RocksDB uses in a way that is 129 | // compatible with HyperClockCache. 130 | // * Requires an extra tuning parameter: see estimated_entry_charge below. 131 | // Similarly, substantially changing the capacity with SetCapacity could 132 | // harm efficiency. 133 | // * SecondaryCache is not yet supported. 134 | // * Cache priorities are less aggressively enforced, which could cause 135 | // cache dilution from long range scans (unless they use fill_cache=false). 136 | // * Can be worse for small caches, because if almost all of a cache shard is 137 | // pinned (more likely with non-partitioned filters), then CLOCK eviction 138 | // becomes very CPU intensive. 139 | // 140 | // See internal cache/clock_cache.h for full description. 141 | type HyperClockCacheOptions struct { 142 | c *C.rocksdb_hyper_clock_cache_options_t 143 | } 144 | 145 | // NewHyperClockCacheOptions creates new options for hyper clock cache. 146 | func NewHyperClockCacheOptions(capacity, estimatedEntryCharge int) *HyperClockCacheOptions { 147 | return &HyperClockCacheOptions{ 148 | c: C.rocksdb_hyper_clock_cache_options_create(C.size_t(capacity), C.size_t(estimatedEntryCharge)), 149 | } 150 | } 151 | 152 | // SetCapacity sets the capacity of the cache. 153 | func (h *HyperClockCacheOptions) SetCapacity(capacity int) { 154 | C.rocksdb_hyper_clock_cache_options_set_capacity(h.c, C.size_t(capacity)) 155 | } 156 | 157 | // SetEstimatedEntryCharge sets the estimated average `charge` associated with cache entries. 158 | // 159 | // This is a critical configuration parameter for good performance from the hyper 160 | // cache, because having a table size that is fixed at creation time greatly 161 | // reduces the required synchronization between threads. 162 | // * If the estimate is substantially too low (e.g. less than half the true 163 | // average) then metadata space overhead with be substantially higher (e.g. 164 | // 200 bytes per entry rather than 100). With kFullChargeCacheMetadata, this 165 | // can slightly reduce cache hit rates, and slightly reduce access times due 166 | // to the larger working memory size. 167 | // * If the estimate is substantially too high (e.g. 25% higher than the true 168 | // average) then there might not be sufficient slots in the hash table for 169 | // both efficient operation and capacity utilization (hit rate). The hyper 170 | // cache will evict entries to prevent load factors that could dramatically 171 | // affect lookup times, instead letting the hit rate suffer by not utilizing 172 | // the full capacity. 173 | // 174 | // A reasonable choice is the larger of block_size and metadata_block_size. 175 | // When WriteBufferManager (and similar) charge memory usage to the block 176 | // cache, this can lead to the same effect as estimate being too low, which 177 | // is better than the opposite. Therefore, the general recommendation is to 178 | // assume that other memory charged to block cache could be negligible, and 179 | // ignore it in making the estimate. 180 | // 181 | // The best parameter choice based on a cache in use is given by 182 | // GetUsage() / GetOccupancyCount(), ignoring metadata overheads such as 183 | // with kDontChargeCacheMetadata. More precisely with 184 | // kFullChargeCacheMetadata is (GetUsage() - 64 * GetTableAddressCount()) / 185 | // GetOccupancyCount(). However, when the average value size might vary 186 | // (e.g. balance between metadata and data blocks in cache), it is better 187 | // to estimate toward the lower side than the higher side. 188 | func (h *HyperClockCacheOptions) SetEstimatedEntryCharge(v int) { 189 | C.rocksdb_hyper_clock_cache_options_set_estimated_entry_charge(h.c, C.size_t(v)) 190 | } 191 | 192 | // SetCapacity sets number of shards used for this cache. 193 | func (h *HyperClockCacheOptions) SetNumShardBits(n int) { 194 | C.rocksdb_hyper_clock_cache_options_set_num_shard_bits(h.c, C.int(n)) 195 | } 196 | 197 | // SetMemoryAllocator for this cache. 198 | func (h *HyperClockCacheOptions) SetMemoryAllocator(m *MemoryAllocator) { 199 | C.rocksdb_hyper_clock_cache_options_set_memory_allocator(h.c, m.c) 200 | } 201 | 202 | // Destroy the options. 203 | func (h *HyperClockCacheOptions) Destroy() { 204 | C.rocksdb_hyper_clock_cache_options_destroy(h.c) 205 | h.c = nil 206 | } 207 | -------------------------------------------------------------------------------- /cache_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestLRUCache(t *testing.T) { 10 | t.Parallel() 11 | 12 | cache := NewLRUCache(19) 13 | defer cache.Destroy() 14 | 15 | require.EqualValues(t, 19, cache.GetCapacity()) 16 | cache.SetCapacity(128) 17 | require.EqualValues(t, 128, cache.GetCapacity()) 18 | 19 | cache.DisownData() 20 | } 21 | 22 | func TestHyperClockCache(t *testing.T) { 23 | t.Parallel() 24 | 25 | cache := NewHyperClockCache(100, 10) 26 | defer cache.Destroy() 27 | 28 | require.EqualValues(t, 100, cache.GetCapacity()) 29 | cache.SetCapacity(128) 30 | require.EqualValues(t, 128, cache.GetCapacity()) 31 | 32 | cache.DisownData() 33 | } 34 | 35 | func TestLRUCacheWithOpts(t *testing.T) { 36 | t.Parallel() 37 | 38 | opts := NewLRUCacheOptions() 39 | opts.SetCapacity(19) 40 | opts.SetNumShardBits(2) 41 | defer opts.Destroy() 42 | 43 | cache := NewLRUCacheWithOptions(opts) 44 | defer cache.Destroy() 45 | 46 | require.EqualValues(t, 19, cache.GetCapacity()) 47 | cache.SetCapacity(128) 48 | require.EqualValues(t, 128, cache.GetCapacity()) 49 | 50 | cache.DisownData() 51 | } 52 | 53 | func TestHyperClockCacheWithOpts(t *testing.T) { 54 | t.Parallel() 55 | 56 | opts := NewHyperClockCacheOptions(100, 10) 57 | opts.SetCapacity(19) 58 | opts.SetEstimatedEntryCharge(10) 59 | opts.SetNumShardBits(2) 60 | defer opts.Destroy() 61 | 62 | cache := NewHyperClockCacheWithOpts(opts) 63 | defer cache.Destroy() 64 | 65 | require.EqualValues(t, 19, cache.GetCapacity()) 66 | cache.SetCapacity(128) 67 | require.EqualValues(t, 128, cache.GetCapacity()) 68 | 69 | cache.DisownData() 70 | } 71 | -------------------------------------------------------------------------------- /cf_handle.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | // ColumnFamilyHandle represents a handle to a ColumnFamily. 8 | type ColumnFamilyHandle struct { 9 | c *C.rocksdb_column_family_handle_t 10 | } 11 | 12 | // NewNativeColumnFamilyHandle creates a ColumnFamilyHandle object. 13 | func newNativeColumnFamilyHandle(c *C.rocksdb_column_family_handle_t) *ColumnFamilyHandle { 14 | return &ColumnFamilyHandle{c: c} 15 | } 16 | 17 | // ID returned id of Column family. 18 | func (h *ColumnFamilyHandle) ID() uint32 { 19 | return uint32(C.rocksdb_column_family_handle_get_id(h.c)) 20 | } 21 | 22 | // Name returned name of Column family. 23 | func (h *ColumnFamilyHandle) Name() string { 24 | var len C.size_t 25 | cValue := C.rocksdb_column_family_handle_get_name(h.c, &len) 26 | return toString(cValue, C.int(len)) 27 | } 28 | 29 | // Destroy calls the destructor of the underlying column family handle. 30 | func (h *ColumnFamilyHandle) Destroy() { 31 | C.rocksdb_column_family_handle_destroy(h.c) 32 | h.c = nil 33 | } 34 | 35 | // ColumnFamilyHandles represents collection of multiple column family handle. 36 | type ColumnFamilyHandles []*ColumnFamilyHandle 37 | 38 | func (cfs ColumnFamilyHandles) toCSlice() columnFamilySlice { 39 | cCFs := make(columnFamilySlice, len(cfs)) 40 | for i, cf := range cfs { 41 | cCFs[i] = cf.c 42 | } 43 | return cCFs 44 | } 45 | -------------------------------------------------------------------------------- /cf_metadata.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // ColumnFamilyMetadata contains metadata info of column family. 8 | type ColumnFamilyMetadata struct { 9 | size uint64 10 | fileCount int 11 | name string 12 | levelMetas []LevelMetadata 13 | } 14 | 15 | func newColumnFamilyMetadata(c *C.rocksdb_column_family_metadata_t) *ColumnFamilyMetadata { 16 | return &ColumnFamilyMetadata{ 17 | size: uint64(C.rocksdb_column_family_metadata_get_size(c)), 18 | fileCount: int(C.rocksdb_column_family_metadata_get_file_count(c)), 19 | name: C.GoString(C.rocksdb_column_family_metadata_get_name(c)), 20 | levelMetas: levelMetas(c), 21 | } 22 | } 23 | 24 | // GetSize returns size of this column family in bytes, which is equal to the sum of 25 | // the file size of its "levels". 26 | func (cm *ColumnFamilyMetadata) Size() uint64 { 27 | return cm.size 28 | } 29 | 30 | // FileCount returns number of files in this column family. 31 | func (cm *ColumnFamilyMetadata) FileCount() int { 32 | return cm.fileCount 33 | } 34 | 35 | // Name returns name of this column family. 36 | func (cm *ColumnFamilyMetadata) Name() string { 37 | return cm.name 38 | } 39 | 40 | // LevelMetas returns metadata(s) of each level. 41 | func (cm *ColumnFamilyMetadata) LevelMetas() []LevelMetadata { 42 | return cm.levelMetas 43 | } 44 | 45 | // LevelMetadata represents the metadata that describes a level. 46 | type LevelMetadata struct { 47 | level int 48 | size uint64 49 | sstMetas []SstMetadata 50 | } 51 | 52 | func levelMetas(c *C.rocksdb_column_family_metadata_t) []LevelMetadata { 53 | n := int(C.rocksdb_column_family_metadata_get_level_count(c)) 54 | 55 | metas := make([]LevelMetadata, n) 56 | for i := range metas { 57 | lm := C.rocksdb_column_family_metadata_get_level_metadata(c, C.size_t(i)) 58 | metas[i].level = int(C.rocksdb_level_metadata_get_level(lm)) 59 | metas[i].size = uint64(C.rocksdb_level_metadata_get_size(lm)) 60 | metas[i].sstMetas = sstMetas(lm) 61 | } 62 | 63 | C.rocksdb_column_family_metadata_destroy(c) 64 | 65 | return metas 66 | } 67 | 68 | // Level returns level value. 69 | func (l *LevelMetadata) Level() int { 70 | return l.level 71 | } 72 | 73 | // Size returns the sum of the file size in this level. 74 | func (l *LevelMetadata) Size() uint64 { 75 | return l.size 76 | } 77 | 78 | // SstMetas returns metadata(s) of sst-file(s) in this level. 79 | func (l *LevelMetadata) SstMetas() []SstMetadata { 80 | return l.sstMetas 81 | } 82 | 83 | // SstMetadata represents metadata of sst file. 84 | type SstMetadata struct { 85 | relativeFileName string 86 | directory string 87 | size uint64 88 | smallestKey []byte 89 | largestKey []byte 90 | } 91 | 92 | func sstMetas(c *C.rocksdb_level_metadata_t) []SstMetadata { 93 | n := int(C.rocksdb_level_metadata_get_file_count(c)) 94 | 95 | metas := make([]SstMetadata, n) 96 | for i := range metas { 97 | sm := C.rocksdb_level_metadata_get_sst_file_metadata(c, C.size_t(i)) 98 | metas[i].relativeFileName = C.GoString(C.rocksdb_sst_file_metadata_get_relative_filename(sm)) 99 | metas[i].directory = C.GoString(C.rocksdb_sst_file_metadata_get_directory(sm)) 100 | metas[i].size = uint64(C.rocksdb_sst_file_metadata_get_size(sm)) 101 | 102 | var ln C.size_t 103 | sk := C.rocksdb_sst_file_metadata_get_smallestkey(sm, &ln) 104 | metas[i].smallestKey = C.GoBytes(unsafe.Pointer(sk), C.int(ln)) 105 | C.rocksdb_free(unsafe.Pointer(sk)) 106 | 107 | sk = C.rocksdb_sst_file_metadata_get_largestkey(sm, &ln) 108 | metas[i].largestKey = C.GoBytes(unsafe.Pointer(sk), C.int(ln)) 109 | C.rocksdb_free(unsafe.Pointer(sk)) 110 | 111 | C.rocksdb_sst_file_metadata_destroy(sm) 112 | } 113 | 114 | C.rocksdb_level_metadata_destroy(c) 115 | 116 | return metas 117 | } 118 | 119 | // RelativeFileName returns relative file name. 120 | func (s *SstMetadata) RelativeFileName() string { 121 | return s.relativeFileName 122 | } 123 | 124 | // Size returns size of this sst file. 125 | func (s *SstMetadata) Size() uint64 { 126 | return s.size 127 | } 128 | 129 | // SmallestKey returns smallest-key in this sst file. 130 | func (s *SstMetadata) SmallestKey() []byte { 131 | return s.smallestKey 132 | } 133 | 134 | // LargestKey returns largest-key in this sst file. 135 | func (s *SstMetadata) LargestKey() []byte { 136 | return s.largestKey 137 | } 138 | -------------------------------------------------------------------------------- /cf_ts_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestColumnFamilyPutGetDeleteWithTS(t *testing.T) { 10 | t.Parallel() 11 | 12 | dir := t.TempDir() 13 | 14 | givenNames := []string{"default", "guide"} 15 | opts := NewDefaultOptions() 16 | opts.SetCreateIfMissingColumnFamilies(true) 17 | opts.SetCreateIfMissing(true) 18 | opts.SetCompression(SnappyCompression) 19 | opts.SetComparator(newDefaultComparatorWithTS()) 20 | db, cfh, err := OpenDbColumnFamilies(opts, dir, givenNames, []*Options{opts, opts}) 21 | require.Nil(t, err) 22 | defer db.Close() 23 | require.EqualValues(t, len(cfh), 2) 24 | defer cfh[0].Destroy() 25 | defer cfh[1].Destroy() 26 | 27 | wo := NewDefaultWriteOptions() 28 | defer wo.Destroy() 29 | ro := NewDefaultReadOptions() 30 | defer ro.Destroy() 31 | 32 | givenKey0 := []byte("hello0") 33 | givenKey1 := []byte("hello1") 34 | givenVal0 := []byte("world0") 35 | givenVal1 := []byte("world1") 36 | givenTs0 := marshalTimestamp(1) 37 | givenTs1 := marshalTimestamp(2) 38 | givenTs2 := marshalTimestamp(3) 39 | 40 | { 41 | ro.SetTimestamp(givenTs2) 42 | 43 | require.Nil(t, db.PutCFWithTS(wo, cfh[0], givenKey0, givenTs0, givenVal0)) 44 | actualVal0, actualTs0, err := db.GetCFWithTS(ro, cfh[0], givenKey0) 45 | require.Nil(t, err) 46 | require.EqualValues(t, actualVal0.Data(), givenVal0) 47 | require.EqualValues(t, actualTs0.Data(), givenTs0) 48 | actualVal0.Free() 49 | actualTs0.Free() 50 | 51 | require.Nil(t, db.PutCFWithTS(wo, cfh[1], givenKey1, givenTs1, givenVal1)) 52 | actualVal1, actualTs1, err := db.GetCFWithTS(ro, cfh[1], givenKey1) 53 | require.Nil(t, err) 54 | require.EqualValues(t, actualVal1.Data(), givenVal1) 55 | require.EqualValues(t, actualTs1.Data(), givenTs1) 56 | actualVal1.Free() 57 | actualTs1.Free() 58 | 59 | actualVal, actualTs, err := db.GetCFWithTS(ro, cfh[0], givenKey1) 60 | require.Nil(t, err) 61 | require.EqualValues(t, actualVal.Size(), 0) 62 | require.EqualValues(t, actualTs.Size(), 0) 63 | actualVal.Free() 64 | actualTs.Free() 65 | 66 | actualVal, actualTs, err = db.GetCFWithTS(ro, cfh[1], givenKey0) 67 | require.Nil(t, err) 68 | require.EqualValues(t, actualVal.Size(), 0) 69 | require.EqualValues(t, actualTs.Size(), 0) 70 | actualVal.Free() 71 | actualTs.Free() 72 | 73 | require.Nil(t, db.DeleteCFWithTS(wo, cfh[0], givenKey0, givenTs2)) 74 | actualVal, actualTs, err = db.GetCFWithTS(ro, cfh[0], givenKey0) 75 | require.Nil(t, err) 76 | require.EqualValues(t, actualVal.Size(), 0) 77 | require.EqualValues(t, actualTs.Size(), 0) 78 | actualVal.Free() 79 | actualTs.Free() 80 | } 81 | 82 | { 83 | require.Nil(t, db.PutCFWithTS(wo, cfh[0], givenKey0, givenTs2, givenVal0)) 84 | actualVal0, actualTs0, err := db.GetCFWithTS(ro, cfh[0], givenKey0) 85 | require.Nil(t, err) 86 | require.EqualValues(t, actualVal0.Data(), givenVal0) 87 | require.EqualValues(t, actualTs0.Data(), givenTs2) 88 | actualVal0.Free() 89 | actualTs0.Free() 90 | 91 | actualVal1, actualTs1, err := db.GetCFWithTS(ro, cfh[1], givenKey1) 92 | require.Nil(t, err) 93 | require.EqualValues(t, actualVal1.Data(), givenVal1) 94 | require.EqualValues(t, actualTs1.Data(), givenTs1) 95 | actualVal1.Free() 96 | actualTs1.Free() 97 | } 98 | } 99 | 100 | func TestColumnFamilyMultiGetWithTS(t *testing.T) { 101 | t.Parallel() 102 | 103 | db, cfh, cleanup := newTestDBMultiCF(t, []string{"default", "custom"}, func(opts *Options) { 104 | opts.SetComparator(newDefaultComparatorWithTS()) 105 | }) 106 | defer cleanup() 107 | 108 | var ( 109 | givenKey1 = []byte("hello1") 110 | givenKey2 = []byte("hello2") 111 | givenKey3 = []byte("hello3") 112 | givenVal1 = []byte("world1") 113 | givenVal2 = []byte("world2") 114 | givenVal3 = []byte("world3") 115 | givenTs1 = marshalTimestamp(0) 116 | givenTs2 = marshalTimestamp(2) 117 | givenTs3 = marshalTimestamp(3) 118 | ) 119 | 120 | wo := NewDefaultWriteOptions() 121 | defer wo.Destroy() 122 | 123 | ro := NewDefaultReadOptions() 124 | ro.SetTimestamp(givenTs3) 125 | defer ro.Destroy() 126 | 127 | // create 128 | require.Nil(t, db.PutCFWithTS(wo, cfh[0], givenKey1, givenTs1, givenVal1)) 129 | require.Nil(t, db.PutCFWithTS(wo, cfh[1], givenKey2, givenTs2, givenVal2)) 130 | require.Nil(t, db.PutCFWithTS(wo, cfh[1], givenKey3, givenTs3, givenVal3)) 131 | 132 | // column family 0 only has givenKey1 133 | values, times, err := db.MultiGetCFWithTS(ro, cfh[0], []byte("noexist"), givenKey1, givenKey2, givenKey3) 134 | require.Nil(t, err) 135 | require.EqualValues(t, len(values), 4) 136 | require.EqualValues(t, values[0].Data(), []byte(nil)) 137 | require.EqualValues(t, values[1].Data(), givenVal1) 138 | require.EqualValues(t, values[2].Data(), []byte(nil)) 139 | require.EqualValues(t, values[3].Data(), []byte(nil)) 140 | 141 | require.EqualValues(t, times[0].Data(), []byte(nil)) 142 | require.EqualValues(t, times[1].Data(), givenTs1) 143 | require.EqualValues(t, times[2].Data(), []byte(nil)) 144 | require.EqualValues(t, times[3].Data(), []byte(nil)) 145 | values.Destroy() 146 | times.Destroy() 147 | 148 | // column family 1 only has givenKey2 and givenKey3 149 | values, times, err = db.MultiGetCFWithTS(ro, cfh[1], []byte("noexist"), givenKey1, givenKey2, givenKey3) 150 | require.Nil(t, err) 151 | require.EqualValues(t, len(values), 4) 152 | require.EqualValues(t, values[0].Data(), []byte(nil)) 153 | require.EqualValues(t, values[1].Data(), []byte(nil)) 154 | require.EqualValues(t, values[2].Data(), givenVal2) 155 | require.EqualValues(t, values[3].Data(), givenVal3) 156 | require.EqualValues(t, times[0].Data(), []byte(nil)) 157 | require.EqualValues(t, times[1].Data(), []byte(nil)) 158 | require.EqualValues(t, times[2].Data(), givenTs2) 159 | require.EqualValues(t, times[3].Data(), givenTs3) 160 | values.Destroy() 161 | times.Destroy() 162 | 163 | // getting them all from the right CF 164 | values, times, err = db.MultiGetMultiCFWithTS(ro, 165 | ColumnFamilyHandles{cfh[0], cfh[1], cfh[1]}, 166 | [][]byte{givenKey1, givenKey2, givenKey3}, 167 | ) 168 | require.Nil(t, err) 169 | require.EqualValues(t, len(values), 3) 170 | require.EqualValues(t, values[0].Data(), givenVal1) 171 | require.EqualValues(t, values[1].Data(), givenVal2) 172 | require.EqualValues(t, values[2].Data(), givenVal3) 173 | require.EqualValues(t, times[0].Data(), []byte{0, 0, 0, 0, 0, 0, 0, 0}) 174 | require.EqualValues(t, times[1].Data(), givenTs2) 175 | require.EqualValues(t, times[2].Data(), givenTs3) 176 | values.Destroy() 177 | times.Destroy() 178 | } 179 | -------------------------------------------------------------------------------- /checkpoint.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // Checkpoint provides persistent snapshots of RocksDB databases. 12 | type Checkpoint struct { 13 | c *C.rocksdb_checkpoint_t 14 | } 15 | 16 | // NewNativeCheckpoint creates a new checkpoint. 17 | func newNativeCheckpoint(c *C.rocksdb_checkpoint_t) *Checkpoint { 18 | return &Checkpoint{c: c} 19 | } 20 | 21 | // CreateCheckpoint builds an openable snapshot of RocksDB on the same disk, which 22 | // accepts an output directory on the same disk, and under the directory 23 | // (1) hard-linked SST files pointing to existing live SST files 24 | // SST files will be copied if output directory is on a different filesystem 25 | // (2) a copied manifest files and other files 26 | // The directory should not already exist and will be created by this API. 27 | // The directory will be an absolute path 28 | // log_size_for_flush: if the total log file size is equal or larger than 29 | // this value, then a flush is triggered for all the column families. The 30 | // default value is 0, which means flush is always triggered. If you move 31 | // away from the default, the checkpoint may not contain up-to-date data 32 | // if WAL writing is not always enabled. 33 | // Flush will always trigger if it is 2PC. 34 | func (checkpoint *Checkpoint) CreateCheckpoint(checkpointDir string, logSizeForFlush uint64) (err error) { 35 | cDir := C.CString(checkpointDir) 36 | 37 | var cErr *C.char 38 | C.rocksdb_checkpoint_create(checkpoint.c, cDir, C.uint64_t(logSizeForFlush), &cErr) 39 | err = fromCError(cErr) 40 | 41 | C.free(unsafe.Pointer(cDir)) 42 | return 43 | } 44 | 45 | // Destroy deallocates the Checkpoint object. 46 | func (checkpoint *Checkpoint) Destroy() { 47 | C.rocksdb_checkpoint_object_destroy(checkpoint.c) 48 | checkpoint.c = nil 49 | } 50 | -------------------------------------------------------------------------------- /checkpoint_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestCheckpoint(t *testing.T) { 11 | t.Parallel() 12 | 13 | dir := t.TempDir() 14 | err := os.RemoveAll(dir) 15 | require.Nil(t, err) 16 | 17 | db := newTestDB(t, nil) 18 | defer db.Close() 19 | 20 | // insert keys 21 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 22 | givenVal := []byte("val") 23 | wo := NewDefaultWriteOptions() 24 | for _, k := range givenKeys { 25 | require.Nil(t, db.Put(wo, k, givenVal)) 26 | } 27 | 28 | var dbCheck *DB 29 | var checkpoint *Checkpoint 30 | 31 | checkpoint, err = db.NewCheckpoint() 32 | defer checkpoint.Destroy() 33 | require.NotNil(t, checkpoint) 34 | require.Nil(t, err) 35 | 36 | err = checkpoint.CreateCheckpoint(dir, 0) 37 | require.Nil(t, err) 38 | 39 | opts := NewDefaultOptions() 40 | opts.SetCreateIfMissing(true) 41 | dbCheck, err = OpenDb(opts, dir) 42 | require.Nil(t, err) 43 | defer dbCheck.Close() 44 | 45 | // test keys 46 | var value *Slice 47 | ro := NewDefaultReadOptions() 48 | for _, k := range givenKeys { 49 | value, err = dbCheck.Get(ro, k) 50 | require.Nil(t, err) 51 | require.EqualValues(t, value.Data(), givenVal) 52 | value.Free() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /compaction_filter.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // A CompactionFilter can be used to filter keys during compaction time. 8 | type CompactionFilter interface { 9 | // If the Filter function returns false, it indicates 10 | // that the kv should be preserved, while a return value of true 11 | // indicates that this key-value should be removed from the 12 | // output of the compaction. The application can inspect 13 | // the existing value of the key and make decision based on it. 14 | // 15 | // When the value is to be preserved, the application has the option 16 | // to modify the existing value and pass it back through a new value. 17 | // To retain the previous value, simply return nil 18 | // 19 | // If multithreaded compaction is being used *and* a single CompactionFilter 20 | // instance was supplied via SetCompactionFilter, this the Filter function may be 21 | // called from different threads concurrently. The application must ensure 22 | // that the call is thread-safe. 23 | Filter(level int, key, val []byte) (remove bool, newVal []byte) 24 | 25 | // The name of the compaction filter, for logging 26 | Name() string 27 | 28 | // SetIgnoreSnapshots before release 6.0, if there is a snapshot taken later than 29 | // the key/value pair, RocksDB always try to prevent the key/value pair from being 30 | // filtered by compaction filter so that users can preserve the same view from a 31 | // snapshot, unless the compaction filter returns IgnoreSnapshots() = true. However, 32 | // this feature is deleted since 6.0, after realized that the feature has a bug which 33 | // can't be easily fixed. Since release 6.0, with compaction filter enabled, RocksDB 34 | // always invoke filtering for any key, even if it knows it will make a snapshot 35 | // not repeatable. 36 | SetIgnoreSnapshots(value bool) 37 | 38 | // Destroy underlying pointer/data. 39 | Destroy() 40 | } 41 | 42 | // NewNativeCompactionFilter creates a CompactionFilter object. 43 | func NewNativeCompactionFilter(c unsafe.Pointer) CompactionFilter { 44 | return &nativeCompactionFilter{c: (*C.rocksdb_compactionfilter_t)(c)} 45 | } 46 | 47 | type nativeCompactionFilter struct { 48 | c *C.rocksdb_compactionfilter_t 49 | } 50 | 51 | func (c *nativeCompactionFilter) Filter(level int, key, val []byte) (remove bool, newVal []byte) { 52 | return false, nil 53 | } 54 | func (c *nativeCompactionFilter) Name() string { return "" } 55 | 56 | func (c *nativeCompactionFilter) SetIgnoreSnapshots(value bool) { 57 | C.rocksdb_compactionfilter_set_ignore_snapshots(c.c, boolToChar(value)) 58 | } 59 | 60 | func (c *nativeCompactionFilter) Destroy() { 61 | C.rocksdb_compactionfilter_destroy(c.c) 62 | c.c = nil 63 | } 64 | 65 | // Hold references to compaction filters. 66 | var compactionFilters = NewCOWList() 67 | 68 | type compactionFilterWrapper struct { 69 | name *C.char 70 | filter CompactionFilter 71 | } 72 | 73 | func registerCompactionFilter(filter CompactionFilter) int { 74 | return compactionFilters.Append(compactionFilterWrapper{C.CString(filter.Name()), filter}) 75 | } 76 | 77 | //export gorocksdb_compactionfilter_filter 78 | func gorocksdb_compactionfilter_filter(idx int, cLevel C.int, cKey *C.char, cKeyLen C.size_t, cVal *C.char, cValLen C.size_t, cNewVal **C.char, cNewValLen *C.size_t, cValChanged *C.uchar) C.int { 79 | key := refCBytes(cKey, cKeyLen) 80 | val := refCBytes(cVal, cValLen) 81 | 82 | remove, newVal := compactionFilters.Get(idx).(compactionFilterWrapper).filter.Filter(int(cLevel), key, val) 83 | if remove { 84 | return C.int(1) 85 | } else if newVal != nil { 86 | *cNewVal = refGoBytes(newVal) 87 | *cNewValLen = C.size_t(len(newVal)) 88 | *cValChanged = C.uchar(1) 89 | } 90 | return C.int(0) 91 | } 92 | 93 | //export gorocksdb_compactionfilter_name 94 | func gorocksdb_compactionfilter_name(idx int) *C.char { 95 | return compactionFilters.Get(idx).(compactionFilterWrapper).name 96 | } 97 | -------------------------------------------------------------------------------- /compaction_filter_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestCompactionFilter(t *testing.T) { 11 | t.Parallel() 12 | 13 | var ( 14 | changeKey = []byte("change") 15 | changeValOld = []byte("old") 16 | changeValNew = []byte("new") 17 | deleteKey = []byte("delete") 18 | ) 19 | db := newTestDB(t, func(opts *Options) { 20 | opts.SetCompactionFilter(&mockCompactionFilter{ 21 | filter: func(_ int, key, val []byte) (remove bool, newVal []byte) { 22 | if bytes.Equal(key, changeKey) { 23 | return false, changeValNew 24 | } 25 | if bytes.Equal(key, deleteKey) { 26 | return true, val 27 | } 28 | t.Errorf("key %q not expected during compaction", key) 29 | return false, nil 30 | }, 31 | }) 32 | }) 33 | defer db.Close() 34 | 35 | // insert the test keys 36 | wo := NewDefaultWriteOptions() 37 | require.Nil(t, db.Put(wo, changeKey, changeValOld)) 38 | require.Nil(t, db.Put(wo, deleteKey, changeValNew)) 39 | 40 | // trigger a compaction 41 | db.CompactRange(Range{}) 42 | require.NoError(t, db.SuggestCompactRange(Range{})) 43 | 44 | // ensure that the value is changed after compaction 45 | ro := NewDefaultReadOptions() 46 | v1, err := db.Get(ro, changeKey) 47 | defer v1.Free() 48 | require.Nil(t, err) 49 | require.EqualValues(t, v1.Data(), changeValNew) 50 | 51 | // ensure that the key is deleted after compaction 52 | v2, err := db.Get(ro, deleteKey) 53 | require.Nil(t, err) 54 | require.Nil(t, v2.Data()) 55 | } 56 | 57 | type mockCompactionFilter struct { 58 | filter func(level int, key, val []byte) (remove bool, newVal []byte) 59 | } 60 | 61 | func (m *mockCompactionFilter) Name() string { return "grocksdb.test" } 62 | 63 | func (m *mockCompactionFilter) Filter(level int, key, val []byte) (bool, []byte) { 64 | return m.filter(level, key, val) 65 | } 66 | 67 | func (m *mockCompactionFilter) SetIgnoreSnapshots(value bool) { 68 | } 69 | 70 | func (m *mockCompactionFilter) Destroy() {} 71 | -------------------------------------------------------------------------------- /comparator.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | // #include "grocksdb.h" 5 | import "C" 6 | 7 | // Comparing functor. 8 | // 9 | // Three-way comparison. Returns value: 10 | // 11 | // < 0 iff "a" < "b", 12 | // == 0 iff "a" == "b", 13 | // > 0 iff "a" > "b" 14 | // 15 | // Note that Compare(a, b) also compares timestamp if timestamp size is 16 | // non-zero. For the same user key with different timestamps, larger (newer) 17 | // timestamp comes first. 18 | type Comparing = func(a, b []byte) int 19 | 20 | // ComparingWithoutTimestamp functor. 21 | // 22 | // Three-way comparison. Returns value: 23 | // 24 | // < 0 if "a" < "b", 25 | // == 0 if "a" == "b", 26 | // > 0 if "a" > "b" 27 | type ComparingWithoutTimestamp = func(a []byte, aHasTs bool, b []byte, bHasTs bool) int 28 | 29 | // NewComparator creates a Comparator object which contains native c-comparator pointer. 30 | func NewComparator(name string, compare Comparing) *Comparator { 31 | cmp := &Comparator{ 32 | name: name, 33 | compare: compare, 34 | } 35 | idx := registerComperator(cmp) 36 | cmp.c = C.gorocksdb_comparator_create(C.uintptr_t(idx)) 37 | return cmp 38 | } 39 | 40 | // NewComparatorWithTimestamp creates a Timestamp Aware Comparator object which contains native c-comparator pointer. 41 | func NewComparatorWithTimestamp(name string, tsSize uint64, compare, compareTs Comparing, compareWithoutTs ComparingWithoutTimestamp) *Comparator { 42 | cmp := &Comparator{ 43 | name: name, 44 | tsSize: tsSize, 45 | compare: compare, 46 | compareTs: compareTs, 47 | compareWithoutTs: compareWithoutTs, 48 | } 49 | idx := registerComperator(cmp) 50 | cmp.c = C.gorocksdb_comparator_with_ts_create(C.uintptr_t(idx), C.size_t(tsSize)) 51 | return cmp 52 | } 53 | 54 | // NativeComparator wraps c-comparator pointer. 55 | type Comparator struct { 56 | c *C.rocksdb_comparator_t 57 | 58 | name string 59 | tsSize uint64 60 | 61 | compare Comparing 62 | compareTs Comparing 63 | compareWithoutTs ComparingWithoutTimestamp 64 | } 65 | 66 | func (c *Comparator) Compare(a, b []byte) int { return c.compare(a, b) } 67 | func (c *Comparator) CompareTimestamp(a, b []byte) int { return c.compareTs(a, b) } 68 | func (c *Comparator) CompareWithoutTimestamp(a []byte, aHasTs bool, b []byte, bHasTs bool) int { 69 | return c.compareWithoutTs(a, aHasTs, b, bHasTs) 70 | } 71 | func (c *Comparator) Name() string { return c.name } 72 | func (c *Comparator) TimestampSize() uint64 { return c.tsSize } 73 | func (c *Comparator) Destroy() { 74 | C.rocksdb_comparator_destroy(c.c) 75 | c.c = nil 76 | } 77 | 78 | // Hold references to comperators. 79 | var comperators = NewCOWList() 80 | 81 | type comperatorWrapper struct { 82 | name *C.char 83 | comparator *Comparator 84 | } 85 | 86 | func registerComperator(cmp *Comparator) int { 87 | return comperators.Append(comperatorWrapper{C.CString(cmp.Name()), cmp}) 88 | } 89 | 90 | //export gorocksdb_comparator_compare 91 | func gorocksdb_comparator_compare(idx int, cKeyA *C.char, cKeyALen C.size_t, cKeyB *C.char, cKeyBLen C.size_t) C.int { 92 | keyA := refCBytes(cKeyA, cKeyALen) 93 | keyB := refCBytes(cKeyB, cKeyBLen) 94 | return C.int(comperators.Get(idx).(comperatorWrapper).comparator.Compare(keyA, keyB)) 95 | } 96 | 97 | //export gorocksdb_comparator_compare_ts 98 | func gorocksdb_comparator_compare_ts(idx int, cTsA *C.char, cTsALen C.size_t, cTsB *C.char, cTsBLen C.size_t) C.int { 99 | tsA := refCBytes(cTsA, cTsALen) 100 | tsB := refCBytes(cTsB, cTsBLen) 101 | return C.int(comperators.Get(idx).(comperatorWrapper).comparator.CompareTimestamp(tsA, tsB)) 102 | } 103 | 104 | //export gorocksdb_comparator_compare_without_ts 105 | func gorocksdb_comparator_compare_without_ts(idx int, cKeyA *C.char, cKeyALen C.size_t, cAHasTs C.uchar, cKeyB *C.char, cKeyBLen C.size_t, cBHasTs C.uchar) C.int { 106 | keyA := refCBytes(cKeyA, cKeyALen) 107 | keyB := refCBytes(cKeyB, cKeyBLen) 108 | keyAHasTs := charToBool(cAHasTs) 109 | keyBHasTs := charToBool(cBHasTs) 110 | return C.int(comperators.Get(idx).(comperatorWrapper).comparator.CompareWithoutTimestamp(keyA, keyAHasTs, keyB, keyBHasTs)) 111 | } 112 | 113 | //export gorocksdb_comparator_name 114 | func gorocksdb_comparator_name(idx int) *C.char { 115 | return comperators.Get(idx).(comperatorWrapper).name 116 | } 117 | -------------------------------------------------------------------------------- /comparator_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "bytes" 5 | "runtime" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestComparator(t *testing.T) { 12 | t.Parallel() 13 | 14 | db, opts := newTestDBAndOpts(t, func(opts *Options) { 15 | opts.SetComparator(NewComparator("rev", func(a, b []byte) int { 16 | return bytes.Compare(a, b) * -1 17 | })) 18 | }) 19 | defer func() { 20 | db.Close() 21 | opts.Destroy() 22 | }() 23 | 24 | runtime.GC() 25 | 26 | // insert keys 27 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 28 | wo := NewDefaultWriteOptions() 29 | for _, k := range givenKeys { 30 | require.Nil(t, db.Put(wo, k, []byte("val"))) 31 | runtime.GC() 32 | } 33 | 34 | // create a iterator to collect the keys 35 | ro := NewDefaultReadOptions() 36 | iter := db.NewIterator(ro) 37 | defer iter.Close() 38 | 39 | // we seek to the last key and iterate in reverse order 40 | // to match given keys 41 | var actualKeys [][]byte 42 | for iter.SeekToLast(); iter.Valid(); iter.Prev() { 43 | key := make([]byte, 4) 44 | copy(key, iter.Key().Data()) 45 | actualKeys = append(actualKeys, key) 46 | runtime.GC() 47 | } 48 | require.Nil(t, iter.Err()) 49 | 50 | // ensure that the order is correct 51 | require.EqualValues(t, actualKeys, givenKeys) 52 | } 53 | -------------------------------------------------------------------------------- /comparator_ts_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "runtime" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestComparatorWithTS(t *testing.T) { 13 | t.Parallel() 14 | 15 | db, opts := newTestDBAndOpts(t, func(opts *Options) { 16 | comp := newComparatorWithTimeStamp( 17 | "rev", 18 | func(a, b []byte) int { 19 | return bytes.Compare(a, b) * -1 20 | }, 21 | binary.LittleEndian.Uint64, 22 | ) 23 | opts.SetComparator(comp) 24 | }) 25 | defer func() { 26 | db.Close() 27 | opts.Destroy() 28 | }() 29 | 30 | runtime.GC() 31 | 32 | // insert keys 33 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 34 | givenTimes := [][]byte{marshalTimestamp(1), marshalTimestamp(2), marshalTimestamp(3)} 35 | 36 | wo := NewDefaultWriteOptions() 37 | for i, k := range givenKeys { 38 | require.Nil(t, db.PutWithTS(wo, k, givenTimes[i], []byte("val"))) 39 | runtime.GC() 40 | } 41 | 42 | // create a iterator to collect the keys 43 | ro := NewDefaultReadOptions() 44 | ro.SetTimestamp(marshalTimestamp(4)) 45 | iter := db.NewIterator(ro) 46 | defer iter.Close() 47 | 48 | // we seek to the last key and iterate in reverse order 49 | // to match given keys 50 | var actualKeys, actualTimes [][]byte 51 | for iter.SeekToLast(); iter.Valid(); iter.Prev() { 52 | key := make([]byte, 4) 53 | ts := make([]byte, timestampSize) 54 | copy(key, iter.Key().Data()) 55 | copy(ts, iter.Timestamp().Data()) 56 | actualKeys = append(actualKeys, key) 57 | actualTimes = append(actualTimes, ts) 58 | runtime.GC() 59 | } 60 | require.Nil(t, iter.Err()) 61 | 62 | // ensure that the order is correct 63 | require.EqualValues(t, actualKeys, givenKeys) 64 | require.EqualValues(t, actualTimes, givenTimes) 65 | } 66 | -------------------------------------------------------------------------------- /cow.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | // COWList implements a copy-on-write list. It is intended to be used by go 9 | // callback registry for CGO, which is read-heavy with occasional writes. 10 | // Reads do not block; Writes do not block reads (or vice versa), but only 11 | // one write can occur at once; 12 | type COWList struct { 13 | v atomic.Value 14 | mu sync.Mutex 15 | } 16 | 17 | // NewCOWList creates a new COWList. 18 | func NewCOWList() *COWList { 19 | l := &COWList{} 20 | l.v.Store([]interface{}{}) 21 | return l 22 | } 23 | 24 | // Append appends an item to the COWList and returns the index for that item. 25 | func (c *COWList) Append(i interface{}) (index int) { 26 | c.mu.Lock() 27 | list := c.v.Load().([]interface{}) 28 | newLen := len(list) + 1 29 | newList := make([]interface{}, newLen) 30 | copy(newList, list) 31 | newList[newLen-1] = i 32 | c.v.Store(newList) 33 | c.mu.Unlock() 34 | index = newLen - 1 35 | return 36 | } 37 | 38 | // Get gets the item at index. 39 | func (c *COWList) Get(index int) interface{} { 40 | list := c.v.Load().([]interface{}) 41 | return list[index] 42 | } 43 | -------------------------------------------------------------------------------- /cow_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestCOWList(t *testing.T) { 12 | t.Parallel() 13 | 14 | cl := NewCOWList() 15 | cl.Append("hello") 16 | cl.Append("world") 17 | cl.Append("!") 18 | require.EqualValues(t, cl.Get(0), "hello") 19 | require.EqualValues(t, cl.Get(1), "world") 20 | require.EqualValues(t, cl.Get(2), "!") 21 | } 22 | 23 | func TestCOWListMT(t *testing.T) { 24 | t.Parallel() 25 | 26 | cl := NewCOWList() 27 | expectedRes := make([]int, 3) 28 | var wg sync.WaitGroup 29 | for i := 0; i < 3; i++ { 30 | wg.Add(1) 31 | go func(v int) { 32 | defer wg.Done() 33 | index := cl.Append(v) 34 | expectedRes[index] = v 35 | }(i) 36 | } 37 | wg.Wait() 38 | for i, v := range expectedRes { 39 | require.EqualValues(t, cl.Get(i), v) 40 | } 41 | } 42 | 43 | func BenchmarkCOWList_Get(b *testing.B) { 44 | cl := NewCOWList() 45 | for i := 0; i < 10; i++ { 46 | cl.Append(fmt.Sprintf("helloworld%d", i)) 47 | } 48 | b.ResetTimer() 49 | for i := 0; i < b.N; i++ { 50 | _ = cl.Get(i % 10).(string) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cuckoo_table.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // CuckooTableOptions are options for cuckoo table. 7 | type CuckooTableOptions struct { 8 | c *C.rocksdb_cuckoo_table_options_t 9 | } 10 | 11 | // NewCuckooTableOptions returns new cuckoo table options. 12 | func NewCuckooTableOptions() *CuckooTableOptions { 13 | return &CuckooTableOptions{ 14 | c: C.rocksdb_cuckoo_options_create(), 15 | } 16 | } 17 | 18 | // Destroy options. 19 | func (opts *CuckooTableOptions) Destroy() { 20 | C.rocksdb_cuckoo_options_destroy(opts.c) 21 | opts.c = nil 22 | } 23 | 24 | // SetHashRatio determines the utilization of hash tables. Smaller values 25 | // result in larger hash tables with fewer collisions. 26 | // 27 | // Default: 0.9. 28 | func (opts *CuckooTableOptions) SetHashRatio(value float64) { 29 | C.rocksdb_cuckoo_options_set_hash_ratio(opts.c, C.double(value)) 30 | } 31 | 32 | // SetMaxSearchDepth property used by builder to determine the depth to go to 33 | // to search for a path to displace elements in case of 34 | // collision. See Builder.MakeSpaceForKey method. Higher 35 | // values result in more efficient hash tables with fewer 36 | // lookups but take more time to build. 37 | // 38 | // Default: 100. 39 | func (opts *CuckooTableOptions) SetMaxSearchDepth(value uint32) { 40 | C.rocksdb_cuckoo_options_set_max_search_depth(opts.c, C.uint32_t(value)) 41 | } 42 | 43 | // SetCuckooBlockSize in case of collision while inserting, the builder 44 | // attempts to insert in the next cuckoo_block_size 45 | // locations before skipping over to the next Cuckoo hash 46 | // function. This makes lookups more cache friendly in case 47 | // of collisions. 48 | // 49 | // Default: 5. 50 | func (opts *CuckooTableOptions) SetCuckooBlockSize(value uint32) { 51 | C.rocksdb_cuckoo_options_set_cuckoo_block_size(opts.c, C.uint32_t(value)) 52 | } 53 | 54 | // SetIdentityAsFirstHash if this option is enabled, user key is treated as uint64_t and its value 55 | // is used as hash value directly. This option changes builder's behavior. 56 | // Reader ignore this option and behave according to what specified in table 57 | // property. 58 | // 59 | // Default: false. 60 | func (opts *CuckooTableOptions) SetIdentityAsFirstHash(value bool) { 61 | C.rocksdb_cuckoo_options_set_identity_as_first_hash(opts.c, boolToChar(value)) 62 | } 63 | 64 | // SetUseModuleHash if this option is set to true, module is used during hash calculation. 65 | // This often yields better space efficiency at the cost of performance. 66 | // If this option is set to false, # of entries in table is constrained to be 67 | // power of two, and bit and is used to calculate hash, which is faster in 68 | // general. 69 | // 70 | // Default: true 71 | func (opts *CuckooTableOptions) SetUseModuleHash(value bool) { 72 | C.rocksdb_cuckoo_options_set_use_module_hash(opts.c, boolToChar(value)) 73 | } 74 | -------------------------------------------------------------------------------- /db_external_file_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestExternalFile(t *testing.T) { 11 | t.Parallel() 12 | 13 | db := newTestDB(t, nil) 14 | defer db.Close() 15 | 16 | envOpts := NewDefaultEnvOptions() 17 | opts := NewDefaultOptions() 18 | w := NewSSTFileWriter(envOpts, opts) 19 | defer w.Destroy() 20 | 21 | filePath, err := os.CreateTemp("", "sst-file-test") 22 | require.Nil(t, err) 23 | defer os.Remove(filePath.Name()) 24 | 25 | err = w.Open(filePath.Name()) 26 | require.Nil(t, err) 27 | 28 | err = w.Add([]byte("aaa"), []byte("aaaValue")) 29 | require.Nil(t, err) 30 | err = w.Add([]byte("bbb"), []byte("bbbValue")) 31 | require.Nil(t, err) 32 | err = w.Add([]byte("ccc"), []byte("cccValue")) 33 | require.Nil(t, err) 34 | err = w.Add([]byte("ddd"), []byte("dddValue")) 35 | require.Nil(t, err) 36 | 37 | err = w.Finish() 38 | require.Nil(t, err) 39 | 40 | ingestOpts := NewDefaultIngestExternalFileOptions() 41 | err = db.IngestExternalFile([]string{filePath.Name()}, ingestOpts) 42 | require.Nil(t, err) 43 | 44 | readOpts := NewDefaultReadOptions() 45 | 46 | v1, err := db.Get(readOpts, []byte("aaa")) 47 | require.Nil(t, err) 48 | require.EqualValues(t, v1.Data(), []byte("aaaValue")) 49 | v2, err := db.Get(readOpts, []byte("bbb")) 50 | require.Nil(t, err) 51 | require.EqualValues(t, v2.Data(), []byte("bbbValue")) 52 | v3, err := db.Get(readOpts, []byte("ccc")) 53 | require.Nil(t, err) 54 | require.EqualValues(t, v3.Data(), []byte("cccValue")) 55 | v4, err := db.Get(readOpts, []byte("ddd")) 56 | require.Nil(t, err) 57 | require.EqualValues(t, v4.Data(), []byte("dddValue")) 58 | } 59 | 60 | func TestExternalFileWithTS(t *testing.T) { 61 | t.Parallel() 62 | 63 | db := newTestDB(t, func(opts *Options) { 64 | opts.SetComparator(newDefaultComparatorWithTS()) 65 | }) 66 | defer db.Close() 67 | 68 | envOpts := NewDefaultEnvOptions() 69 | opts := NewDefaultOptions() 70 | opts.SetComparator(newDefaultComparatorWithTS()) 71 | w := NewSSTFileWriter(envOpts, opts) 72 | defer w.Destroy() 73 | 74 | filePath, err := os.CreateTemp("", "sst-file-ts-test") 75 | require.Nil(t, err) 76 | defer os.Remove(filePath.Name()) 77 | 78 | err = w.Open(filePath.Name()) 79 | require.Nil(t, err) 80 | 81 | err = w.PutWithTS([]byte("aaa"), marshalTimestamp(1), []byte("aaaValue")) 82 | require.Nil(t, err) 83 | err = w.PutWithTS([]byte("bbb"), marshalTimestamp(2), []byte("bbbValue")) 84 | require.Nil(t, err) 85 | err = w.PutWithTS([]byte("ccc"), marshalTimestamp(3), []byte("cccValue")) 86 | require.Nil(t, err) 87 | err = w.PutWithTS([]byte("ddd"), marshalTimestamp(4), []byte("dddValue")) 88 | require.Nil(t, err) 89 | 90 | err = w.Finish() 91 | require.Nil(t, err) 92 | 93 | ingestOpts := NewDefaultIngestExternalFileOptions() 94 | err = db.IngestExternalFile([]string{filePath.Name()}, ingestOpts) 95 | require.Nil(t, err) 96 | 97 | readOpts := NewDefaultReadOptions() 98 | readOpts.SetTimestamp(marshalTimestamp(5)) 99 | 100 | v1, t1, err := db.GetWithTS(readOpts, []byte("aaa")) 101 | require.Nil(t, err) 102 | require.EqualValues(t, v1.Data(), []byte("aaaValue")) 103 | require.EqualValues(t, t1.Data(), marshalTimestamp(1)) 104 | 105 | v2, t2, err := db.GetWithTS(readOpts, []byte("bbb")) 106 | require.Nil(t, err) 107 | require.EqualValues(t, v2.Data(), []byte("bbbValue")) 108 | require.EqualValues(t, t2.Data(), marshalTimestamp(2)) 109 | 110 | v3, t3, err := db.GetWithTS(readOpts, []byte("ccc")) 111 | require.Nil(t, err) 112 | require.EqualValues(t, v3.Data(), []byte("cccValue")) 113 | require.EqualValues(t, t3.Data(), marshalTimestamp(3)) 114 | 115 | v4, t4, err := db.GetWithTS(readOpts, []byte("ddd")) 116 | require.Nil(t, err) 117 | require.EqualValues(t, v4.Data(), []byte("dddValue")) 118 | require.EqualValues(t, t4.Data(), marshalTimestamp(4)) 119 | } 120 | -------------------------------------------------------------------------------- /db_ts_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestDBCRUDWithTS(t *testing.T) { 12 | t.Parallel() 13 | 14 | db := newTestDB(t, func(opts *Options) { 15 | opts.SetComparator(newDefaultComparatorWithTS()) 16 | }) 17 | defer db.Close() 18 | 19 | var ( 20 | givenKey = []byte("hello") 21 | givenVal1 = []byte("") 22 | givenVal2 = []byte("world1") 23 | 24 | givenTs1 = marshalTimestamp(1) 25 | givenTs2 = marshalTimestamp(2) 26 | givenTs3 = marshalTimestamp(3) 27 | ) 28 | 29 | wo := NewDefaultWriteOptions() 30 | ro := NewDefaultReadOptions() 31 | ro.SetTimestamp(givenTs1) 32 | 33 | // create 34 | require.Nil(t, db.PutWithTS(wo, givenKey, givenTs1, givenVal1)) 35 | 36 | // retrieve 37 | v1, t1, err := db.GetWithTS(ro, givenKey) 38 | require.Nil(t, err) 39 | require.EqualValues(t, v1.Data(), givenVal1) 40 | require.EqualValues(t, t1.Data(), givenTs1) 41 | v1.Free() 42 | t1.Free() 43 | 44 | // retrieve bytes 45 | _v1, _ts1, err := db.GetBytesWithTS(ro, givenKey) 46 | require.Nil(t, err) 47 | require.EqualValues(t, _v1, givenVal1) 48 | require.EqualValues(t, _ts1, givenTs1) 49 | 50 | // update 51 | require.Nil(t, db.PutWithTS(wo, givenKey, givenTs2, givenVal2)) 52 | ro.SetTimestamp(givenTs2) 53 | v2, t2, err := db.GetWithTS(ro, givenKey) 54 | require.Nil(t, err) 55 | require.EqualValues(t, v2.Data(), givenVal2) 56 | require.EqualValues(t, t2.Data(), givenTs2) 57 | v2.Free() 58 | t2.Free() 59 | 60 | // delete 61 | require.Nil(t, db.DeleteWithTS(wo, givenKey, givenTs3)) 62 | ro.SetTimestamp(givenTs3) 63 | v3, t3, err := db.GetWithTS(ro, givenKey) 64 | require.Nil(t, err) 65 | require.True(t, v3.Data() == nil) 66 | require.True(t, t3.Data() == nil) 67 | v3.Free() 68 | t3.Free() 69 | 70 | // ts2 should read deleted data 71 | ro.SetTimestamp(givenTs2) 72 | v2, t2, err = db.GetWithTS(ro, givenKey) 73 | require.Nil(t, err) 74 | require.EqualValues(t, v2.Data(), givenVal2) 75 | require.EqualValues(t, t2.Data(), givenTs2) 76 | v2.Free() 77 | t2.Free() 78 | 79 | // ts1 should read old data 80 | ro.SetTimestamp(givenTs1) 81 | v1, t1, err = db.GetWithTS(ro, givenKey) 82 | require.Nil(t, err) 83 | require.EqualValues(t, v1.Data(), givenVal1) 84 | require.EqualValues(t, t1.Data(), givenTs1) 85 | v1.Free() 86 | t1.Free() 87 | } 88 | 89 | func TestDBMultiGetWithTS(t *testing.T) { 90 | t.Parallel() 91 | 92 | db := newTestDB(t, func(opts *Options) { 93 | opts.SetComparator(newDefaultComparatorWithTS()) 94 | }) 95 | defer db.Close() 96 | 97 | var ( 98 | givenKey1 = []byte("hello1") 99 | givenKey2 = []byte("hello2") 100 | givenKey3 = []byte("hello3") 101 | givenVal1 = []byte("world1") 102 | givenVal2 = []byte("world2") 103 | givenVal3 = []byte("world3") 104 | 105 | givenTs1 = marshalTimestamp(1) 106 | givenTs2 = marshalTimestamp(2) 107 | givenTs3 = marshalTimestamp(3) 108 | ) 109 | 110 | wo := NewDefaultWriteOptions() 111 | ro := NewDefaultReadOptions() 112 | ro.SetTimestamp(marshalTimestamp(4)) 113 | 114 | // create 115 | require.Nil(t, db.PutWithTS(wo, givenKey1, givenTs1, givenVal1)) 116 | require.Nil(t, db.PutWithTS(wo, givenKey2, givenTs2, givenVal2)) 117 | require.Nil(t, db.PutWithTS(wo, givenKey3, givenTs3, givenVal3)) 118 | 119 | // retrieve 120 | values, times, err := db.MultiGetWithTS(ro, []byte("noexist"), givenKey1, givenKey2, givenKey3) 121 | defer values.Destroy() 122 | defer times.Destroy() 123 | require.Nil(t, err) 124 | require.EqualValues(t, len(values), 4) 125 | 126 | require.EqualValues(t, values[0].Data(), []byte(nil)) 127 | require.EqualValues(t, values[1].Data(), givenVal1) 128 | require.EqualValues(t, values[2].Data(), givenVal2) 129 | require.EqualValues(t, values[3].Data(), givenVal3) 130 | 131 | require.EqualValues(t, times[0].Data(), []byte(nil)) 132 | require.EqualValues(t, times[1].Data(), givenTs1) 133 | require.EqualValues(t, times[2].Data(), givenTs2) 134 | require.EqualValues(t, times[3].Data(), givenTs3) 135 | } 136 | 137 | const timestampSize = 8 138 | 139 | func marshalTimestamp(ts uint64) []byte { 140 | b := make([]byte, timestampSize) 141 | binary.BigEndian.PutUint64(b, ts) 142 | return b 143 | } 144 | 145 | func marshalTimestampLittleEndian(ts uint64) []byte { 146 | b := make([]byte, timestampSize) 147 | binary.LittleEndian.PutUint64(b, ts) 148 | return b 149 | } 150 | 151 | func extractUserKey(key []byte) []byte { 152 | return key[:len(key)-timestampSize] 153 | } 154 | 155 | func extractUserTimestamp(key []byte) []byte { 156 | return key[len(key)-timestampSize:] 157 | } 158 | 159 | func extractFromInteralKey(internalKey []byte) (key, timestamp []byte) { 160 | internalKeySize := 8 // rocksdb internal key size 161 | userKey := internalKey[:len(internalKey)-internalKeySize] 162 | key = extractUserKey(userKey) 163 | timestamp = extractUserTimestamp(userKey) 164 | return 165 | } 166 | 167 | func newDefaultComparatorWithTS() *Comparator { 168 | return newComparatorWithTimeStamp("default", func(a, b []byte) int { 169 | return bytes.Compare(a, b) 170 | }, binary.BigEndian.Uint64) 171 | } 172 | 173 | func newLittleEndianComparatorWithTS() *Comparator { 174 | return newComparatorWithTimeStamp("default", func(a, b []byte) int { 175 | return bytes.Compare(a, b) 176 | }, binary.LittleEndian.Uint64) 177 | } 178 | 179 | func newComparatorWithTimeStamp(name string, userCompare Comparing, encodeFn func([]byte) uint64) *Comparator { 180 | compTS := func(a, b []byte) int { 181 | aTs := encodeFn(a) 182 | bTs := encodeFn(b) 183 | if aTs < bTs { 184 | return -1 185 | } 186 | if aTs > bTs { 187 | return 1 188 | } 189 | return 0 190 | } 191 | 192 | comp := func(a, b []byte) int { 193 | aKey := extractUserKey(a) 194 | bKey := extractUserKey(b) 195 | res := userCompare(aKey, bKey) 196 | if res != 0 { 197 | return res 198 | } 199 | 200 | aTs := extractUserTimestamp(a) 201 | bTs := extractUserTimestamp(b) 202 | return compTS(aTs, bTs) * -1 // timestamp should be reverse ordered 203 | } 204 | 205 | compWithoutTS := func(a []byte, aHasTs bool, b []byte, bHasTs bool) int { 206 | var aWithOutTs []byte 207 | if aHasTs { 208 | aWithOutTs = extractUserKey(a) 209 | } else { 210 | aWithOutTs = a 211 | } 212 | 213 | var bWithOutTs []byte 214 | if bHasTs { 215 | bWithOutTs = extractUserKey(b) 216 | } else { 217 | bWithOutTs = b 218 | } 219 | return userCompare(aWithOutTs, bWithOutTs) 220 | } 221 | 222 | return NewComparatorWithTimestamp(name, timestampSize, comp, compTS, compWithoutTS) 223 | } 224 | -------------------------------------------------------------------------------- /dbpath.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // DBPath represents options for a dbpath. 9 | type DBPath struct { 10 | c *C.rocksdb_dbpath_t 11 | } 12 | 13 | // NewDBPath creates a DBPath object 14 | // with the given path and target_size. 15 | func NewDBPath(path string, targetSize uint64) (dbPath *DBPath) { 16 | cpath := C.CString(path) 17 | cDBPath := C.rocksdb_dbpath_create(cpath, C.uint64_t(targetSize)) 18 | dbPath = newNativeDBPath(cDBPath) 19 | C.free(unsafe.Pointer(cpath)) 20 | return 21 | } 22 | 23 | // NewNativeDBPath creates a DBPath object. 24 | func newNativeDBPath(c *C.rocksdb_dbpath_t) *DBPath { 25 | return &DBPath{c: c} 26 | } 27 | 28 | // Destroy deallocates the DBPath object. 29 | func (dbpath *DBPath) Destroy() { 30 | C.rocksdb_dbpath_destroy(dbpath.c) 31 | dbpath.c = nil 32 | } 33 | 34 | // NewDBPathsFromData creates a slice with allocated DBPath objects 35 | // from paths and target_sizes. 36 | func NewDBPathsFromData(paths []string, targetSizes []uint64) []*DBPath { 37 | dbpaths := make([]*DBPath, len(paths)) 38 | for i, path := range paths { 39 | targetSize := targetSizes[i] 40 | dbpaths[i] = NewDBPath(path, targetSize) 41 | } 42 | 43 | return dbpaths 44 | } 45 | 46 | // DestroyDBPaths deallocates all DBPath objects in dbpaths. 47 | func DestroyDBPaths(dbpaths []*DBPath) { 48 | for _, dbpath := range dbpaths { 49 | dbpath.Destroy() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package grocksdb provides the ability to create and access RocksDB databases. 3 | 4 | grocksdb.OpenDb opens and creates databases. 5 | 6 | bbto := grocksdb.NewDefaultBlockBasedTableOptions() 7 | bbto.SetBlockCache(grocksdb.NewLRUCache(3 << 30)) 8 | 9 | opts := grocksdb.NewDefaultOptions() 10 | opts.SetBlockBasedTableFactory(bbto) 11 | opts.SetCreateIfMissing(true) 12 | 13 | db, err := grocksdb.OpenDb(opts, "/path/to/db") 14 | 15 | The DB struct returned by OpenDb provides DB.Get, DB.Put, DB.Merge and DB.Delete to modify 16 | and query the database. 17 | 18 | ro := grocksdb.NewDefaultReadOptions() 19 | wo := grocksdb.NewDefaultWriteOptions() 20 | 21 | // if ro and wo are not used again, be sure to Close them. 22 | err = db.Put(wo, []byte("foo"), []byte("bar")) 23 | ... 24 | value, err := db.Get(ro, []byte("foo")) 25 | defer value.Free() 26 | ... 27 | err = db.Delete(wo, []byte("foo")) 28 | 29 | For bulk reads, use an Iterator. If you want to avoid disturbing your live 30 | traffic while doing the bulk read, be sure to call SetFillCache(false) on the 31 | ReadOptions you use when creating the Iterator. 32 | 33 | ro := grocksdb.NewDefaultReadOptions() 34 | ro.SetFillCache(false) 35 | 36 | it := db.NewIterator(ro) 37 | defer it.Close() 38 | 39 | it.Seek([]byte("foo")) 40 | for it = it; it.Valid(); it.Next() { 41 | key := it.Key() 42 | value := it.Value() 43 | fmt.Printf("Key: %v Value: %v\n", key.Data(), value.Data()) 44 | key.Free() 45 | value.Free() 46 | } 47 | if err := it.Err(); err != nil { 48 | ... 49 | } 50 | 51 | Batched, atomic writes can be performed with a WriteBatch and 52 | DB.Write. 53 | 54 | wb := grocksdb.NewWriteBatch() 55 | // defer wb.Close or use wb.Clear and reuse. 56 | wb.Delete([]byte("foo")) 57 | 58 | wb.Put([]byte("foo"), []byte("bar")) 59 | wb.Put([]byte("bar"), []byte("foo")) 60 | 61 | err := db.Write(wo, wb) 62 | 63 | If your working dataset does not fit in memory, you'll want to add a bloom 64 | filter to your database. NewBloomFilter and 65 | BlockBasedTableOptions.SetFilterPolicy is what you want. NewBloomFilter is 66 | amount of bits in the filter to use per key in your database. 67 | 68 | filter := grocksdb.NewBloomFilter(10) 69 | bbto := grocksdb.NewDefaultBlockBasedTableOptions() 70 | bbto.SetFilterPolicy(filter) 71 | opts.SetBlockBasedTableFactory(bbto) 72 | db, err := grocksdb.OpenDb(opts, "/path/to/db") 73 | 74 | If you're using a custom comparator in your code, be aware you may have to 75 | make your own filter policy object. 76 | 77 | This documentation is not a complete discussion of RocksDB. Please read the 78 | RocksDB documentation for information on its 79 | operation. You'll find lots of goodies there. 80 | */ 81 | package grocksdb 82 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // Env is a system call environment used by a database. 7 | type Env struct { 8 | c *C.rocksdb_env_t 9 | } 10 | 11 | // NewDefaultEnv creates a default environment. 12 | func NewDefaultEnv() *Env { 13 | return newNativeEnv(C.rocksdb_create_default_env()) 14 | } 15 | 16 | // NewMemEnv returns a new environment that stores its data in memory and delegates 17 | // all non-file-storage tasks to base_env. 18 | func NewMemEnv() *Env { 19 | return newNativeEnv(C.rocksdb_create_mem_env()) 20 | } 21 | 22 | // NewNativeEnv creates a Environment object. 23 | func newNativeEnv(c *C.rocksdb_env_t) *Env { 24 | return &Env{c: c} 25 | } 26 | 27 | // SetBackgroundThreads sets the number of background worker threads 28 | // of a specific thread pool for this environment. 29 | // 'LOW' is the default pool. 30 | // 31 | // Default: 1 32 | func (env *Env) SetBackgroundThreads(n int) { 33 | C.rocksdb_env_set_background_threads(env.c, C.int(n)) 34 | } 35 | 36 | // GetBackgroundThreads sets the number of background worker threads 37 | // of a specific thread pool for this environment. 38 | // 'LOW' is the default pool. 39 | func (env *Env) GetBackgroundThreads() int { 40 | return int(C.rocksdb_env_get_background_threads(env.c)) 41 | } 42 | 43 | // SetHighPriorityBackgroundThreads sets the size of the high priority 44 | // thread pool that can be used to prevent compactions from stalling 45 | // memtable flushes. 46 | func (env *Env) SetHighPriorityBackgroundThreads(n int) { 47 | C.rocksdb_env_set_high_priority_background_threads(env.c, C.int(n)) 48 | } 49 | 50 | // GetHighPriorityBackgroundThreads gets the size of the high priority 51 | // thread pool that can be used to prevent compactions from stalling 52 | // memtable flushes. 53 | func (env *Env) GetHighPriorityBackgroundThreads() int { 54 | return int(C.rocksdb_env_get_high_priority_background_threads(env.c)) 55 | } 56 | 57 | // SetLowPriorityBackgroundThreads sets the size of the low priority 58 | // thread pool that can be used to prevent compactions from stalling 59 | // memtable flushes. 60 | func (env *Env) SetLowPriorityBackgroundThreads(n int) { 61 | C.rocksdb_env_set_low_priority_background_threads(env.c, C.int(n)) 62 | } 63 | 64 | // GetLowPriorityBackgroundThreads gets the size of the low priority 65 | // thread pool that can be used to prevent compactions from stalling 66 | // memtable flushes. 67 | func (env *Env) GetLowPriorityBackgroundThreads() int { 68 | return int(C.rocksdb_env_get_low_priority_background_threads(env.c)) 69 | } 70 | 71 | // SetBottomPriorityBackgroundThreads sets the size of 72 | // thread pool that can be used to prevent bottommost compactions 73 | // from stalling memtable flushes. 74 | func (env *Env) SetBottomPriorityBackgroundThreads(n int) { 75 | C.rocksdb_env_set_bottom_priority_background_threads(env.c, C.int(n)) 76 | } 77 | 78 | // GetBottomPriorityBackgroundThreads gets the size of 79 | // thread pool that can be used to prevent bottommost compactions 80 | // from stalling memtable flushes. 81 | func (env *Env) GetBottomPriorityBackgroundThreads() int { 82 | return int(C.rocksdb_env_get_bottom_priority_background_threads(env.c)) 83 | } 84 | 85 | // JoinAllThreads wait for all threads started by StartThread to terminate. 86 | func (env *Env) JoinAllThreads() { 87 | C.rocksdb_env_join_all_threads(env.c) 88 | } 89 | 90 | // LowerThreadPoolIOPriority lower IO priority for threads from the specified pool. 91 | func (env *Env) LowerThreadPoolIOPriority() { 92 | C.rocksdb_env_lower_thread_pool_io_priority(env.c) 93 | } 94 | 95 | // LowerHighPriorityThreadPoolIOPriority lower IO priority for high priority 96 | // thread pool. 97 | func (env *Env) LowerHighPriorityThreadPoolIOPriority() { 98 | C.rocksdb_env_lower_high_priority_thread_pool_io_priority(env.c) 99 | } 100 | 101 | // LowerThreadPoolCPUPriority lower CPU priority for threads from the specified pool. 102 | func (env *Env) LowerThreadPoolCPUPriority() { 103 | C.rocksdb_env_lower_thread_pool_cpu_priority(env.c) 104 | } 105 | 106 | // LowerHighPriorityThreadPoolCPUPriority lower CPU priority for high priority 107 | // thread pool. 108 | func (env *Env) LowerHighPriorityThreadPoolCPUPriority() { 109 | C.rocksdb_env_lower_high_priority_thread_pool_cpu_priority(env.c) 110 | } 111 | 112 | // Destroy deallocates the Env object. 113 | func (env *Env) Destroy() { 114 | C.rocksdb_env_destroy(env.c) 115 | env.c = nil 116 | } 117 | -------------------------------------------------------------------------------- /env_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestEnv(t *testing.T) { 10 | t.Parallel() 11 | 12 | env := NewDefaultEnv() 13 | defer env.Destroy() 14 | 15 | env.SetBackgroundThreads(2) 16 | require.Equal(t, 2, env.GetBackgroundThreads()) 17 | 18 | env.SetHighPriorityBackgroundThreads(5) 19 | require.Equal(t, 5, env.GetHighPriorityBackgroundThreads()) 20 | 21 | env.SetLowPriorityBackgroundThreads(6) 22 | require.Equal(t, 6, env.GetLowPriorityBackgroundThreads()) 23 | 24 | env.SetBottomPriorityBackgroundThreads(14) 25 | require.Equal(t, 14, env.GetBottomPriorityBackgroundThreads()) 26 | 27 | env.JoinAllThreads() 28 | env.LowerHighPriorityThreadPoolCPUPriority() 29 | env.LowerHighPriorityThreadPoolIOPriority() 30 | env.LowerThreadPoolCPUPriority() 31 | env.LowerThreadPoolIOPriority() 32 | } 33 | -------------------------------------------------------------------------------- /filter_policy.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // NativeFilterPolicy wraps over rocksdb filter policy. 7 | type NativeFilterPolicy struct { 8 | c *C.rocksdb_filterpolicy_t 9 | } 10 | 11 | func (fp *NativeFilterPolicy) Destroy() { 12 | C.rocksdb_filterpolicy_destroy(fp.c) 13 | fp.c = nil 14 | } 15 | 16 | // creates a FilterPolicy object. 17 | func newNativeFilterPolicy(c *C.rocksdb_filterpolicy_t) *NativeFilterPolicy { 18 | return &NativeFilterPolicy{c: c} 19 | } 20 | 21 | // NewBloomFilter returns a new filter policy that uses a bloom filter with approximately 22 | // the specified number of bits per key. A good value for bits_per_key 23 | // is 10, which yields a filter with ~1% false positive rate. 24 | // 25 | // Note: if you are using a custom comparator that ignores some parts 26 | // of the keys being compared, you must not use NewBloomFilterPolicy() 27 | // and must provide your own FilterPolicy that also ignores the 28 | // corresponding parts of the keys. For example, if the comparator 29 | // ignores trailing spaces, it would be incorrect to use a 30 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 31 | // trailing spaces in keys. 32 | func NewBloomFilter(bitsPerKey float64) *NativeFilterPolicy { 33 | cFilter := C.rocksdb_filterpolicy_create_bloom(C.double(bitsPerKey)) 34 | return newNativeFilterPolicy(cFilter) 35 | } 36 | 37 | // NewBloomFilterFull returns a new filter policy that uses a full bloom filter 38 | // with approximately the specified number of bits per key. A good value for 39 | // bits_per_key is 10, which yields a filter with ~1% false positive rate. 40 | // 41 | // Note: if you are using a custom comparator that ignores some parts 42 | // of the keys being compared, you must not use NewBloomFilterPolicy() 43 | // and must provide your own FilterPolicy that also ignores the 44 | // corresponding parts of the keys. For example, if the comparator 45 | // ignores trailing spaces, it would be incorrect to use a 46 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 47 | // trailing spaces in keys. 48 | func NewBloomFilterFull(bitsPerKey float64) *NativeFilterPolicy { 49 | cFilter := C.rocksdb_filterpolicy_create_bloom_full(C.double(bitsPerKey)) 50 | return newNativeFilterPolicy(cFilter) 51 | } 52 | 53 | // NewRibbonFilterPolicy creates a new Bloom alternative that saves about 54 | // 30% space compared to Bloom filters, with similar query times but 55 | // roughly 3-4x CPU time and 3x temporary space usage during construction. 56 | // 57 | // For example: 58 | // if you pass in 10 for bloom_equivalent_bits_per_key, you'll get the same 59 | // 0.95% FP rate as Bloom filter but only using about 7 bits per key. 60 | // 61 | // The space savings of Ribbon filters makes sense for lower (higher 62 | // numbered; larger; longer-lived) levels of LSM, whereas the speed of 63 | // Bloom filters make sense for highest levels of LSM. 64 | // 65 | // Ribbon filters are compatible with RocksDB >= 6.15.0. Earlier 66 | // versions reading the data will behave as if no filter was used 67 | // (degraded performance until compaction rebuilds filters). All 68 | // built-in FilterPolicies (Bloom or Ribbon) are able to read other 69 | // kinds of built-in filters. 70 | // 71 | // Note: the current Ribbon filter schema uses some extra resources 72 | // when constructing very large filters. For example, for 100 million 73 | // keys in a single filter (one SST file without partitioned filters), 74 | // 3GB of temporary, untracked memory is used, vs. 1GB for Bloom. 75 | // However, the savings in filter space from just ~60 open SST files 76 | // makes up for the additional temporary memory use. 77 | // 78 | // Also consider using optimize_filters_for_memory to save filter 79 | // memory. 80 | func NewRibbonFilterPolicy(bloomEquivalentBitsPerKey float64) *NativeFilterPolicy { 81 | cFilter := C.rocksdb_filterpolicy_create_ribbon(C.double(bloomEquivalentBitsPerKey)) 82 | return newNativeFilterPolicy(cFilter) 83 | } 84 | 85 | // NewRibbonHybridFilterPolicy similar to Ribbon. 86 | // 87 | // Setting bloom_before_level allows for this design with Level and Universal 88 | // compaction styles. For example, bloom_before_level=1 means that Bloom 89 | // filters will be used in level 0, including flushes, and Ribbon 90 | // filters elsewhere, including FIFO compaction and external SST files. 91 | // For this option, memtable flushes are considered level -1 (so that 92 | // flushes can be distinguished from intra-L0 compaction). 93 | // bloom_before_level=0 (default) -> Generate Bloom filters only for 94 | // flushes under Level and Universal compaction styles. 95 | // bloom_before_level=-1 -> Always generate Ribbon filters (except in 96 | // some extreme or exceptional cases). 97 | func NewRibbonHybridFilterPolicy(bloomEquivalentBitsPerKey float64, bloomBeforeLevel int) *NativeFilterPolicy { 98 | cFilter := C.rocksdb_filterpolicy_create_ribbon_hybrid(C.double(bloomEquivalentBitsPerKey), C.int(bloomBeforeLevel)) 99 | return newNativeFilterPolicy(cFilter) 100 | } 101 | -------------------------------------------------------------------------------- /filter_policy_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFilterPolicy(t *testing.T) { 8 | t.Parallel() 9 | 10 | t.Run("Bloom", func(t *testing.T) { 11 | t.Parallel() 12 | 13 | flt := NewBloomFilter(1.2) 14 | defer flt.Destroy() 15 | }) 16 | 17 | t.Run("BloomFull", func(t *testing.T) { 18 | t.Parallel() 19 | 20 | flt := NewBloomFilterFull(1.2) 21 | defer flt.Destroy() 22 | }) 23 | 24 | t.Run("Ribbon", func(t *testing.T) { 25 | t.Parallel() 26 | 27 | flt := NewRibbonFilterPolicy(1.2) 28 | defer flt.Destroy() 29 | }) 30 | 31 | t.Run("RibbonHybrid", func(t *testing.T) { 32 | t.Parallel() 33 | 34 | flt := NewRibbonHybridFilterPolicy(1.2, 1) 35 | defer flt.Destroy() 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/linxGnu/grocksdb 2 | 3 | go 1.17 4 | 5 | require github.com/stretchr/testify v1.10.0 6 | 7 | require ( 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 9 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 10 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 11 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 12 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 13 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 14 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 15 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 19 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | -------------------------------------------------------------------------------- /grocksdb.c: -------------------------------------------------------------------------------- 1 | #include "grocksdb.h" 2 | #include "_cgo_export.h" 3 | 4 | /* Base */ 5 | 6 | void gorocksdb_destruct_handler(void* state) { } 7 | 8 | /* Comparator */ 9 | 10 | rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx) { 11 | return rocksdb_comparator_create( 12 | (void*)idx, 13 | gorocksdb_destruct_handler, 14 | (int (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_comparator_compare), 15 | (const char *(*)(void*))(gorocksdb_comparator_name)); 16 | } 17 | 18 | rocksdb_comparator_t* gorocksdb_comparator_with_ts_create(uintptr_t idx, size_t ts_size) { 19 | return rocksdb_comparator_with_ts_create( 20 | (void*)idx, 21 | gorocksdb_destruct_handler, 22 | (int (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_comparator_compare), 23 | (int (*)(void*, const char*, size_t, const char*, size_t))(gorocksdb_comparator_compare_ts), 24 | (int (*)(void*, const char*, size_t, unsigned char, const char*, size_t, unsigned char))(gorocksdb_comparator_compare_without_ts), 25 | (const char* (*)(void*))(gorocksdb_comparator_name), 26 | ts_size); 27 | } 28 | 29 | /* CompactionFilter */ 30 | 31 | rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx) { 32 | return rocksdb_compactionfilter_create( 33 | (void*)idx, 34 | gorocksdb_destruct_handler, 35 | (unsigned char (*)(void*, int, const char*, size_t, const char*, size_t, char**, size_t*, unsigned char*))(gorocksdb_compactionfilter_filter), 36 | (const char *(*)(void*))(gorocksdb_compactionfilter_name)); 37 | } 38 | 39 | /* Merge Operator */ 40 | 41 | rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx) { 42 | return rocksdb_mergeoperator_create( 43 | (void*)idx, 44 | gorocksdb_destruct_handler, 45 | (char* (*)(void*, const char*, size_t, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_full_merge), 46 | (char* (*)(void*, const char*, size_t, const char* const*, const size_t*, int, unsigned char*, size_t*))(gorocksdb_mergeoperator_partial_merge_multi), 47 | gorocksdb_mergeoperator_delete_value, 48 | (const char* (*)(void*))(gorocksdb_mergeoperator_name)); 49 | } 50 | 51 | void gorocksdb_mergeoperator_delete_value(void* id, const char* v, size_t s) { 52 | free((char*)v); 53 | } 54 | 55 | /* Slice Transform */ 56 | 57 | rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx) { 58 | return rocksdb_slicetransform_create( 59 | (void*)idx, 60 | gorocksdb_destruct_handler, 61 | (char* (*)(void*, const char*, size_t, size_t*))(gorocksdb_slicetransform_transform), 62 | (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_domain), 63 | (unsigned char (*)(void*, const char*, size_t))(gorocksdb_slicetransform_in_range), 64 | (const char* (*)(void*))(gorocksdb_slicetransform_name)); 65 | } 66 | -------------------------------------------------------------------------------- /grocksdb.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "rocksdb/c.h" 3 | 4 | // This API provides convenient C wrapper functions for rocksdb client. 5 | 6 | /* Base */ 7 | 8 | extern void gorocksdb_destruct_handler(void* state); 9 | 10 | /* CompactionFilter */ 11 | 12 | extern rocksdb_compactionfilter_t* gorocksdb_compactionfilter_create(uintptr_t idx); 13 | 14 | /* Comparator */ 15 | 16 | extern rocksdb_comparator_t* gorocksdb_comparator_create(uintptr_t idx); 17 | 18 | extern rocksdb_comparator_t* gorocksdb_comparator_with_ts_create(uintptr_t idx, size_t ts_size); 19 | 20 | /* Merge Operator */ 21 | 22 | extern rocksdb_mergeoperator_t* gorocksdb_mergeoperator_create(uintptr_t idx); 23 | extern void gorocksdb_mergeoperator_delete_value(void* state, const char* v, size_t s); 24 | 25 | /* Slice Transform */ 26 | 27 | extern rocksdb_slicetransform_t* gorocksdb_slicetransform_create(uintptr_t idx); 28 | -------------------------------------------------------------------------------- /iterator.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "bytes" 9 | ) 10 | 11 | // Iterator provides a way to seek to specific keys and iterate through 12 | // the keyspace from that point, as well as access the values of those keys. 13 | // 14 | // For example: 15 | // 16 | // it := db.NewIterator(readOpts) 17 | // defer it.Close() 18 | // 19 | // it.Seek([]byte("foo")) 20 | // for ; it.Valid(); it.Next() { 21 | // fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data()) 22 | // } 23 | // 24 | // if err := it.Err(); err != nil { 25 | // return err 26 | // } 27 | type Iterator struct { 28 | c *C.rocksdb_iterator_t 29 | opts *ReadOptions 30 | } 31 | 32 | // NewNativeIterator creates a Iterator object. 33 | func newNativeIterator(c *C.rocksdb_iterator_t) *Iterator { 34 | return &Iterator{c: c} 35 | } 36 | 37 | // newNativeIteratorWithReadOptions creates a Iterator object with ReadOptions. 38 | func newNativeIteratorWithReadOptions(c *C.rocksdb_iterator_t, opts *ReadOptions) *Iterator { 39 | return &Iterator{ 40 | c: c, 41 | opts: opts, 42 | } 43 | } 44 | 45 | // Valid returns false only when an Iterator has iterated past either the 46 | // first or the last key in the database. 47 | func (iter *Iterator) Valid() bool { 48 | return C.rocksdb_iter_valid(iter.c) != 0 49 | } 50 | 51 | // ValidForPrefix returns false only when an Iterator has iterated past the 52 | // first or the last key in the database or the specified prefix. 53 | func (iter *Iterator) ValidForPrefix(prefix []byte) bool { 54 | if C.rocksdb_iter_valid(iter.c) == 0 { 55 | return false 56 | } 57 | 58 | key := iter.Key() 59 | result := bytes.HasPrefix(key.Data(), prefix) 60 | key.Free() 61 | return result 62 | } 63 | 64 | // Key returns the key the iterator currently holds. 65 | func (iter *Iterator) Key() *Slice { 66 | var cLen C.size_t 67 | cKey := C.rocksdb_iter_key(iter.c, &cLen) 68 | if cKey == nil { 69 | return nil 70 | } 71 | return &Slice{data: cKey, size: cLen, freed: true} 72 | } 73 | 74 | // Timestamp returns the timestamp in the database the iterator currently holds. 75 | func (iter *Iterator) Timestamp() *Slice { 76 | var cLen C.size_t 77 | cTs := C.rocksdb_iter_timestamp(iter.c, &cLen) 78 | if cTs == nil { 79 | return nil 80 | } 81 | return &Slice{data: cTs, size: cLen, freed: true} 82 | } 83 | 84 | // Value returns the value in the database the iterator currently holds. 85 | func (iter *Iterator) Value() *Slice { 86 | var cLen C.size_t 87 | cVal := C.rocksdb_iter_value(iter.c, &cLen) 88 | if cVal == nil { 89 | return nil 90 | } 91 | return &Slice{data: cVal, size: cLen, freed: true} 92 | } 93 | 94 | // Next moves the iterator to the next sequential key in the database. 95 | func (iter *Iterator) Next() { 96 | C.rocksdb_iter_next(iter.c) 97 | } 98 | 99 | // Prev moves the iterator to the previous sequential key in the database. 100 | func (iter *Iterator) Prev() { 101 | C.rocksdb_iter_prev(iter.c) 102 | } 103 | 104 | // SeekToFirst moves the iterator to the first key in the database. 105 | func (iter *Iterator) SeekToFirst() { 106 | C.rocksdb_iter_seek_to_first(iter.c) 107 | } 108 | 109 | // SeekToLast moves the iterator to the last key in the database. 110 | func (iter *Iterator) SeekToLast() { 111 | C.rocksdb_iter_seek_to_last(iter.c) 112 | } 113 | 114 | // Seek moves the iterator to the position greater than or equal to the key. 115 | func (iter *Iterator) Seek(key []byte) { 116 | cKey := refGoBytes(key) 117 | C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key))) 118 | } 119 | 120 | // SeekForPrev moves the iterator to the last key that less than or equal 121 | // to the target key, in contrast with Seek. 122 | func (iter *Iterator) SeekForPrev(key []byte) { 123 | cKey := refGoBytes(key) 124 | C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key))) 125 | } 126 | 127 | // Err returns nil if no errors happened during iteration, or the actual 128 | // error otherwise. 129 | func (iter *Iterator) Err() (err error) { 130 | var cErr *C.char 131 | C.rocksdb_iter_get_error(iter.c, &cErr) 132 | err = fromCError(cErr) 133 | return 134 | } 135 | 136 | // Refresh if supported, the DB state that the iterator reads from is updated to 137 | // the latest state. The iterator will be invalidated after the call. 138 | // Regardless of whether the iterator was created/refreshed previously 139 | // with or without a snapshot, the iterator will be reading the 140 | // latest DB state after this call. 141 | // Note that you will need to call a Seek*() function to get the iterator 142 | // back into a valid state before calling a function that assumes the 143 | // state is already valid, like Next(). 144 | func (iter *Iterator) Refresh() (err error) { 145 | var cErr *C.char 146 | C.rocksdb_iter_refresh(iter.c, &cErr) 147 | err = fromCError(cErr) 148 | return 149 | } 150 | 151 | // Close closes the iterator. 152 | func (iter *Iterator) Close() { 153 | C.rocksdb_iter_destroy(iter.c) 154 | iter.c = nil 155 | } 156 | -------------------------------------------------------------------------------- /iterator_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestIterator(t *testing.T) { 11 | t.Parallel() 12 | 13 | db := newTestDB(t, nil) 14 | defer db.Close() 15 | 16 | // insert keys 17 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 18 | wo := NewDefaultWriteOptions() 19 | for _, k := range givenKeys { 20 | require.Nil(t, db.Put(wo, k, []byte("val"))) 21 | } 22 | 23 | ro := NewDefaultReadOptions() 24 | iter := db.NewIterator(ro) 25 | defer iter.Close() 26 | var actualKeys [][]byte 27 | for iter.SeekToFirst(); iter.Valid(); iter.Next() { 28 | key := make([]byte, 4) 29 | copy(key, iter.Key().Data()) 30 | actualKeys = append(actualKeys, key) 31 | } 32 | require.Nil(t, iter.Err()) 33 | require.EqualValues(t, actualKeys, givenKeys) 34 | 35 | require.NoError(t, iter.Refresh()) 36 | } 37 | 38 | func TestIteratorWriteManyThenIter(t *testing.T) { 39 | t.Parallel() 40 | 41 | db := newTestDB(t, nil) 42 | defer db.Close() 43 | 44 | numKey := 10_000 45 | 46 | // insert keys 47 | wo := NewDefaultWriteOptions() 48 | for i := 0; i < numKey; i++ { 49 | require.Nil(t, db.Put(wo, []byte(fmt.Sprintf("key_%d", i)), []byte("val"))) 50 | } 51 | 52 | for attempt := 0; attempt < 400; attempt++ { 53 | ro := NewDefaultReadOptions() 54 | ro.SetIterateUpperBound([]byte("keya")) 55 | 56 | iter, count := db.NewIterator(ro), 0 57 | for iter.SeekToFirst(); iter.Valid(); iter.Next() { 58 | count++ 59 | } 60 | 61 | require.NoError(t, iter.Err()) 62 | require.EqualValues(t, numKey, count) 63 | 64 | ro.Destroy() 65 | iter.Close() 66 | } 67 | } 68 | 69 | func TestIteratorCF(t *testing.T) { 70 | t.Parallel() 71 | 72 | db, cfs, cleanup := newTestDBMultiCF(t, []string{"default", "c1", "c2", "c3"}, nil) 73 | defer cleanup() 74 | 75 | // insert keys 76 | givenKeys := [][]byte{[]byte("key1"), []byte("key2"), []byte("key3")} 77 | wo := NewDefaultWriteOptions() 78 | for _, k := range givenKeys { 79 | for i := range cfs { 80 | require.Nil(t, db.PutCF(wo, cfs[i], k, []byte("val"))) 81 | } 82 | } 83 | 84 | { 85 | ro := NewDefaultReadOptions() 86 | iter := db.NewIteratorCF(ro, cfs[0]) 87 | defer iter.Close() 88 | var actualKeys [][]byte 89 | for iter.SeekToFirst(); iter.Valid(); iter.Next() { 90 | key := make([]byte, 4) 91 | copy(key, iter.Key().Data()) 92 | actualKeys = append(actualKeys, key) 93 | } 94 | require.Nil(t, iter.Err()) 95 | require.EqualValues(t, actualKeys, givenKeys) 96 | } 97 | 98 | { 99 | ro := NewDefaultReadOptions() 100 | iters, err := db.NewIterators(ro, cfs) 101 | require.Nil(t, err) 102 | require.EqualValues(t, len(iters), 4) 103 | defer func() { 104 | for i := range iters { 105 | iters[i].Close() 106 | } 107 | }() 108 | 109 | for _, iter := range iters { 110 | var actualKeys [][]byte 111 | for iter.SeekToFirst(); iter.Valid(); iter.Next() { 112 | key := make([]byte, 4) 113 | copy(key, iter.Key().Data()) 114 | actualKeys = append(actualKeys, key) 115 | } 116 | require.Nil(t, iter.Err()) 117 | require.EqualValues(t, actualKeys, givenKeys) 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /jemalloc.go: -------------------------------------------------------------------------------- 1 | //go:build !testing && jemalloc 2 | 3 | package grocksdb 4 | 5 | // #include "rocksdb/c.h" 6 | import "C" 7 | 8 | // CreateJemallocNodumpAllocator generates memory allocator which allocates through 9 | // Jemalloc and utilize MADV_DONTDUMP through madvise to exclude cache items from core dump. 10 | // Applications can use the allocator with block cache to exclude block cache 11 | // usage from core dump. 12 | // 13 | // Implementation details: 14 | // The JemallocNodumpAllocator creates a dedicated jemalloc arena, and all 15 | // allocations of the JemallocNodumpAllocator are through the same arena. 16 | // The memory allocator hooks memory allocation of the arena, and calls 17 | // madvise() with MADV_DONTDUMP flag to exclude the piece of memory from 18 | // core dump. Side benefit of using single arena would be reduction of jemalloc 19 | // metadata for some workloads. 20 | // 21 | // To mitigate mutex contention for using one single arena, jemalloc tcache 22 | // (thread-local cache) is enabled to cache unused allocations for future use. 23 | // The tcache normally incurs 0.5M extra memory usage per-thread. The usage 24 | // can be reduced by limiting allocation sizes to cache. 25 | func CreateJemallocNodumpAllocator() (*MemoryAllocator, error) { 26 | var cErr *C.char 27 | 28 | c := C.rocksdb_jemalloc_nodump_allocator_create(&cErr) 29 | 30 | // check error 31 | if err := fromCError(cErr); err != nil { 32 | return nil, err 33 | } 34 | 35 | return &MemoryAllocator{ 36 | c: c, 37 | }, nil 38 | } 39 | -------------------------------------------------------------------------------- /jemalloc_test.go: -------------------------------------------------------------------------------- 1 | //go:build !testing && jemalloc 2 | 3 | package grocksdb 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestMemAlloc(t *testing.T) { 12 | t.Parallel() 13 | 14 | m, err := CreateJemallocNodumpAllocator() 15 | require.NoError(t, err) 16 | m.Destroy() 17 | } 18 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | // #include "grocksdb.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // Logger struct. 9 | type Logger struct { 10 | c *C.rocksdb_logger_t 11 | } 12 | 13 | func NewStderrLogger(level InfoLogLevel, prefix string) *Logger { 14 | prefix_ := C.CString(prefix) 15 | defer C.free(unsafe.Pointer(prefix_)) 16 | 17 | return &Logger{ 18 | c: C.rocksdb_logger_create_stderr_logger(C.int(level), prefix_), 19 | } 20 | } 21 | 22 | // Destroy Logger. 23 | func (l *Logger) Destroy() { 24 | C.rocksdb_logger_destroy(l.c) 25 | l.c = nil 26 | } 27 | -------------------------------------------------------------------------------- /mem_alloc.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | // MemoryAllocator wraps memory allocator for rocksdb. 8 | type MemoryAllocator struct { 9 | c *C.rocksdb_memory_allocator_t 10 | } 11 | 12 | // Destroy this mem allocator. 13 | func (m *MemoryAllocator) Destroy() { 14 | C.rocksdb_memory_allocator_destroy(m.c) 15 | } 16 | -------------------------------------------------------------------------------- /memory_usage.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | // MemoryUsage contains memory usage statistics provided by RocksDB 8 | type MemoryUsage struct { 9 | // MemTableTotal estimates memory usage of all mem-tables 10 | MemTableTotal uint64 11 | // MemTableUnflushed estimates memory usage of unflushed mem-tables 12 | MemTableUnflushed uint64 13 | // MemTableReadersTotal memory usage of table readers (indexes and bloom filters) 14 | MemTableReadersTotal uint64 15 | // CacheTotal memory usage of cache 16 | CacheTotal uint64 17 | } 18 | 19 | // GetApproximateMemoryUsageByType returns summary 20 | // memory usage stats for given databases and caches. 21 | func GetApproximateMemoryUsageByType(dbs []*DB, caches []*Cache) (result *MemoryUsage, err error) { 22 | // register memory consumers 23 | consumers := C.rocksdb_memory_consumers_create() 24 | 25 | for _, db := range dbs { 26 | if db != nil { 27 | C.rocksdb_memory_consumers_add_db(consumers, db.c) 28 | } 29 | } 30 | 31 | for _, cache := range caches { 32 | if cache != nil { 33 | C.rocksdb_memory_consumers_add_cache(consumers, cache.c) 34 | } 35 | } 36 | 37 | // obtain memory usage stats 38 | var cErr *C.char 39 | memoryUsage := C.rocksdb_approximate_memory_usage_create(consumers, &cErr) 40 | if err = fromCError(cErr); err == nil { 41 | result = &MemoryUsage{ 42 | MemTableTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_total(memoryUsage)), 43 | MemTableUnflushed: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_unflushed(memoryUsage)), 44 | MemTableReadersTotal: uint64(C.rocksdb_approximate_memory_usage_get_mem_table_readers_total(memoryUsage)), 45 | CacheTotal: uint64(C.rocksdb_approximate_memory_usage_get_cache_total(memoryUsage)), 46 | } 47 | } 48 | 49 | C.rocksdb_approximate_memory_usage_destroy(memoryUsage) 50 | C.rocksdb_memory_consumers_destroy(consumers) 51 | 52 | return 53 | } 54 | -------------------------------------------------------------------------------- /memory_usage_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "crypto/rand" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestMemoryUsage(t *testing.T) { 12 | t.Parallel() 13 | 14 | // create database with cache 15 | cache := NewLRUCache(8 * 1024 * 1024) 16 | cache.SetCapacity(90) 17 | 18 | bbto := NewDefaultBlockBasedTableOptions() 19 | bbto.SetBlockCache(cache) 20 | defer cache.Destroy() 21 | 22 | rowCache := NewLRUCache(8 * 1024 * 1024) 23 | defer rowCache.Destroy() 24 | 25 | blobCache := NewLRUCache(8 * 1024 * 1024) 26 | defer blobCache.Destroy() 27 | 28 | applyOpts := func(opts *Options) { 29 | opts.SetBlockBasedTableFactory(bbto) 30 | opts.SetRowCache(rowCache) 31 | opts.SetBlobCache(blobCache) 32 | } 33 | 34 | db := newTestDB(t, applyOpts) 35 | defer db.Close() 36 | 37 | // take first memory usage snapshot 38 | mu1, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) 39 | require.Nil(t, err) 40 | 41 | // perform IO operations that will affect in-memory tables (and maybe cache as well) 42 | wo := NewDefaultWriteOptions() 43 | defer wo.Destroy() 44 | ro := NewDefaultReadOptions() 45 | defer ro.Destroy() 46 | 47 | key := []byte("key") 48 | value := make([]byte, 1024) 49 | _, err = rand.Read(value) 50 | require.Nil(t, err) 51 | 52 | err = db.Put(wo, key, value) 53 | require.Nil(t, err) 54 | _, err = db.Get(ro, key) 55 | require.Nil(t, err) 56 | 57 | // take second memory usage snapshot 58 | mu2, err := GetApproximateMemoryUsageByType([]*DB{db}, []*Cache{cache}) 59 | require.Nil(t, err) 60 | 61 | // the amount of memory used by memtables should increase after write/read; 62 | // cache memory usage is not likely to be changed, perhaps because requested key is kept by memtable 63 | assert.True(t, mu2.CacheTotal >= mu1.CacheTotal) 64 | assert.True(t, mu2.MemTableReadersTotal >= mu1.MemTableReadersTotal) 65 | 66 | // check cached 67 | require.EqualValues(t, 0, rowCache.GetPinnedUsage()) 68 | require.EqualValues(t, 0, rowCache.GetUsage()) 69 | } 70 | -------------------------------------------------------------------------------- /merge_operator.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // A MergeOperator specifies the SEMANTICS of a merge, which only 8 | // client knows. It could be numeric addition, list append, string 9 | // concatenation, edit data structure, ... , anything. 10 | // The library, on the other hand, is concerned with the exercise of this 11 | // interface, at the right time (during get, iteration, compaction...) 12 | // 13 | // Please read the RocksDB documentation for 14 | // more details and example implementations. 15 | type MergeOperator interface { 16 | // Gives the client a way to express the read -> modify -> write semantics 17 | // key: The key that's associated with this merge operation. 18 | // Client could multiplex the merge operator based on it 19 | // if the key space is partitioned and different subspaces 20 | // refer to different types of data which have different 21 | // merge operation semantics. 22 | // existingValue: null indicates that the key does not exist before this op. 23 | // operands: the sequence of merge operations to apply, front() first. 24 | // 25 | // Return true on success. 26 | // 27 | // All values passed in will be client-specific values. So if this method 28 | // returns false, it is because client specified bad data or there was 29 | // internal corruption. This will be treated as an error by the library. 30 | FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) 31 | 32 | // The name of the MergeOperator. 33 | Name() string 34 | } 35 | 36 | // PartialMerger implements PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, err) 37 | // When a MergeOperator implements this interface, PartialMerge will be called in addition 38 | // to FullMerge for compactions across levels 39 | type PartialMerger interface { 40 | // This function performs merge(left_op, right_op) 41 | // when both the operands are themselves merge operation types 42 | // that you would have passed to a db.Merge() call in the same order 43 | // (i.e.: db.Merge(key,left_op), followed by db.Merge(key,right_op)). 44 | // 45 | // PartialMerge should combine them into a single merge operation. 46 | // The return value should be constructed such that a call to 47 | // db.Merge(key, new_value) would yield the same result as a call 48 | // to db.Merge(key, left_op) followed by db.Merge(key, right_op). 49 | // 50 | // If it is impossible or infeasible to combine the two operations, return false. 51 | // The library will internally keep track of the operations, and apply them in the 52 | // correct order once a base-value (a Put/Delete/End-of-Database) is seen. 53 | PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) 54 | } 55 | 56 | // MultiMerger implements PartialMergeMulti(key []byte, operands [][]byte) ([]byte, err) 57 | // When a MergeOperator implements this interface, PartialMergeMulti will be called in addition 58 | // to FullMerge for compactions across levels 59 | type MultiMerger interface { 60 | // PartialMerge performs merge on multiple operands 61 | // when all of the operands are themselves merge operation types 62 | // that you would have passed to a db.Merge() call in the same order 63 | // (i.e.: db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), 64 | // ... db.Merge(key, operand[n])). 65 | // 66 | // PartialMerge should combine them into a single merge operation. 67 | // The return value should be constructed such that a call to 68 | // db.Merge(key, new_value) would yield the same result as a call 69 | // to db.Merge(key,operand[0]), followed by db.Merge(key,operand[1]), 70 | // ... db.Merge(key, operand[n])). 71 | // 72 | // If it is impossible or infeasible to combine the operations, return false. 73 | // The library will internally keep track of the operations, and apply them in the 74 | // correct order once a base-value (a Put/Delete/End-of-Database) is seen. 75 | PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) 76 | 77 | // Destroy pointer/underlying data 78 | Destroy() 79 | } 80 | 81 | // NewNativeMergeOperator creates a MergeOperator object. 82 | func NewNativeMergeOperator(c unsafe.Pointer) MergeOperator { 83 | return &nativeMergeOperator{c: (*C.rocksdb_mergeoperator_t)(c)} 84 | } 85 | 86 | type nativeMergeOperator struct { 87 | c *C.rocksdb_mergeoperator_t 88 | } 89 | 90 | func (mo *nativeMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 91 | return nil, false 92 | } 93 | 94 | func (mo *nativeMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { 95 | return nil, false 96 | } 97 | func (mo *nativeMergeOperator) Name() string { return "" } 98 | func (mo *nativeMergeOperator) Destroy() { 99 | C.rocksdb_mergeoperator_destroy(mo.c) 100 | mo.c = nil 101 | } 102 | 103 | // Hold references to merge operators. 104 | var mergeOperators = NewCOWList() 105 | 106 | type mergeOperatorWrapper struct { 107 | name *C.char 108 | mergeOperator MergeOperator 109 | } 110 | 111 | func registerMergeOperator(merger MergeOperator) int { 112 | return mergeOperators.Append(mergeOperatorWrapper{C.CString(merger.Name()), merger}) 113 | } 114 | 115 | //export gorocksdb_mergeoperator_full_merge 116 | func gorocksdb_mergeoperator_full_merge(idx int, cKey *C.char, cKeyLen C.size_t, cExistingValue *C.char, cExistingValueLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { 117 | key := refCBytes(cKey, cKeyLen) 118 | rawOperands := charSlice(cOperands, cNumOperands) 119 | operandsLen := sizeSlice(cOperandsLen, cNumOperands) 120 | existingValue := refCBytes(cExistingValue, cExistingValueLen) 121 | operands := make([][]byte, int(cNumOperands)) 122 | for i, len := range operandsLen { 123 | operands[i] = refCBytes(rawOperands[i], len) 124 | } 125 | 126 | newValue, success := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator.FullMerge(key, existingValue, operands) 127 | newValueLen := len(newValue) 128 | 129 | *cNewValueLen = C.size_t(newValueLen) 130 | *cSuccess = boolToChar(success) 131 | 132 | return cByteSlice(newValue) 133 | } 134 | 135 | //export gorocksdb_mergeoperator_partial_merge_multi 136 | func gorocksdb_mergeoperator_partial_merge_multi(idx int, cKey *C.char, cKeyLen C.size_t, cOperands **C.char, cOperandsLen *C.size_t, cNumOperands C.int, cSuccess *C.uchar, cNewValueLen *C.size_t) *C.char { 137 | key := refCBytes(cKey, cKeyLen) 138 | rawOperands := charSlice(cOperands, cNumOperands) 139 | operandsLen := sizeSlice(cOperandsLen, cNumOperands) 140 | operands := make([][]byte, int(cNumOperands)) 141 | for i, len := range operandsLen { 142 | operands[i] = refCBytes(rawOperands[i], len) 143 | } 144 | 145 | var newValue []byte 146 | success := true 147 | 148 | merger := mergeOperators.Get(idx).(mergeOperatorWrapper).mergeOperator 149 | 150 | // check if this MergeOperator supports partial or multi merges 151 | switch v := merger.(type) { 152 | case MultiMerger: 153 | newValue, success = v.PartialMergeMulti(key, operands) 154 | case PartialMerger: 155 | leftOperand := operands[0] 156 | for i := 1; i < int(cNumOperands); i++ { 157 | newValue, success = v.PartialMerge(key, leftOperand, operands[i]) 158 | if !success { 159 | break 160 | } 161 | leftOperand = newValue 162 | } 163 | default: 164 | success = false 165 | } 166 | 167 | newValueLen := len(newValue) 168 | *cNewValueLen = C.size_t(newValueLen) 169 | *cSuccess = boolToChar(success) 170 | 171 | return cByteSlice(newValue) 172 | } 173 | 174 | //export gorocksdb_mergeoperator_name 175 | func gorocksdb_mergeoperator_name(idx int) *C.char { 176 | return mergeOperators.Get(idx).(mergeOperatorWrapper).name 177 | } 178 | -------------------------------------------------------------------------------- /merge_operator_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestMergeOperator(t *testing.T) { 10 | t.Parallel() 11 | 12 | var ( 13 | givenKey = []byte("hello") 14 | givenVal1 = []byte("foo") 15 | givenVal2 = []byte("bar") 16 | givenMerged = []byte("foobar") 17 | ) 18 | merger := &mockMergeOperator{ 19 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 20 | require.EqualValues(t, key, givenKey) 21 | require.EqualValues(t, existingValue, givenVal1) 22 | require.EqualValues(t, operands, [][]byte{givenVal2}) 23 | return givenMerged, true 24 | }, 25 | } 26 | db := newTestDB(t, func(opts *Options) { 27 | opts.SetMergeOperator(merger) 28 | }) 29 | defer db.Close() 30 | 31 | wo := NewDefaultWriteOptions() 32 | require.Nil(t, db.Put(wo, givenKey, givenVal1)) 33 | require.Nil(t, db.Merge(wo, givenKey, givenVal2)) 34 | 35 | // trigger a compaction to ensure that a merge is performed 36 | db.CompactRange(Range{nil, nil}) 37 | 38 | ro := NewDefaultReadOptions() 39 | v1, err := db.Get(ro, givenKey) 40 | require.Nil(t, err) 41 | require.EqualValues(t, v1.Data(), givenMerged) 42 | v1.Free() 43 | } 44 | 45 | func TestPartialMergeOperator(t *testing.T) { 46 | t.Parallel() 47 | 48 | var ( 49 | givenKey = []byte("hello") 50 | startingVal = []byte("foo") 51 | mergeVal1 = []byte("bar") 52 | mergeVal2 = []byte("baz") 53 | fMergeResult = []byte("foobarbaz") 54 | pMergeResult = []byte("barbaz") 55 | ) 56 | 57 | merger := &mockMergePartialOperator{ 58 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 59 | require.EqualValues(t, key, givenKey) 60 | require.EqualValues(t, existingValue, startingVal) 61 | require.EqualValues(t, operands[0], pMergeResult) 62 | return fMergeResult, true 63 | }, 64 | partialMerge: func(key, leftOperand, rightOperand []byte) ([]byte, bool) { 65 | require.EqualValues(t, key, givenKey) 66 | require.EqualValues(t, leftOperand, mergeVal1) 67 | require.EqualValues(t, rightOperand, mergeVal2) 68 | return pMergeResult, true 69 | }, 70 | } 71 | db := newTestDB(t, func(opts *Options) { 72 | opts.SetMergeOperator(merger) 73 | }) 74 | defer db.Close() 75 | 76 | wo := NewDefaultWriteOptions() 77 | defer wo.Destroy() 78 | 79 | // insert a starting value and compact to trigger merges 80 | require.Nil(t, db.Put(wo, givenKey, startingVal)) 81 | 82 | // trigger a compaction to ensure that a merge is performed 83 | db.CompactRange(Range{nil, nil}) 84 | 85 | // we expect these two operands to be passed to merge partial 86 | require.Nil(t, db.Merge(wo, givenKey, mergeVal1)) 87 | require.Nil(t, db.Merge(wo, givenKey, mergeVal2)) 88 | 89 | // trigger a compaction to ensure that a 90 | // partial and full merge are performed 91 | db.CompactRange(Range{nil, nil}) 92 | 93 | ro := NewDefaultReadOptions() 94 | v1, err := db.Get(ro, givenKey) 95 | require.Nil(t, err) 96 | require.EqualValues(t, v1.Data(), fMergeResult) 97 | v1.Free() 98 | } 99 | 100 | func TestMergeMultiOperator(t *testing.T) { 101 | t.Parallel() 102 | 103 | var ( 104 | givenKey = []byte("hello") 105 | startingVal = []byte("foo") 106 | mergeVal1 = []byte("bar") 107 | mergeVal2 = []byte("baz") 108 | fMergeResult = []byte("foobarbaz") 109 | pMergeResult = []byte("bar") 110 | ) 111 | 112 | merger := &mockMergeMultiOperator{ 113 | fullMerge: func(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 114 | require.EqualValues(t, key, givenKey) 115 | require.EqualValues(t, existingValue, startingVal) 116 | require.EqualValues(t, operands[0], pMergeResult) 117 | return fMergeResult, true 118 | }, 119 | partialMergeMulti: func(key []byte, operands [][]byte) ([]byte, bool) { 120 | require.EqualValues(t, key, givenKey) 121 | require.EqualValues(t, operands[0], mergeVal1) 122 | require.EqualValues(t, operands[1], mergeVal2) 123 | return pMergeResult, true 124 | }, 125 | } 126 | db := newTestDB(t, func(opts *Options) { 127 | opts.SetMergeOperator(merger) 128 | }) 129 | defer db.Close() 130 | 131 | wo := NewDefaultWriteOptions() 132 | defer wo.Destroy() 133 | 134 | // insert a starting value and compact to trigger merges 135 | require.Nil(t, db.Put(wo, givenKey, startingVal)) 136 | 137 | // trigger a compaction to ensure that a merge is performed 138 | db.CompactRange(Range{nil, nil}) 139 | 140 | // we expect these two operands to be passed to merge multi 141 | require.Nil(t, db.Merge(wo, givenKey, mergeVal1)) 142 | require.Nil(t, db.Merge(wo, givenKey, mergeVal2)) 143 | 144 | // trigger a compaction to ensure that a 145 | // partial and full merge are performed 146 | db.CompactRange(Range{nil, nil}) 147 | 148 | ro := NewDefaultReadOptions() 149 | v1, err := db.Get(ro, givenKey) 150 | require.Nil(t, err) 151 | require.EqualValues(t, v1.Data(), fMergeResult) 152 | v1.Free() 153 | } 154 | 155 | // Mock Objects 156 | type mockMergeOperator struct { 157 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 158 | } 159 | 160 | func (m *mockMergeOperator) Name() string { return "grocksdb.test" } 161 | func (m *mockMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 162 | return m.fullMerge(key, existingValue, operands) 163 | } 164 | 165 | type mockMergeMultiOperator struct { 166 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 167 | partialMergeMulti func(key []byte, operands [][]byte) ([]byte, bool) 168 | } 169 | 170 | func (m *mockMergeMultiOperator) Name() string { return "grocksdb.multi" } 171 | func (m *mockMergeMultiOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 172 | return m.fullMerge(key, existingValue, operands) 173 | } 174 | 175 | func (m *mockMergeMultiOperator) PartialMergeMulti(key []byte, operands [][]byte) ([]byte, bool) { 176 | return m.partialMergeMulti(key, operands) 177 | } 178 | 179 | type mockMergePartialOperator struct { 180 | fullMerge func(key, existingValue []byte, operands [][]byte) ([]byte, bool) 181 | partialMerge func(key, leftOperand, rightOperand []byte) ([]byte, bool) 182 | } 183 | 184 | func (m *mockMergePartialOperator) Name() string { return "grocksdb.partial" } 185 | func (m *mockMergePartialOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 186 | return m.fullMerge(key, existingValue, operands) 187 | } 188 | 189 | func (m *mockMergePartialOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { 190 | return m.partialMerge(key, leftOperand, rightOperand) 191 | } 192 | -------------------------------------------------------------------------------- /non_builtin.go: -------------------------------------------------------------------------------- 1 | //go:build !grocksdb_no_link && !grocksdb_clean_link 2 | 3 | // The default link options, to customize it, you can try build tag `grocksdb_clean_link` for a cleaner set of flags, 4 | // or `grocksdb_no_link` where you have full control through `CGO_LDFLAGS` environment variable. 5 | package grocksdb 6 | 7 | // #cgo LDFLAGS: -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy 8 | import "C" 9 | -------------------------------------------------------------------------------- /non_builtin_clean_link.go: -------------------------------------------------------------------------------- 1 | //go:build !grocksdb_no_link && grocksdb_clean_link 2 | 3 | package grocksdb 4 | 5 | // #cgo LDFLAGS: -lrocksdb -pthread -lstdc++ -ldl 6 | import "C" 7 | -------------------------------------------------------------------------------- /optimistic_transaction_db.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // OptimisticTransactionDB is a reusable handle to a RocksDB optimistic transactional database on disk. 12 | type OptimisticTransactionDB struct { 13 | c *C.rocksdb_optimistictransactiondb_t 14 | name string 15 | opts *Options 16 | } 17 | 18 | // OpenOptimisticTransactionDb opens a database with the specified options. 19 | func OpenOptimisticTransactionDb( 20 | opts *Options, 21 | name string, 22 | ) (tdb *OptimisticTransactionDB, err error) { 23 | var ( 24 | cErr *C.char 25 | cName = C.CString(name) 26 | ) 27 | 28 | db := C.rocksdb_optimistictransactiondb_open( 29 | opts.c, cName, &cErr) 30 | if err = fromCError(cErr); err == nil { 31 | tdb = &OptimisticTransactionDB{ 32 | name: name, 33 | c: db, 34 | opts: opts, 35 | } 36 | } 37 | 38 | C.free(unsafe.Pointer(cName)) 39 | return 40 | } 41 | 42 | // OpenOptimisticTransactionDbColumnFamilies opens a database with the specified column families. 43 | func OpenOptimisticTransactionDbColumnFamilies( 44 | opts *Options, 45 | name string, 46 | cfNames []string, 47 | cfOpts []*Options, 48 | ) (db *OptimisticTransactionDB, cfHandles []*ColumnFamilyHandle, err error) { 49 | numColumnFamilies := len(cfNames) 50 | if numColumnFamilies != len(cfOpts) { 51 | err = ErrColumnFamilyMustMatch 52 | return 53 | } 54 | 55 | cName := C.CString(name) 56 | cNames := make([]*C.char, numColumnFamilies) 57 | for i, s := range cfNames { 58 | cNames[i] = C.CString(s) 59 | } 60 | 61 | cOpts := make([]*C.rocksdb_options_t, numColumnFamilies) 62 | for i, o := range cfOpts { 63 | cOpts[i] = o.c 64 | } 65 | 66 | cHandles := make([]*C.rocksdb_column_family_handle_t, numColumnFamilies) 67 | 68 | var cErr *C.char 69 | _db := C.rocksdb_optimistictransactiondb_open_column_families( 70 | opts.c, 71 | cName, 72 | C.int(numColumnFamilies), 73 | &cNames[0], 74 | &cOpts[0], 75 | &cHandles[0], 76 | &cErr, 77 | ) 78 | if err = fromCError(cErr); err == nil { 79 | db = &OptimisticTransactionDB{ 80 | name: name, 81 | c: _db, 82 | opts: opts, 83 | } 84 | cfHandles = make([]*ColumnFamilyHandle, numColumnFamilies) 85 | for i, c := range cHandles { 86 | cfHandles[i] = newNativeColumnFamilyHandle(c) 87 | } 88 | } 89 | 90 | C.free(unsafe.Pointer(cName)) 91 | for _, s := range cNames { 92 | C.free(unsafe.Pointer(s)) 93 | } 94 | return 95 | } 96 | 97 | // TransactionBegin begins a new transaction 98 | // with the WriteOptions and TransactionOptions given. 99 | func (db *OptimisticTransactionDB) TransactionBegin( 100 | opts *WriteOptions, 101 | transactionOpts *OptimisticTransactionOptions, 102 | oldTransaction *Transaction, 103 | ) *Transaction { 104 | if oldTransaction != nil { 105 | cTx := C.rocksdb_optimistictransaction_begin( 106 | db.c, 107 | opts.c, 108 | transactionOpts.c, 109 | oldTransaction.c, 110 | ) 111 | return newNativeTransaction(cTx) 112 | } 113 | 114 | cTx := C.rocksdb_optimistictransaction_begin(db.c, opts.c, transactionOpts.c, nil) 115 | return newNativeTransaction(cTx) 116 | } 117 | 118 | // NewCheckpoint creates a new Checkpoint for this db. 119 | func (db *OptimisticTransactionDB) NewCheckpoint() (cp *Checkpoint, err error) { 120 | var cErr *C.char 121 | 122 | cCheckpoint := C.rocksdb_optimistictransactiondb_checkpoint_object_create( 123 | db.c, &cErr, 124 | ) 125 | if err = fromCError(cErr); err == nil { 126 | cp = newNativeCheckpoint(cCheckpoint) 127 | } 128 | 129 | return 130 | } 131 | 132 | // Write batch. 133 | func (db *OptimisticTransactionDB) Write(opts *WriteOptions, batch *WriteBatch) (err error) { 134 | var cErr *C.char 135 | 136 | C.rocksdb_optimistictransactiondb_write(db.c, opts.c, batch.c, &cErr) 137 | err = fromCError(cErr) 138 | 139 | return 140 | } 141 | 142 | // Close closes the database. 143 | func (db *OptimisticTransactionDB) Close() { 144 | C.rocksdb_optimistictransactiondb_close(db.c) 145 | db.c = nil 146 | } 147 | 148 | // GetBaseDB returns base-database. 149 | func (db *OptimisticTransactionDB) GetBaseDB() *DB { 150 | return &DB{ 151 | c: C.rocksdb_optimistictransactiondb_get_base_db(db.c), 152 | name: db.name, 153 | opts: db.opts, 154 | } 155 | } 156 | 157 | // CloseBaseDB closes base-database. 158 | func (db *OptimisticTransactionDB) CloseBaseDB(base *DB) { 159 | C.rocksdb_optimistictransactiondb_close_base_db(base.c) 160 | base.c = nil 161 | } 162 | -------------------------------------------------------------------------------- /options_backup_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestBackupableDBOptions(t *testing.T) { 11 | t.Parallel() 12 | 13 | opts := NewBackupableDBOptions("/tmp/v1") 14 | defer opts.Destroy() 15 | 16 | env := NewDefaultEnv() 17 | defer env.Destroy() 18 | 19 | opts.SetEnv(env) 20 | opts.SetBackupDir("/tmp/v2") 21 | 22 | require.True(t, opts.IsShareTableFiles()) // check default value 23 | opts.ShareTableFiles(false) 24 | require.False(t, opts.IsShareTableFiles()) 25 | 26 | require.True(t, opts.IsSync()) 27 | opts.SetSync(false) 28 | require.False(t, opts.IsSync()) 29 | 30 | require.False(t, opts.IsDestroyOldData()) 31 | opts.DestroyOldData(true) 32 | require.True(t, opts.IsDestroyOldData()) 33 | 34 | require.True(t, opts.IsBackupLogFiles()) 35 | opts.BackupLogFiles(false) 36 | require.False(t, opts.IsBackupLogFiles()) 37 | 38 | require.EqualValues(t, 0, opts.GetBackupRateLimit()) 39 | opts.SetBackupRateLimit(531 << 10) 40 | require.EqualValues(t, 531<<10, opts.GetBackupRateLimit()) 41 | 42 | require.EqualValues(t, 0, opts.GetRestoreRateLimit()) 43 | opts.SetRestoreRateLimit(53 << 10) 44 | require.EqualValues(t, 53<<10, opts.GetRestoreRateLimit()) 45 | 46 | require.EqualValues(t, 1, opts.GetMaxBackgroundOperations()) 47 | opts.SetMaxBackgroundOperations(3) 48 | require.EqualValues(t, 3, opts.GetMaxBackgroundOperations()) 49 | 50 | require.EqualValues(t, 4194304, opts.GetCallbackTriggerIntervalSize()) 51 | opts.SetCallbackTriggerIntervalSize(800 << 10) 52 | require.EqualValues(t, 800<<10, opts.GetCallbackTriggerIntervalSize()) 53 | 54 | require.EqualValues(t, math.MaxInt32, opts.GetMaxValidBackupsToOpen()) 55 | opts.SetMaxValidBackupsToOpen(29) 56 | require.EqualValues(t, 29, opts.GetMaxValidBackupsToOpen()) 57 | 58 | require.EqualValues(t, 0x80000002, opts.GetShareFilesWithChecksumNaming()) 59 | opts.SetShareFilesWithChecksumNaming(UseDBSessionID | LegacyCrc32cAndFileSize) 60 | require.EqualValues(t, UseDBSessionID|LegacyCrc32cAndFileSize, opts.GetShareFilesWithChecksumNaming()) 61 | } 62 | -------------------------------------------------------------------------------- /options_blob_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestOptionBlobFile(t *testing.T) { 10 | t.Parallel() 11 | 12 | opt := NewDefaultOptions() 13 | defer opt.Destroy() 14 | 15 | opt.EnableBlobFiles(true) 16 | require.True(t, opt.IsBlobFilesEnabled()) 17 | 18 | opt.SetMinBlobSize(1024) 19 | require.EqualValues(t, 1024, opt.GetMinBlobSize()) 20 | 21 | require.EqualValues(t, 256<<20, opt.GetBlobFileSize()) 22 | opt.SetBlobFileSize(128 << 20) 23 | require.EqualValues(t, 128<<20, opt.GetBlobFileSize()) 24 | 25 | require.Equal(t, NoCompression, opt.GetBlobCompressionType()) 26 | opt.SetBlobCompressionType(SnappyCompression) 27 | require.Equal(t, SnappyCompression, opt.GetBlobCompressionType()) 28 | 29 | require.False(t, opt.IsBlobGCEnabled()) 30 | opt.EnableBlobGC(true) 31 | require.True(t, opt.IsBlobGCEnabled()) 32 | 33 | require.EqualValues(t, 0.25, opt.GetBlobGCAgeCutoff()) 34 | opt.SetBlobGCAgeCutoff(0.3) 35 | require.EqualValues(t, 0.3, opt.GetBlobGCAgeCutoff()) 36 | 37 | require.EqualValues(t, 1.0, opt.GetBlobGCForceThreshold()) 38 | opt.SetBlobGCForceThreshold(1.3) 39 | require.EqualValues(t, 1.3, opt.GetBlobGCForceThreshold()) 40 | } 41 | -------------------------------------------------------------------------------- /options_block_based_table_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBBT(t *testing.T) { 8 | t.Parallel() 9 | 10 | b := NewDefaultBlockBasedTableOptions() 11 | defer b.Destroy() 12 | 13 | b.SetBlockSize(123) 14 | b.SetOptimizeFiltersForMemory(true) 15 | b.SetTopLevelIndexPinningTier(KFallbackPinningTier) 16 | b.SetPartitionPinningTier(KNonePinningTier) 17 | b.SetUnpartitionedPinningTier(KAllPinningTier) 18 | } 19 | -------------------------------------------------------------------------------- /options_compaction_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestOptionCompactions(t *testing.T) { 10 | t.Parallel() 11 | 12 | co := NewCompactRangeOptions() 13 | defer co.Destroy() 14 | 15 | co.SetFullHistoryTsLow([]byte{1, 2, 3}) 16 | 17 | require.EqualValues(t, false, co.GetExclusiveManualCompaction()) 18 | co.SetExclusiveManualCompaction(true) 19 | require.EqualValues(t, true, co.GetExclusiveManualCompaction()) 20 | 21 | require.EqualValues(t, KIfHaveCompactionFilter, co.BottommostLevelCompaction()) 22 | co.SetBottommostLevelCompaction(KForce) 23 | require.EqualValues(t, KForce, co.BottommostLevelCompaction()) 24 | 25 | require.EqualValues(t, false, co.ChangeLevel()) 26 | co.SetChangeLevel(true) 27 | require.EqualValues(t, true, co.ChangeLevel()) 28 | 29 | require.EqualValues(t, -1, co.TargetLevel()) 30 | co.SetTargetLevel(2) 31 | require.EqualValues(t, 2, co.TargetLevel()) 32 | 33 | require.EqualValues(t, 0, co.TargetPathID()) 34 | co.SetTargetPathID(1) 35 | require.EqualValues(t, 1, co.TargetPathID()) 36 | 37 | require.False(t, co.AllowWriteStall()) 38 | co.SetAllowWriteStall(true) 39 | require.True(t, co.AllowWriteStall()) 40 | 41 | require.EqualValues(t, 0, co.MaxSubCompactions()) 42 | co.SetMaxSubCompactions(2) 43 | require.EqualValues(t, 2, co.MaxSubCompactions()) 44 | } 45 | 46 | func TestFifoCompactOption(t *testing.T) { 47 | t.Parallel() 48 | 49 | fo := NewDefaultFIFOCompactionOptions() 50 | defer fo.Destroy() 51 | 52 | fo.SetMaxTableFilesSize(2 << 10) 53 | require.EqualValues(t, 2<<10, fo.GetMaxTableFilesSize()) 54 | 55 | require.False(t, fo.AllowCompaction()) 56 | fo.SetAllowCompaction(true) 57 | require.True(t, fo.AllowCompaction()) 58 | } 59 | 60 | func TestUniversalCompactOption(t *testing.T) { 61 | t.Parallel() 62 | 63 | uo := NewDefaultUniversalCompactionOptions() 64 | defer uo.Destroy() 65 | 66 | uo.SetSizeRatio(2) 67 | require.EqualValues(t, 2, uo.GetSizeRatio()) 68 | 69 | uo.SetMinMergeWidth(3) 70 | require.EqualValues(t, 3, uo.GetMinMergeWidth()) 71 | 72 | uo.SetMaxMergeWidth(123) 73 | require.EqualValues(t, 123, uo.GetMaxMergeWidth()) 74 | 75 | uo.SetMaxSizeAmplificationPercent(20) 76 | require.EqualValues(t, 20, uo.GetMaxSizeAmplificationPercent()) 77 | 78 | uo.SetCompressionSizePercent(18) 79 | require.EqualValues(t, 18, uo.GetCompressionSizePercent()) 80 | 81 | uo.SetStopStyle(CompactionStopStyleTotalSize) 82 | require.EqualValues(t, CompactionStopStyleTotalSize, uo.GetStopStyle()) 83 | } 84 | -------------------------------------------------------------------------------- /options_compression.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // CompressionOptions represents options for different compression algorithms like Zlib. 4 | type CompressionOptions struct { 5 | WindowBits int 6 | Level int 7 | Strategy int 8 | MaxDictBytes int 9 | } 10 | 11 | // NewDefaultCompressionOptions creates a default CompressionOptions object. 12 | func NewDefaultCompressionOptions() CompressionOptions { 13 | return NewCompressionOptions(-14, -1, 0, 0) 14 | } 15 | 16 | // NewCompressionOptions creates a CompressionOptions object. 17 | func NewCompressionOptions(windowBits, level, strategy, maxDictBytes int) CompressionOptions { 18 | return CompressionOptions{ 19 | WindowBits: windowBits, 20 | Level: level, 21 | Strategy: strategy, 22 | MaxDictBytes: maxDictBytes, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /options_env.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // EnvOptions represents options for env. 7 | type EnvOptions struct { 8 | c *C.rocksdb_envoptions_t 9 | } 10 | 11 | // NewDefaultEnvOptions creates a default EnvOptions object. 12 | func NewDefaultEnvOptions() *EnvOptions { 13 | return newNativeEnvOptions(C.rocksdb_envoptions_create()) 14 | } 15 | 16 | // NewNativeEnvOptions creates a EnvOptions object. 17 | func newNativeEnvOptions(c *C.rocksdb_envoptions_t) *EnvOptions { 18 | return &EnvOptions{c: c} 19 | } 20 | 21 | // Destroy deallocates the EnvOptions object. 22 | func (opts *EnvOptions) Destroy() { 23 | C.rocksdb_envoptions_destroy(opts.c) 24 | opts.c = nil 25 | } 26 | -------------------------------------------------------------------------------- /options_env_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import "testing" 4 | 5 | func TestOptEnv(t *testing.T) { 6 | t.Parallel() 7 | 8 | opt := NewDefaultEnvOptions() 9 | defer opt.Destroy() 10 | } 11 | -------------------------------------------------------------------------------- /options_flush.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // FlushOptions represent all of the available options when manual flushing the 7 | // database. 8 | type FlushOptions struct { 9 | c *C.rocksdb_flushoptions_t 10 | } 11 | 12 | // NewDefaultFlushOptions creates a default FlushOptions object. 13 | func NewDefaultFlushOptions() *FlushOptions { 14 | return newNativeFlushOptions(C.rocksdb_flushoptions_create()) 15 | } 16 | 17 | // NewNativeFlushOptions creates a FlushOptions object. 18 | func newNativeFlushOptions(c *C.rocksdb_flushoptions_t) *FlushOptions { 19 | return &FlushOptions{c: c} 20 | } 21 | 22 | // SetWait specify if the flush will wait until the flush is done. 23 | // 24 | // Default: true 25 | func (opts *FlushOptions) SetWait(value bool) { 26 | C.rocksdb_flushoptions_set_wait(opts.c, boolToChar(value)) 27 | } 28 | 29 | // IsWait returns if the flush will wait until the flush is done. 30 | func (opts *FlushOptions) IsWait() bool { 31 | return charToBool(C.rocksdb_flushoptions_get_wait(opts.c)) 32 | } 33 | 34 | // Destroy deallocates the FlushOptions object. 35 | func (opts *FlushOptions) Destroy() { 36 | C.rocksdb_flushoptions_destroy(opts.c) 37 | opts.c = nil 38 | } 39 | -------------------------------------------------------------------------------- /options_flush_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestFlushOption(t *testing.T) { 10 | t.Parallel() 11 | 12 | fo := NewDefaultFlushOptions() 13 | defer fo.Destroy() 14 | 15 | require.EqualValues(t, true, fo.IsWait()) 16 | fo.SetWait(false) 17 | require.EqualValues(t, false, fo.IsWait()) 18 | } 19 | -------------------------------------------------------------------------------- /options_ingest.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // IngestExternalFileOptions represents available options when ingesting external files. 7 | type IngestExternalFileOptions struct { 8 | c *C.rocksdb_ingestexternalfileoptions_t 9 | } 10 | 11 | // NewDefaultIngestExternalFileOptions creates a default IngestExternalFileOptions object. 12 | func NewDefaultIngestExternalFileOptions() *IngestExternalFileOptions { 13 | return newNativeIngestExternalFileOptions(C.rocksdb_ingestexternalfileoptions_create()) 14 | } 15 | 16 | // NewNativeIngestExternalFileOptions creates a IngestExternalFileOptions object. 17 | func newNativeIngestExternalFileOptions(c *C.rocksdb_ingestexternalfileoptions_t) *IngestExternalFileOptions { 18 | return &IngestExternalFileOptions{c: c} 19 | } 20 | 21 | // SetMoveFiles specifies if it should move the files instead of copying them. 22 | // Default to false. 23 | func (opts *IngestExternalFileOptions) SetMoveFiles(flag bool) { 24 | C.rocksdb_ingestexternalfileoptions_set_move_files(opts.c, boolToChar(flag)) 25 | } 26 | 27 | // SetSnapshotConsistency if specifies the consistency. 28 | // If set to false, an ingested file key could appear in existing snapshots that were created before the 29 | // file was ingested. 30 | // Default to true. 31 | func (opts *IngestExternalFileOptions) SetSnapshotConsistency(flag bool) { 32 | C.rocksdb_ingestexternalfileoptions_set_snapshot_consistency(opts.c, boolToChar(flag)) 33 | } 34 | 35 | // SetAllowGlobalSeqNo sets allow_global_seqno. If set to false,IngestExternalFile() will fail if the file key 36 | // range overlaps with existing keys or tombstones in the DB. 37 | // Default true. 38 | func (opts *IngestExternalFileOptions) SetAllowGlobalSeqNo(flag bool) { 39 | C.rocksdb_ingestexternalfileoptions_set_allow_global_seqno(opts.c, boolToChar(flag)) 40 | } 41 | 42 | // SetAllowBlockingFlush sets allow_blocking_flush. If set to false and the file key range overlaps with 43 | // the memtable key range (memtable flush required), IngestExternalFile will fail. 44 | // Default to true. 45 | func (opts *IngestExternalFileOptions) SetAllowBlockingFlush(flag bool) { 46 | C.rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(opts.c, boolToChar(flag)) 47 | } 48 | 49 | // SetIngestionBehind sets ingest_behind 50 | // Set to true if you would like duplicate keys in the file being ingested 51 | // to be skipped rather than overwriting existing data under that key. 52 | // Usecase: back-fill of some historical data in the database without 53 | // over-writing existing newer version of data. 54 | // This option could only be used if the DB has been running 55 | // with allow_ingest_behind=true since the dawn of time. 56 | // All files will be ingested at the bottommost level with seqno=0. 57 | func (opts *IngestExternalFileOptions) SetIngestionBehind(flag bool) { 58 | C.rocksdb_ingestexternalfileoptions_set_ingest_behind(opts.c, boolToChar(flag)) 59 | } 60 | 61 | // SetFailIfNotBottommostLevel sets to TRUE if user wants file to be ingested to the bottommost level. An 62 | // error of Status::TryAgain() will be returned if a file cannot fit in the bottommost level when calling 63 | // DB::IngestExternalFile()/DB::IngestExternalFiles(). 64 | // 65 | // The user should clear the bottommost level in the overlapping range before re-attempt. 66 | // Ingest_behind takes precedence over fail_if_not_bottommost_level. 67 | func (opts *IngestExternalFileOptions) SetFailIfNotBottommostLevel(flag bool) { 68 | C.rocksdb_ingestexternalfileoptions_set_fail_if_not_bottommost_level(opts.c, boolToChar(flag)) 69 | } 70 | 71 | // Destroy deallocates the IngestExternalFileOptions object. 72 | func (opts *IngestExternalFileOptions) Destroy() { 73 | C.rocksdb_ingestexternalfileoptions_destroy(opts.c) 74 | opts.c = nil 75 | } 76 | -------------------------------------------------------------------------------- /options_read_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestReadOptions(t *testing.T) { 10 | t.Parallel() 11 | 12 | ro := NewDefaultReadOptions() 13 | defer ro.Destroy() 14 | 15 | require.EqualValues(t, true, ro.VerifyChecksums()) 16 | ro.SetVerifyChecksums(false) 17 | require.EqualValues(t, false, ro.VerifyChecksums()) 18 | 19 | require.EqualValues(t, true, ro.FillCache()) 20 | ro.SetFillCache(false) 21 | require.EqualValues(t, false, ro.FillCache()) 22 | 23 | ro.SetSnapshot(newNativeSnapshot(nil)) 24 | 25 | ro.SetIterateUpperBound([]byte{1, 2, 3}) 26 | ro.SetIterateLowerBound([]byte{1, 1, 1}) 27 | ro.SetTimestamp([]byte{1, 2, 3}) 28 | ro.SetIterStartTimestamp([]byte{1, 2, 3}) 29 | 30 | require.EqualValues(t, ReadAllTier, ro.GetReadTier()) 31 | ro.SetReadTier(BlockCacheTier) 32 | require.EqualValues(t, BlockCacheTier, ro.GetReadTier()) 33 | 34 | require.EqualValues(t, false, ro.Tailing()) 35 | ro.SetTailing(true) 36 | require.EqualValues(t, true, ro.Tailing()) 37 | 38 | require.EqualValues(t, 0, ro.GetReadaheadSize()) 39 | ro.SetReadaheadSize(1 << 20) 40 | require.EqualValues(t, 1<<20, ro.GetReadaheadSize()) 41 | 42 | require.EqualValues(t, false, ro.PrefixSameAsStart()) 43 | ro.SetPrefixSameAsStart(true) 44 | require.EqualValues(t, true, ro.PrefixSameAsStart()) 45 | 46 | require.EqualValues(t, false, ro.PinData()) 47 | ro.SetPinData(true) 48 | require.EqualValues(t, true, ro.PinData()) 49 | 50 | require.EqualValues(t, false, ro.GetTotalOrderSeek()) 51 | ro.SetTotalOrderSeek(true) 52 | require.EqualValues(t, true, ro.GetTotalOrderSeek()) 53 | 54 | require.EqualValues(t, 0, ro.GetMaxSkippableInternalKeys()) 55 | ro.SetMaxSkippableInternalKeys(123) 56 | require.EqualValues(t, 123, ro.GetMaxSkippableInternalKeys()) 57 | 58 | require.EqualValues(t, false, ro.GetBackgroundPurgeOnIteratorCleanup()) 59 | ro.SetBackgroundPurgeOnIteratorCleanup(true) 60 | require.EqualValues(t, true, ro.GetBackgroundPurgeOnIteratorCleanup()) 61 | 62 | require.EqualValues(t, false, ro.IgnoreRangeDeletions()) 63 | ro.SetIgnoreRangeDeletions(true) 64 | require.EqualValues(t, true, ro.IgnoreRangeDeletions()) 65 | 66 | require.EqualValues(t, 0, ro.GetDeadline()) 67 | ro.SetDeadline(1000) 68 | require.EqualValues(t, 1000, ro.GetDeadline()) 69 | 70 | require.EqualValues(t, 0, ro.GetIOTimeout()) 71 | ro.SetIOTimeout(1212) 72 | require.EqualValues(t, 1212, ro.GetIOTimeout()) 73 | 74 | require.False(t, ro.IsAsyncIO()) 75 | ro.SetAsyncIO(true) 76 | require.True(t, ro.IsAsyncIO()) 77 | 78 | ro.SetAutoReadaheadSize(true) 79 | } 80 | -------------------------------------------------------------------------------- /options_restore.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | // RestoreOptions captures the options to be used during 8 | // restoration of a backup. 9 | type RestoreOptions struct { 10 | c *C.rocksdb_restore_options_t 11 | } 12 | 13 | // NewRestoreOptions creates a RestoreOptions instance. 14 | func NewRestoreOptions() *RestoreOptions { 15 | return &RestoreOptions{ 16 | c: C.rocksdb_restore_options_create(), 17 | } 18 | } 19 | 20 | // SetKeepLogFiles is used to set or unset the keep_log_files option 21 | // If true, restore won't overwrite the existing log files in wal_dir. It will 22 | // also move all log files from archive directory to wal_dir. 23 | // By default, this is false. 24 | func (ro *RestoreOptions) SetKeepLogFiles(v int) { 25 | C.rocksdb_restore_options_set_keep_log_files(ro.c, C.int(v)) 26 | } 27 | 28 | // Destroy destroys this RestoreOptions instance. 29 | func (ro *RestoreOptions) Destroy() { 30 | C.rocksdb_restore_options_destroy(ro.c) 31 | ro.c = nil 32 | } 33 | -------------------------------------------------------------------------------- /options_restore_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import "testing" 4 | 5 | func TestRestoreOption(t *testing.T) { 6 | t.Parallel() 7 | 8 | ro := NewRestoreOptions() 9 | defer ro.Destroy() 10 | 11 | ro.SetKeepLogFiles(123) 12 | } 13 | -------------------------------------------------------------------------------- /options_transaction.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // TransactionOptions represent all of the available options options for 7 | // a transaction on the database. 8 | type TransactionOptions struct { 9 | c *C.rocksdb_transaction_options_t 10 | } 11 | 12 | // NewDefaultTransactionOptions creates a default TransactionOptions object. 13 | func NewDefaultTransactionOptions() *TransactionOptions { 14 | return newNativeTransactionOptions(C.rocksdb_transaction_options_create()) 15 | } 16 | 17 | // NewNativeTransactionOptions creates a TransactionOptions object. 18 | func newNativeTransactionOptions(c *C.rocksdb_transaction_options_t) *TransactionOptions { 19 | return &TransactionOptions{c: c} 20 | } 21 | 22 | // SetSetSnapshot to true is the same as calling 23 | // Transaction::SetSnapshot(). 24 | func (opts *TransactionOptions) SetSetSnapshot(value bool) { 25 | C.rocksdb_transaction_options_set_set_snapshot(opts.c, boolToChar(value)) 26 | } 27 | 28 | // SetDeadlockDetect to true means that before acquiring locks, this transaction will 29 | // check if doing so will cause a deadlock. If so, it will return with 30 | // Status::Busy. The user should retry their transaction. 31 | func (opts *TransactionOptions) SetDeadlockDetect(value bool) { 32 | C.rocksdb_transaction_options_set_deadlock_detect(opts.c, boolToChar(value)) 33 | } 34 | 35 | // SetLockTimeout positive, specifies the wait timeout in milliseconds when 36 | // a transaction attempts to lock a key. 37 | // If 0, no waiting is done if a lock cannot instantly be acquired. 38 | // If negative, TransactionDBOptions::transaction_lock_timeout will be used 39 | func (opts *TransactionOptions) SetLockTimeout(lockTimeout int64) { 40 | C.rocksdb_transaction_options_set_lock_timeout(opts.c, C.int64_t(lockTimeout)) 41 | } 42 | 43 | // SetExpiration sets the Expiration duration in milliseconds. 44 | // If non-negative, transactions that last longer than this many milliseconds will fail to commit. 45 | // If not set, a forgotten transaction that is never committed, rolled back, or deleted 46 | // will never relinquish any locks it holds. This could prevent keys from 47 | // being written by other writers. 48 | func (opts *TransactionOptions) SetExpiration(expiration int64) { 49 | C.rocksdb_transaction_options_set_expiration(opts.c, C.int64_t(expiration)) 50 | } 51 | 52 | // SetDeadlockDetectDepth sets the number of traversals to make during deadlock detection. 53 | func (opts *TransactionOptions) SetDeadlockDetectDepth(depth int64) { 54 | C.rocksdb_transaction_options_set_deadlock_detect_depth(opts.c, C.int64_t(depth)) 55 | } 56 | 57 | // SetMaxWriteBatchSize sets the maximum number of bytes used for the write batch. 0 means no limit. 58 | func (opts *TransactionOptions) SetMaxWriteBatchSize(size uint64) { 59 | C.rocksdb_transaction_options_set_max_write_batch_size(opts.c, C.size_t(size)) 60 | } 61 | 62 | // SetSkipPrepare skips prepare phase. 63 | func (opts *TransactionOptions) SetSkipPrepare(skip bool) { 64 | C.rocksdb_transaction_options_set_skip_prepare(opts.c, boolToChar(skip)) 65 | } 66 | 67 | // Destroy deallocates the TransactionOptions object. 68 | func (opts *TransactionOptions) Destroy() { 69 | C.rocksdb_transaction_options_destroy(opts.c) 70 | opts.c = nil 71 | } 72 | 73 | // OptimisticTransactionOptions represent all of the available options options for 74 | // a optimistic transaction on the database. 75 | type OptimisticTransactionOptions struct { 76 | c *C.rocksdb_optimistictransaction_options_t 77 | } 78 | 79 | // NewDefaultOptimisticTransactionOptions creates a default TransactionOptions object. 80 | func NewDefaultOptimisticTransactionOptions() *OptimisticTransactionOptions { 81 | return newNativeOptimisticTransactionOptions(C.rocksdb_optimistictransaction_options_create()) 82 | } 83 | 84 | // NewNativeOptimisticTransactionOptions creates a OptimisticTransactionOptions object. 85 | func newNativeOptimisticTransactionOptions(c *C.rocksdb_optimistictransaction_options_t) *OptimisticTransactionOptions { 86 | return &OptimisticTransactionOptions{c: c} 87 | } 88 | 89 | // SetSetSnapshot to true is the same as calling 90 | // Transaction::SetSnapshot(). 91 | func (opts *OptimisticTransactionOptions) SetSetSnapshot(value bool) { 92 | C.rocksdb_optimistictransaction_options_set_set_snapshot(opts.c, boolToChar(value)) 93 | } 94 | 95 | // Destroy deallocates the TransactionOptions object. 96 | func (opts *OptimisticTransactionOptions) Destroy() { 97 | C.rocksdb_optimistictransaction_options_destroy(opts.c) 98 | opts.c = nil 99 | } 100 | -------------------------------------------------------------------------------- /options_transactiondb.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // TransactionDBOptions represent all of the available options when opening a transactional database 7 | // with OpenTransactionDb. 8 | type TransactionDBOptions struct { 9 | c *C.rocksdb_transactiondb_options_t 10 | } 11 | 12 | // NewDefaultTransactionDBOptions creates a default TransactionDBOptions object. 13 | func NewDefaultTransactionDBOptions() *TransactionDBOptions { 14 | return newNativeTransactionDBOptions(C.rocksdb_transactiondb_options_create()) 15 | } 16 | 17 | // NewNativeTransactionDBOptions creates a TransactionDBOptions from native object. 18 | func newNativeTransactionDBOptions(c *C.rocksdb_transactiondb_options_t) *TransactionDBOptions { 19 | return &TransactionDBOptions{c: c} 20 | } 21 | 22 | // SetMaxNumLocks sets the maximum number of keys that can be locked at the same time 23 | // per column family. 24 | // If the number of locked keys is greater than max_num_locks, transaction 25 | // writes (or GetForUpdate) will return an error. 26 | // If this value is not positive, no limit will be enforced. 27 | func (opts *TransactionDBOptions) SetMaxNumLocks(maxNumLocks int64) { 28 | C.rocksdb_transactiondb_options_set_max_num_locks(opts.c, C.int64_t(maxNumLocks)) 29 | } 30 | 31 | // SetNumStripes sets the concurrency level. 32 | // Increasing this value will increase the concurrency by dividing the lock 33 | // table (per column family) into more sub-tables, each with their own 34 | // separate 35 | // mutex. 36 | func (opts *TransactionDBOptions) SetNumStripes(numStripes uint64) { 37 | C.rocksdb_transactiondb_options_set_num_stripes(opts.c, C.size_t(numStripes)) 38 | } 39 | 40 | // SetTransactionLockTimeout if positive, specifies the default wait timeout in milliseconds when 41 | // a transaction attempts to lock a key if not specified by 42 | // TransactionOptions::lock_timeout. 43 | // 44 | // If 0, no waiting is done if a lock cannot instantly be acquired. 45 | // If negative, there is no timeout. Not using a timeout is not recommended 46 | // as it can lead to deadlocks. Currently, there is no deadlock-detection to 47 | // recover from a deadlock. 48 | func (opts *TransactionDBOptions) SetTransactionLockTimeout(txnLockTimeout int64) { 49 | C.rocksdb_transactiondb_options_set_transaction_lock_timeout(opts.c, C.int64_t(txnLockTimeout)) 50 | } 51 | 52 | // SetDefaultLockTimeout if posititve, specifies the wait timeout in milliseconds when writing a key 53 | // OUTSIDE of a transaction (ie by calling DB::Put(),Merge(),Delete(),Write() 54 | // directly). 55 | // If 0, no waiting is done if a lock cannot instantly be acquired. 56 | // If negative, there is no timeout and will block indefinitely when acquiring 57 | // a lock. 58 | // 59 | // Not using a timeout can lead to deadlocks. Currently, there 60 | // is no deadlock-detection to recover from a deadlock. While DB writes 61 | // cannot deadlock with other DB writes, they can deadlock with a transaction. 62 | // A negative timeout should only be used if all transactions have a small 63 | // expiration set. 64 | func (opts *TransactionDBOptions) SetDefaultLockTimeout(defaultLockTimeout int64) { 65 | C.rocksdb_transactiondb_options_set_default_lock_timeout(opts.c, C.int64_t(defaultLockTimeout)) 66 | } 67 | 68 | // Destroy deallocates the TransactionDBOptions object. 69 | func (opts *TransactionDBOptions) Destroy() { 70 | C.rocksdb_transactiondb_options_destroy(opts.c) 71 | opts.c = nil 72 | } 73 | -------------------------------------------------------------------------------- /options_wait_for_compact.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | type WaitForCompactOptions struct { 7 | p *C.rocksdb_wait_for_compact_options_t 8 | } 9 | 10 | func NewWaitForCompactOptions() *WaitForCompactOptions { 11 | return &WaitForCompactOptions{ 12 | p: C.rocksdb_wait_for_compact_options_create(), 13 | } 14 | } 15 | 16 | // Destroy the object. 17 | func (w *WaitForCompactOptions) Destroy() { 18 | C.rocksdb_wait_for_compact_options_destroy(w.p) 19 | w.p = nil 20 | } 21 | 22 | // SetAbortOnPause toggles the abort_on_pause flag, to abort waiting in case of 23 | // a pause (PauseBackgroundWork() called). 24 | // 25 | // - If true, Status::Aborted will be returned immediately. 26 | // - If false, ContinueBackgroundWork() must be called to resume the background jobs. 27 | // 28 | // Otherwise, jobs that were queued, but not scheduled yet may never finish 29 | // and WaitForCompact() may wait indefinitely (if timeout is set, it will 30 | // expire and return Status::TimedOut). 31 | func (w *WaitForCompactOptions) SetAbortOnPause(v bool) { 32 | C.rocksdb_wait_for_compact_options_set_abort_on_pause(w.p, boolToChar(v)) 33 | } 34 | 35 | // IsAbortOnPause checks if abort_on_pause flag is on. 36 | func (w *WaitForCompactOptions) AbortOnPause() bool { 37 | return charToBool(C.rocksdb_wait_for_compact_options_get_abort_on_pause(w.p)) 38 | } 39 | 40 | // SetFlush toggles the "flush" flag to flush all column families before starting to wait. 41 | func (w *WaitForCompactOptions) SetFlush(v bool) { 42 | C.rocksdb_wait_for_compact_options_set_flush(w.p, boolToChar(v)) 43 | } 44 | 45 | // IsFlush checks if "flush" flag is on. 46 | func (w *WaitForCompactOptions) Flush() bool { 47 | return charToBool(C.rocksdb_wait_for_compact_options_get_flush(w.p)) 48 | } 49 | 50 | // SetCloseDB toggles the "close_db" flag to call Close() after waiting is done. 51 | // By the time Close() is called here, there should be no background jobs in progress 52 | // and no new background jobs should be added. 53 | // 54 | // DB may not have been closed if Close() returned Aborted status due to unreleased snapshots 55 | // in the system. 56 | func (w *WaitForCompactOptions) SetCloseDB(v bool) { 57 | C.rocksdb_wait_for_compact_options_set_close_db(w.p, boolToChar(v)) 58 | } 59 | 60 | // CloseDB checks if "close_db" flag is on. 61 | func (w *WaitForCompactOptions) CloseDB() bool { 62 | return charToBool(C.rocksdb_wait_for_compact_options_get_close_db(w.p)) 63 | } 64 | 65 | // SetTimeout in microseconds for waiting for compaction to complete. 66 | // Status::TimedOut will be returned if timeout expires. 67 | // when timeout == 0, WaitForCompact() will wait as long as there's background 68 | // work to finish. 69 | func (w *WaitForCompactOptions) SetTimeout(microseconds uint64) { 70 | C.rocksdb_wait_for_compact_options_set_timeout(w.p, C.uint64_t(microseconds)) 71 | } 72 | 73 | // GetTimeout in microseconds. 74 | func (w *WaitForCompactOptions) GetTimeout() uint64 { 75 | return uint64(C.rocksdb_wait_for_compact_options_get_timeout(w.p)) 76 | } 77 | -------------------------------------------------------------------------------- /options_wait_for_compact_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestWaitForCompactOptions(t *testing.T) { 10 | t.Parallel() 11 | 12 | opts := NewWaitForCompactOptions() 13 | defer opts.Destroy() 14 | 15 | require.False(t, opts.AbortOnPause()) 16 | opts.SetAbortOnPause(true) 17 | require.True(t, opts.AbortOnPause()) 18 | 19 | require.False(t, opts.Flush()) 20 | opts.SetFlush(true) 21 | require.True(t, opts.Flush()) 22 | 23 | require.False(t, opts.CloseDB()) 24 | opts.SetCloseDB(true) 25 | require.True(t, opts.CloseDB()) 26 | 27 | require.EqualValues(t, 0, opts.GetTimeout()) 28 | opts.SetTimeout(1234) 29 | require.EqualValues(t, 1234, opts.GetTimeout()) 30 | } 31 | -------------------------------------------------------------------------------- /options_write.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // WriteOptions represent all of the available options when writing to a 7 | // database. 8 | type WriteOptions struct { 9 | c *C.rocksdb_writeoptions_t 10 | } 11 | 12 | // NewDefaultWriteOptions creates a default WriteOptions object. 13 | func NewDefaultWriteOptions() *WriteOptions { 14 | return newNativeWriteOptions(C.rocksdb_writeoptions_create()) 15 | } 16 | 17 | // NewNativeWriteOptions creates a WriteOptions object. 18 | func newNativeWriteOptions(c *C.rocksdb_writeoptions_t) *WriteOptions { 19 | return &WriteOptions{c: c} 20 | } 21 | 22 | // SetSync sets the sync mode. If true, the write will be flushed 23 | // from the operating system buffer cache before the write is considered complete. 24 | // If this flag is true, writes will be slower. 25 | // 26 | // Default: false 27 | func (opts *WriteOptions) SetSync(value bool) { 28 | C.rocksdb_writeoptions_set_sync(opts.c, boolToChar(value)) 29 | } 30 | 31 | // IsSync returns if sync mode is turned on. 32 | func (opts *WriteOptions) IsSync() bool { 33 | return charToBool(C.rocksdb_writeoptions_get_sync(opts.c)) 34 | } 35 | 36 | // DisableWAL sets whether WAL should be active or not. 37 | // If true, writes will not first go to the write ahead log, 38 | // and the write may got lost after a crash. 39 | // 40 | // Default: false 41 | func (opts *WriteOptions) DisableWAL(value bool) { 42 | C.rocksdb_writeoptions_disable_WAL(opts.c, C.int(boolToChar(value))) 43 | } 44 | 45 | // IsDisableWAL returns if we turned on DisableWAL flag for writing. 46 | func (opts *WriteOptions) IsDisableWAL() bool { 47 | return charToBool(C.rocksdb_writeoptions_get_disable_WAL(opts.c)) 48 | } 49 | 50 | // SetIgnoreMissingColumnFamilies if true and if user is trying to write 51 | // to column families that don't exist (they were dropped), ignore the 52 | // write (don't return an error). If there are multiple writes in a WriteBatch, 53 | // other writes will succeed. 54 | // 55 | // Default: false 56 | func (opts *WriteOptions) SetIgnoreMissingColumnFamilies(value bool) { 57 | C.rocksdb_writeoptions_set_ignore_missing_column_families(opts.c, boolToChar(value)) 58 | } 59 | 60 | // IgnoreMissingColumnFamilies returns the setting for ignoring missing column famlies. 61 | // 62 | // If true and if user is trying to write 63 | // to column families that don't exist (they were dropped), ignore the 64 | // write (don't return an error). If there are multiple writes in a WriteBatch, 65 | // other writes will succeed. 66 | func (opts *WriteOptions) IgnoreMissingColumnFamilies() bool { 67 | return charToBool(C.rocksdb_writeoptions_get_ignore_missing_column_families(opts.c)) 68 | } 69 | 70 | // SetNoSlowdown if true and we need to wait or sleep for the write request, fails 71 | // immediately with Status::Incomplete(). 72 | // 73 | // Default: false 74 | func (opts *WriteOptions) SetNoSlowdown(value bool) { 75 | C.rocksdb_writeoptions_set_no_slowdown(opts.c, boolToChar(value)) 76 | } 77 | 78 | // IsNoSlowdown returns no_slow_down setting. 79 | func (opts *WriteOptions) IsNoSlowdown() bool { 80 | return charToBool(C.rocksdb_writeoptions_get_no_slowdown(opts.c)) 81 | } 82 | 83 | // SetLowPri if true, this write request is of lower priority if compaction is 84 | // behind. In this case, no_slowdown = true, the request will be cancelled 85 | // immediately with Status::Incomplete() returned. Otherwise, it will be 86 | // slowed down. The slowdown value is determined by RocksDB to guarantee 87 | // it introduces minimum impacts to high priority writes. 88 | // 89 | // Default: false 90 | func (opts *WriteOptions) SetLowPri(value bool) { 91 | C.rocksdb_writeoptions_set_low_pri(opts.c, boolToChar(value)) 92 | } 93 | 94 | // IsLowPri returns if the write request is of lower priority if compaction is behind. 95 | func (opts *WriteOptions) IsLowPri() bool { 96 | return charToBool(C.rocksdb_writeoptions_get_low_pri(opts.c)) 97 | } 98 | 99 | // SetMemtableInsertHintPerBatch if true, this writebatch will maintain the last insert positions of each 100 | // memtable as hints in concurrent write. It can improve write performance 101 | // in concurrent writes if keys in one writebatch are sequential. In 102 | // non-concurrent writes (when concurrent_memtable_writes is false) this 103 | // option will be ignored. 104 | // 105 | // Default: false 106 | func (opts *WriteOptions) SetMemtableInsertHintPerBatch(value bool) { 107 | C.rocksdb_writeoptions_set_memtable_insert_hint_per_batch(opts.c, boolToChar(value)) 108 | } 109 | 110 | // MemtableInsertHintPerBatch returns if this writebatch will maintain the last insert positions of each 111 | // memtable as hints in concurrent write. 112 | func (opts *WriteOptions) MemtableInsertHintPerBatch() bool { 113 | return charToBool(C.rocksdb_writeoptions_get_memtable_insert_hint_per_batch(opts.c)) 114 | } 115 | 116 | // Destroy deallocates the WriteOptions object. 117 | func (opts *WriteOptions) Destroy() { 118 | C.rocksdb_writeoptions_destroy(opts.c) 119 | opts.c = nil 120 | } 121 | -------------------------------------------------------------------------------- /options_write_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestWriteOptions(t *testing.T) { 10 | t.Parallel() 11 | 12 | wo := NewDefaultWriteOptions() 13 | defer wo.Destroy() 14 | 15 | require.EqualValues(t, false, wo.IsSync()) 16 | wo.SetSync(true) 17 | require.EqualValues(t, true, wo.IsSync()) 18 | 19 | require.EqualValues(t, false, wo.IsDisableWAL()) 20 | wo.DisableWAL(true) 21 | require.EqualValues(t, true, wo.IsDisableWAL()) 22 | 23 | require.EqualValues(t, false, wo.IgnoreMissingColumnFamilies()) 24 | wo.SetIgnoreMissingColumnFamilies(true) 25 | require.EqualValues(t, true, wo.IgnoreMissingColumnFamilies()) 26 | 27 | require.EqualValues(t, false, wo.IsNoSlowdown()) 28 | wo.SetNoSlowdown(true) 29 | require.EqualValues(t, true, wo.IsNoSlowdown()) 30 | 31 | require.EqualValues(t, false, wo.IsLowPri()) 32 | wo.SetLowPri(true) 33 | require.EqualValues(t, true, wo.IsLowPri()) 34 | 35 | require.EqualValues(t, false, wo.MemtableInsertHintPerBatch()) 36 | wo.SetMemtableInsertHintPerBatch(true) 37 | require.EqualValues(t, true, wo.MemtableInsertHintPerBatch()) 38 | } 39 | -------------------------------------------------------------------------------- /perf_context.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import ( 7 | "unsafe" 8 | ) 9 | 10 | // PerfContext a thread local context for gathering performance counter efficiently 11 | // and transparently. 12 | type PerfContext struct { 13 | c *C.rocksdb_perfcontext_t 14 | } 15 | 16 | // NewPerfContext returns new perf context. 17 | func NewPerfContext() *PerfContext { 18 | return &PerfContext{ 19 | c: C.rocksdb_perfcontext_create(), 20 | } 21 | } 22 | 23 | // Destroy perf context object. 24 | func (ctx *PerfContext) Destroy() { 25 | C.rocksdb_perfcontext_destroy(ctx.c) 26 | ctx.c = nil 27 | } 28 | 29 | // Reset context. 30 | func (ctx *PerfContext) Reset() { 31 | C.rocksdb_perfcontext_reset(ctx.c) 32 | } 33 | 34 | // Report with exclusion of zero counter. 35 | func (ctx *PerfContext) Report(excludeZeroCounters bool) (value string) { 36 | cValue := C.rocksdb_perfcontext_report(ctx.c, boolToChar(excludeZeroCounters)) 37 | value = C.GoString(cValue) 38 | C.free(unsafe.Pointer(cValue)) 39 | return 40 | } 41 | 42 | // Metric returns value of a metric by its id. 43 | // 44 | // Id is one of: 45 | // 46 | // enum { 47 | // rocksdb_user_key_comparison_count = 0, 48 | // rocksdb_block_cache_hit_count, 49 | // rocksdb_block_read_count, 50 | // rocksdb_block_read_byte, 51 | // rocksdb_block_read_time, 52 | // rocksdb_block_checksum_time, 53 | // rocksdb_block_decompress_time, 54 | // rocksdb_get_read_bytes, 55 | // rocksdb_multiget_read_bytes, 56 | // rocksdb_iter_read_bytes, 57 | // rocksdb_internal_key_skipped_count, 58 | // rocksdb_internal_delete_skipped_count, 59 | // rocksdb_internal_recent_skipped_count, 60 | // rocksdb_internal_merge_count, 61 | // rocksdb_get_snapshot_time, 62 | // rocksdb_get_from_memtable_time, 63 | // rocksdb_get_from_memtable_count, 64 | // rocksdb_get_post_process_time, 65 | // rocksdb_get_from_output_files_time, 66 | // rocksdb_seek_on_memtable_time, 67 | // rocksdb_seek_on_memtable_count, 68 | // rocksdb_next_on_memtable_count, 69 | // rocksdb_prev_on_memtable_count, 70 | // rocksdb_seek_child_seek_time, 71 | // rocksdb_seek_child_seek_count, 72 | // rocksdb_seek_min_heap_time, 73 | // rocksdb_seek_max_heap_time, 74 | // rocksdb_seek_internal_seek_time, 75 | // rocksdb_find_next_user_entry_time, 76 | // rocksdb_write_wal_time, 77 | // rocksdb_write_memtable_time, 78 | // rocksdb_write_delay_time, 79 | // rocksdb_write_pre_and_post_process_time, 80 | // rocksdb_db_mutex_lock_nanos, 81 | // rocksdb_db_condition_wait_nanos, 82 | // rocksdb_merge_operator_time_nanos, 83 | // rocksdb_read_index_block_nanos, 84 | // rocksdb_read_filter_block_nanos, 85 | // rocksdb_new_table_block_iter_nanos, 86 | // rocksdb_new_table_iterator_nanos, 87 | // rocksdb_block_seek_nanos, 88 | // rocksdb_find_table_nanos, 89 | // rocksdb_bloom_memtable_hit_count, 90 | // rocksdb_bloom_memtable_miss_count, 91 | // rocksdb_bloom_sst_hit_count, 92 | // rocksdb_bloom_sst_miss_count, 93 | // rocksdb_key_lock_wait_time, 94 | // rocksdb_key_lock_wait_count, 95 | // rocksdb_env_new_sequential_file_nanos, 96 | // rocksdb_env_new_random_access_file_nanos, 97 | // rocksdb_env_new_writable_file_nanos, 98 | // rocksdb_env_reuse_writable_file_nanos, 99 | // rocksdb_env_new_random_rw_file_nanos, 100 | // rocksdb_env_new_directory_nanos, 101 | // rocksdb_env_file_exists_nanos, 102 | // rocksdb_env_get_children_nanos, 103 | // rocksdb_env_get_children_file_attributes_nanos, 104 | // rocksdb_env_delete_file_nanos, 105 | // rocksdb_env_create_dir_nanos, 106 | // rocksdb_env_create_dir_if_missing_nanos, 107 | // rocksdb_env_delete_dir_nanos, 108 | // rocksdb_env_get_file_size_nanos, 109 | // rocksdb_env_get_file_modification_time_nanos, 110 | // rocksdb_env_rename_file_nanos, 111 | // rocksdb_env_link_file_nanos, 112 | // rocksdb_env_lock_file_nanos, 113 | // rocksdb_env_unlock_file_nanos, 114 | // rocksdb_env_new_logger_nanos, 115 | // rocksdb_number_async_seek, 116 | // rocksdb_blob_cache_hit_count, 117 | // rocksdb_blob_read_count, 118 | // rocksdb_blob_read_byte, 119 | // rocksdb_blob_read_time, 120 | // rocksdb_blob_checksum_time, 121 | // rocksdb_blob_decompress_time, 122 | // rocksdb_total_metric_count = 77 123 | // }; 124 | func (ctx *PerfContext) Metric(id int) uint64 { 125 | value := C.rocksdb_perfcontext_metric(ctx.c, C.int(id)) 126 | return uint64(value) 127 | } 128 | -------------------------------------------------------------------------------- /perf_level.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | // PerfLevel indicates how much perf stats to collect. Affects perf_context and iostats_context. 7 | type PerfLevel int 8 | 9 | const ( 10 | // KUninitialized indicates unknown setting 11 | KUninitialized PerfLevel = 0 12 | // KDisable disables perf stats 13 | KDisable PerfLevel = 1 14 | // KEnableCount enables only count stats 15 | KEnableCount PerfLevel = 2 16 | // KEnableTimeExceptForMutex other than count stats, 17 | // also enable time stats except for mutexes 18 | KEnableTimeExceptForMutex PerfLevel = 3 19 | // KEnableTimeAndCPUTimeExceptForMutex other than time, 20 | // also measure CPU time counters. Still don't measure 21 | // time (neither wall time nor CPU time) for mutexes. 22 | KEnableTimeAndCPUTimeExceptForMutex PerfLevel = 4 23 | // KEnableTime enables count and time stats 24 | KEnableTime PerfLevel = 5 25 | // KOutOfBounds N.B. Must always be the last value! 26 | KOutOfBounds PerfLevel = 6 27 | ) 28 | 29 | // SetPerfLevel sets the perf stats level for current thread. 30 | func SetPerfLevel(level PerfLevel) { 31 | C.rocksdb_set_perf_level(C.int(level)) 32 | } 33 | -------------------------------------------------------------------------------- /ratelimiter.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | type RateLimiterMode int 8 | 9 | const ( 10 | RateLimiterModeReadsOnly RateLimiterMode = iota 11 | RateLimiterModeWritesOnly 12 | RateLimiterModeAllIo 13 | ) 14 | 15 | // RateLimiter is used to control write rate of flush and 16 | // compaction. 17 | type RateLimiter struct { 18 | c *C.rocksdb_ratelimiter_t 19 | } 20 | 21 | // NewRateLimiter creates a RateLimiter object, which can be shared among RocksDB instances to 22 | // control write rate of flush and compaction. 23 | // 24 | // @rate_bytes_per_sec: this is the only parameter you want to set most of the 25 | // time. It controls the total write rate of compaction and flush in bytes per 26 | // second. Currently, RocksDB does not enforce rate limit for anything other 27 | // than flush and compaction, e.g. write to WAL. 28 | // 29 | // @refill_period_us: this controls how often tokens are refilled. For example, 30 | // when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to 31 | // 100ms, then 1MB is refilled every 100ms internally. Larger value can lead to 32 | // burstier writes while smaller value introduces more CPU overhead. 33 | // The default should work for most cases. 34 | // 35 | // @fairness: RateLimiter accepts high-pri requests and low-pri requests. 36 | // A low-pri request is usually blocked in favor of hi-pri request. Currently, 37 | // RocksDB assigns low-pri to request from compaction and high-pri to request 38 | // from flush. Low-pri requests can get blocked if flush requests come in 39 | // continuously. This fairness parameter grants low-pri requests permission by 40 | // 1/fairness chance even though high-pri requests exist to avoid starvation. 41 | // You should be good by leaving it at default 10. 42 | func NewRateLimiter(rateBytesPerSec, refillPeriodMicros int64, fairness int32) *RateLimiter { 43 | cR := C.rocksdb_ratelimiter_create( 44 | C.int64_t(rateBytesPerSec), 45 | C.int64_t(refillPeriodMicros), 46 | C.int32_t(fairness), 47 | ) 48 | return newNativeRateLimiter(cR) 49 | } 50 | 51 | // NewAutoTunedRateLimiter similar to NewRateLimiter, enables dynamic adjustment of rate 52 | // limit within the range `[rate_bytes_per_sec / 20, rate_bytes_per_sec]`, according to 53 | // the recent demand for background I/O. 54 | func NewAutoTunedRateLimiter(rateBytesPerSec, refillPeriodMicros int64, fairness int32) *RateLimiter { 55 | cR := C.rocksdb_ratelimiter_create_auto_tuned( 56 | C.int64_t(rateBytesPerSec), 57 | C.int64_t(refillPeriodMicros), 58 | C.int32_t(fairness), 59 | ) 60 | return newNativeRateLimiter(cR) 61 | } 62 | 63 | // NewGenericRateLimiter creates a RateLimiter object, which can be shared among RocksDB instances to 64 | // control write rate of flush and compaction. 65 | // 66 | // @rate_bytes_per_sec: this is the only parameter you want to set most of the 67 | // time. It controls the total write rate of compaction and flush in bytes per 68 | // second. Currently, RocksDB does not enforce rate limit for anything other 69 | // than flush and compaction, e.g. write to WAL. 70 | // 71 | // @refill_period_us: this controls how often tokens are refilled. For example, 72 | // when rate_bytes_per_sec is set to 10MB/s and refill_period_us is set to 73 | // 100ms, then 1MB is refilled every 100ms internally. Larger value can lead to 74 | // burstier writes while smaller value introduces more CPU overhead. 75 | // The default should work for most cases. 76 | // 77 | // @fairness: RateLimiter accepts high-pri requests and low-pri requests. 78 | // A low-pri request is usually blocked in favor of hi-pri request. Currently, 79 | // RocksDB assigns low-pri to request from compaction and high-pri to request 80 | // from flush. Low-pri requests can get blocked if flush requests come in 81 | // continuously. This fairness parameter grants low-pri requests permission by 82 | // 1/fairness chance even though high-pri requests exist to avoid starvation. 83 | // You should be good by leaving it at default 10. 84 | // 85 | // @mode: Mode indicates which types of operations count against the limit. 86 | // 87 | // @auto_tuned: Enables dynamic adjustment of rate limit within the range 88 | // `[rate_bytes_per_sec / 20, rate_bytes_per_sec]`, according to 89 | // the recent demand for background I/O. 90 | func NewGenericRateLimiter( 91 | rateBytesPerSec, refillPeriodMicros int64, fairness int32, 92 | mode RateLimiterMode, autoTuned bool, 93 | ) *RateLimiter { 94 | cR := C.rocksdb_ratelimiter_create_with_mode( 95 | C.int64_t(rateBytesPerSec), 96 | C.int64_t(refillPeriodMicros), 97 | C.int32_t(fairness), 98 | C.int(mode), 99 | C.bool(autoTuned), 100 | ) 101 | return newNativeRateLimiter(cR) 102 | } 103 | 104 | // NewNativeRateLimiter creates a native RateLimiter object. 105 | func newNativeRateLimiter(c *C.rocksdb_ratelimiter_t) *RateLimiter { 106 | return &RateLimiter{c: c} 107 | } 108 | 109 | // Destroy deallocates the RateLimiter object. 110 | func (r *RateLimiter) Destroy() { 111 | C.rocksdb_ratelimiter_destroy(r.c) 112 | r.c = nil 113 | } 114 | -------------------------------------------------------------------------------- /slice.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // Slice is used as a wrapper for non-copy values 9 | type Slice struct { 10 | data *C.char 11 | size C.size_t 12 | freed bool 13 | } 14 | 15 | // Slices is collection of Slice. 16 | type Slices []*Slice 17 | 18 | // Destroy free slices. 19 | func (slices Slices) Destroy() { 20 | for _, s := range slices { 21 | s.Free() 22 | } 23 | } 24 | 25 | // NewSlice returns a slice with the given data. 26 | func NewSlice(data *C.char, size C.size_t) *Slice { 27 | return &Slice{data, size, false} 28 | } 29 | 30 | // Exists returns if underlying data exists. 31 | func (s *Slice) Exists() bool { 32 | return s.data != nil 33 | } 34 | 35 | // Data returns the data of the slice. If the key doesn't exist this will be a 36 | // nil slice. 37 | func (s *Slice) Data() []byte { 38 | if s.Exists() { 39 | return refCBytes(s.data, s.size) 40 | } 41 | 42 | return nil 43 | } 44 | 45 | // Size returns the size of the data. 46 | func (s *Slice) Size() int { 47 | return int(s.size) 48 | } 49 | 50 | // Free frees the slice data. 51 | func (s *Slice) Free() { 52 | if !s.freed { 53 | C.rocksdb_free(unsafe.Pointer(s.data)) 54 | s.data = nil 55 | s.freed = true 56 | } 57 | } 58 | 59 | type PinnableSlices []*PinnableSliceHandle 60 | 61 | func (s PinnableSlices) Destroy() { 62 | for _, s := range s { 63 | s.Destroy() 64 | } 65 | } 66 | 67 | // PinnableSliceHandle represents a handle to a PinnableSlice. 68 | type PinnableSliceHandle struct { 69 | c *C.rocksdb_pinnableslice_t 70 | } 71 | 72 | // NewNativePinnableSliceHandle creates a PinnableSliceHandle object. 73 | func newNativePinnableSliceHandle(c *C.rocksdb_pinnableslice_t) *PinnableSliceHandle { 74 | return &PinnableSliceHandle{c: c} 75 | } 76 | 77 | // Exists returns if underlying data exists. 78 | func (h *PinnableSliceHandle) Exists() bool { 79 | return h.c != nil 80 | } 81 | 82 | // Data returns the data of the slice. 83 | func (h *PinnableSliceHandle) Data() []byte { 84 | if h.Exists() { 85 | var cValLen C.size_t 86 | cValue := C.rocksdb_pinnableslice_value(h.c, &cValLen) 87 | return refCBytes(cValue, cValLen) 88 | } 89 | 90 | return nil 91 | } 92 | 93 | // Destroy calls the destructor of the underlying pinnable slice handle. 94 | func (h *PinnableSliceHandle) Destroy() { 95 | C.rocksdb_pinnableslice_destroy(h.c) 96 | h.c = nil 97 | } 98 | -------------------------------------------------------------------------------- /slice_transform.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // A SliceTransform can be used as a prefix extractor. 8 | type SliceTransform interface { 9 | // Transform a src in domain to a dst in the range. 10 | Transform(src []byte) []byte 11 | 12 | // Determine whether this is a valid src upon the function applies. 13 | InDomain(src []byte) bool 14 | 15 | // Determine whether dst=Transform(src) for some src. 16 | InRange(src []byte) bool 17 | 18 | // Return the name of this transformation. 19 | Name() string 20 | 21 | // Destroy underlying pointer/data 22 | Destroy() 23 | } 24 | 25 | // NewFixedPrefixTransform creates a new fixed prefix transform. 26 | func NewFixedPrefixTransform(prefixLen int) SliceTransform { 27 | return NewNativeSliceTransform(unsafe.Pointer(C.rocksdb_slicetransform_create_fixed_prefix(C.size_t(prefixLen)))) 28 | } 29 | 30 | // NewNoopPrefixTransform creates a new no-op prefix transform. 31 | func NewNoopPrefixTransform() SliceTransform { 32 | return NewNativeSliceTransform(unsafe.Pointer(C.rocksdb_slicetransform_create_noop())) 33 | } 34 | 35 | // NewNativeSliceTransform creates a SliceTransform object. 36 | func NewNativeSliceTransform(c unsafe.Pointer) SliceTransform { 37 | return &nativeSliceTransform{c: (*C.rocksdb_slicetransform_t)(c)} 38 | } 39 | 40 | type nativeSliceTransform struct { 41 | c *C.rocksdb_slicetransform_t 42 | } 43 | 44 | func (st *nativeSliceTransform) Transform(src []byte) []byte { return nil } 45 | func (st *nativeSliceTransform) InDomain(src []byte) bool { return false } 46 | func (st *nativeSliceTransform) InRange(src []byte) bool { return false } 47 | func (st *nativeSliceTransform) Name() string { return "" } 48 | func (st *nativeSliceTransform) Destroy() { 49 | C.rocksdb_slicetransform_destroy(st.c) 50 | st.c = nil 51 | } 52 | 53 | // Hold references to slice transforms. 54 | var sliceTransforms = NewCOWList() 55 | 56 | type sliceTransformWrapper struct { 57 | name *C.char 58 | sliceTransform SliceTransform 59 | } 60 | 61 | func registerSliceTransform(st SliceTransform) int { 62 | return sliceTransforms.Append(sliceTransformWrapper{C.CString(st.Name()), st}) 63 | } 64 | 65 | //export gorocksdb_slicetransform_transform 66 | func gorocksdb_slicetransform_transform(idx int, cKey *C.char, cKeyLen C.size_t, cDstLen *C.size_t) *C.char { 67 | key := refCBytes(cKey, cKeyLen) 68 | dst := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.Transform(key) 69 | *cDstLen = C.size_t(len(dst)) 70 | return cByteSlice(dst) 71 | } 72 | 73 | //export gorocksdb_slicetransform_in_domain 74 | func gorocksdb_slicetransform_in_domain(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { 75 | key := refCBytes(cKey, cKeyLen) 76 | inDomain := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InDomain(key) 77 | return boolToChar(inDomain) 78 | } 79 | 80 | //export gorocksdb_slicetransform_in_range 81 | func gorocksdb_slicetransform_in_range(idx int, cKey *C.char, cKeyLen C.size_t) C.uchar { 82 | key := refCBytes(cKey, cKeyLen) 83 | inRange := sliceTransforms.Get(idx).(sliceTransformWrapper).sliceTransform.InRange(key) 84 | return boolToChar(inRange) 85 | } 86 | 87 | //export gorocksdb_slicetransform_name 88 | func gorocksdb_slicetransform_name(idx int) *C.char { 89 | return sliceTransforms.Get(idx).(sliceTransformWrapper).name 90 | } 91 | -------------------------------------------------------------------------------- /slice_transform_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestSliceTransform(t *testing.T) { 10 | t.Parallel() 11 | 12 | db := newTestDB(t, func(opts *Options) { 13 | opts.SetPrefixExtractor(&testSliceTransform{}) 14 | }) 15 | defer db.Close() 16 | 17 | wo := NewDefaultWriteOptions() 18 | require.Nil(t, db.Put(wo, []byte("foo1"), []byte("foo"))) 19 | require.Nil(t, db.Put(wo, []byte("foo2"), []byte("foo"))) 20 | require.Nil(t, db.Put(wo, []byte("bar1"), []byte("bar"))) 21 | 22 | iter := db.NewIterator(NewDefaultReadOptions()) 23 | defer iter.Close() 24 | prefix := []byte("foo") 25 | numFound := 0 26 | for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { 27 | numFound++ 28 | } 29 | require.Nil(t, iter.Err()) 30 | require.EqualValues(t, numFound, 2) 31 | } 32 | 33 | func TestFixedPrefixTransformOpen(t *testing.T) { 34 | t.Parallel() 35 | 36 | db := newTestDB(t, func(opts *Options) { 37 | opts.SetPrefixExtractor(NewFixedPrefixTransform(3)) 38 | }) 39 | defer db.Close() 40 | } 41 | 42 | func TestNewNoopPrefixTransform(t *testing.T) { 43 | t.Parallel() 44 | 45 | db := newTestDB(t, func(opts *Options) { 46 | opts.SetPrefixExtractor(NewNoopPrefixTransform()) 47 | }) 48 | defer db.Close() 49 | } 50 | 51 | type testSliceTransform struct{} 52 | 53 | func (st *testSliceTransform) Name() string { return "grocksdb.test" } 54 | func (st *testSliceTransform) Transform(src []byte) []byte { return src[0:3] } 55 | func (st *testSliceTransform) InDomain(src []byte) bool { return len(src) >= 3 } 56 | func (st *testSliceTransform) InRange(src []byte) bool { return len(src) == 3 } 57 | func (st *testSliceTransform) Destroy() {} 58 | -------------------------------------------------------------------------------- /snapshot.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | import "unsafe" 6 | 7 | // Snapshot provides a consistent view of read operations in a DB. 8 | type Snapshot struct { 9 | c *C.rocksdb_snapshot_t 10 | } 11 | 12 | // NewNativeSnapshot creates a Snapshot object. 13 | func newNativeSnapshot(c *C.rocksdb_snapshot_t) *Snapshot { 14 | return &Snapshot{c: c} 15 | } 16 | 17 | // GetSequenceNumber gets sequence number of the Snapshot. 18 | func (snapshot *Snapshot) GetSequenceNumber() uint64 { 19 | return uint64(C.rocksdb_snapshot_get_sequence_number(snapshot.c)) 20 | } 21 | 22 | // Destroy deallocates the Snapshot object. 23 | func (snapshot *Snapshot) Destroy() { 24 | C.rocksdb_free(unsafe.Pointer(snapshot.c)) 25 | snapshot.c = nil 26 | } 27 | -------------------------------------------------------------------------------- /sst_file_writer.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // SSTFileWriter is used to create sst files that can be added to database later. 12 | // All keys in files generated by SstFileWriter will have sequence number = 0. 13 | type SSTFileWriter struct { 14 | c *C.rocksdb_sstfilewriter_t 15 | } 16 | 17 | // NewSSTFileWriter creates an SSTFileWriter object. 18 | func NewSSTFileWriter(opts *EnvOptions, dbOpts *Options) *SSTFileWriter { 19 | c := C.rocksdb_sstfilewriter_create(opts.c, dbOpts.c) 20 | return &SSTFileWriter{c: c} 21 | } 22 | 23 | // NewSSTFileWriterWithComparator creates an SSTFileWriter object with comparator. 24 | func NewSSTFileWriterWithComparator(opts *EnvOptions, dbOpts *Options, cmp *Comparator) *SSTFileWriter { 25 | cmp_ := unsafe.Pointer(cmp.c) 26 | return NewSSTFileWriterWithNativeComparator(opts, dbOpts, cmp_) 27 | } 28 | 29 | // NewSSTFileWriterWithNativeComparator creates an SSTFileWriter object with native comparator. 30 | func NewSSTFileWriterWithNativeComparator(opts *EnvOptions, dbOpts *Options, cmp unsafe.Pointer) *SSTFileWriter { 31 | cmp_ := (*C.rocksdb_comparator_t)(cmp) 32 | c := C.rocksdb_sstfilewriter_create_with_comparator(opts.c, dbOpts.c, cmp_) 33 | return &SSTFileWriter{c: c} 34 | } 35 | 36 | // Open prepares SstFileWriter to write into file located at "path". 37 | func (w *SSTFileWriter) Open(path string) (err error) { 38 | var ( 39 | cErr *C.char 40 | cPath = C.CString(path) 41 | ) 42 | 43 | C.rocksdb_sstfilewriter_open(w.c, cPath, &cErr) 44 | err = fromCError(cErr) 45 | 46 | C.free(unsafe.Pointer(cPath)) 47 | return 48 | } 49 | 50 | // Add adds key, value to currently opened file. 51 | // REQUIRES: key is after any previously added key according to comparator. 52 | func (w *SSTFileWriter) Add(key, value []byte) (err error) { 53 | cKey := refGoBytes(key) 54 | cValue := refGoBytes(value) 55 | var cErr *C.char 56 | C.rocksdb_sstfilewriter_add(w.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 57 | err = fromCError(cErr) 58 | return 59 | } 60 | 61 | // Put key, value to currently opened file. 62 | func (w *SSTFileWriter) Put(key, value []byte) (err error) { 63 | cKey := refGoBytes(key) 64 | cValue := refGoBytes(value) 65 | var cErr *C.char 66 | C.rocksdb_sstfilewriter_put(w.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 67 | err = fromCError(cErr) 68 | return 69 | } 70 | 71 | // Put key with timestamp, value to the currently opened file 72 | func (w *SSTFileWriter) PutWithTS(key, ts, value []byte) (err error) { 73 | cKey := refGoBytes(key) 74 | cValue := refGoBytes(value) 75 | cTs := refGoBytes(ts) 76 | var cErr *C.char 77 | C.rocksdb_sstfilewriter_put_with_ts(w.c, cKey, C.size_t(len(key)), cTs, C.size_t(len(ts)), cValue, C.size_t(len(value)), &cErr) 78 | err = fromCError(cErr) 79 | return 80 | } 81 | 82 | // Merge key, value to currently opened file. 83 | func (w *SSTFileWriter) Merge(key, value []byte) (err error) { 84 | cKey := refGoBytes(key) 85 | cValue := refGoBytes(value) 86 | var cErr *C.char 87 | C.rocksdb_sstfilewriter_merge(w.c, cKey, C.size_t(len(key)), cValue, C.size_t(len(value)), &cErr) 88 | err = fromCError(cErr) 89 | return 90 | } 91 | 92 | // Delete key from currently opened file. 93 | func (w *SSTFileWriter) Delete(key []byte) (err error) { 94 | cKey := refGoBytes(key) 95 | var cErr *C.char 96 | C.rocksdb_sstfilewriter_delete(w.c, cKey, C.size_t(len(key)), &cErr) 97 | err = fromCError(cErr) 98 | return 99 | } 100 | 101 | // DeleteWithTS deletes key with timestamp to the currently opened file 102 | func (w *SSTFileWriter) DeleteWithTS(key, ts []byte) (err error) { 103 | cKey := refGoBytes(key) 104 | cTs := refGoBytes(ts) 105 | var cErr *C.char 106 | C.rocksdb_sstfilewriter_delete_with_ts(w.c, cKey, C.size_t(len(key)), cTs, C.size_t(len(ts)), &cErr) 107 | err = fromCError(cErr) 108 | return 109 | } 110 | 111 | // DeleteRange deletes keys that are between [startKey, endKey) 112 | func (w *SSTFileWriter) DeleteRange(startKey, endKey []byte) (err error) { 113 | cStartKey := refGoBytes(startKey) 114 | cEndKey := refGoBytes(endKey) 115 | var cErr *C.char 116 | C.rocksdb_sstfilewriter_delete_range(w.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey)), &cErr) 117 | err = fromCError(cErr) 118 | return 119 | } 120 | 121 | // FileSize returns size of currently opened file. 122 | func (w *SSTFileWriter) FileSize() (size uint64) { 123 | C.rocksdb_sstfilewriter_file_size(w.c, (*C.uint64_t)(&size)) 124 | return 125 | } 126 | 127 | // Finish finishes writing to sst file and close file. 128 | func (w *SSTFileWriter) Finish() (err error) { 129 | var cErr *C.char 130 | C.rocksdb_sstfilewriter_finish(w.c, &cErr) 131 | err = fromCError(cErr) 132 | return 133 | } 134 | 135 | // Destroy destroys the SSTFileWriter object. 136 | func (w *SSTFileWriter) Destroy() { 137 | C.rocksdb_sstfilewriter_destroy(w.c) 138 | w.c = nil 139 | } 140 | -------------------------------------------------------------------------------- /testing_darwin_arm64.go: -------------------------------------------------------------------------------- 1 | //go:build testing 2 | 3 | package grocksdb 4 | 5 | // #cgo CFLAGS: -I${SRCDIR}/dist/darwin_arm64/include 6 | // #cgo CXXFLAGS: -I${SRCDIR}/dist/darwin_arm64/include 7 | // #cgo LDFLAGS: -L${SRCDIR}/dist/darwin_arm64/lib -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy 8 | import "C" 9 | -------------------------------------------------------------------------------- /testing_linux_amd64.go: -------------------------------------------------------------------------------- 1 | //go:build testing 2 | 3 | package grocksdb 4 | 5 | // #cgo CFLAGS: -I${SRCDIR}/dist/linux_amd64/include 6 | // #cgo CXXFLAGS: -I${SRCDIR}/dist/linux_amd64/include 7 | // #cgo LDFLAGS: -L${SRCDIR}/dist/linux_amd64/lib -L${SRCDIR}/dist/linux_amd64/lib64 -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy 8 | import "C" 9 | -------------------------------------------------------------------------------- /testing_linux_arm64.go: -------------------------------------------------------------------------------- 1 | //go:build testing 2 | 3 | package grocksdb 4 | 5 | // #cgo CFLAGS: -I${SRCDIR}/dist/linux_arm64/include 6 | // #cgo CXXFLAGS: -I${SRCDIR}/dist/linux_arm64/include 7 | // #cgo LDFLAGS: -L${SRCDIR}/dist/linux_arm64/lib -L${SRCDIR}/dist/linux_arm64/lib64 -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy 8 | import "C" 9 | -------------------------------------------------------------------------------- /transactiondb_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestOpenTransactionDb(t *testing.T) { 10 | t.Parallel() 11 | 12 | db := newTestTransactionDB(t, nil) 13 | defer db.Close() 14 | } 15 | 16 | func TestTransactionDBCRUD(t *testing.T) { 17 | t.Parallel() 18 | 19 | db := newTestTransactionDB(t, nil) 20 | defer db.Close() 21 | 22 | var ( 23 | givenKey = []byte("hello") 24 | givenVal1 = []byte("world1") 25 | givenVal2 = []byte("world2") 26 | givenTxnKey = []byte("hello2") 27 | givenTxnKey2 = []byte("hello3") 28 | givenTxnVal1 = []byte("whatawonderful") 29 | wo = NewDefaultWriteOptions() 30 | ro = NewDefaultReadOptions() 31 | to = NewDefaultTransactionOptions() 32 | ) 33 | 34 | // create 35 | require.Nil(t, db.Put(wo, givenKey, givenVal1)) 36 | 37 | // retrieve 38 | v1, err := db.Get(ro, givenKey) 39 | require.Nil(t, err) 40 | require.EqualValues(t, v1.Data(), givenVal1) 41 | v1.Free() 42 | 43 | // update 44 | require.Nil(t, db.Put(wo, givenKey, givenVal2)) 45 | v2, err := db.Get(ro, givenKey) 46 | require.Nil(t, err) 47 | require.EqualValues(t, v2.Data(), givenVal2) 48 | v2.Free() 49 | 50 | // delete 51 | require.Nil(t, db.Delete(wo, givenKey)) 52 | v3, err := db.Get(ro, givenKey) 53 | require.Nil(t, err) 54 | require.True(t, v3.Data() == nil) 55 | v3.Free() 56 | 57 | // transaction 58 | txn := db.TransactionBegin(wo, to, nil) 59 | defer txn.Destroy() 60 | // create 61 | require.Nil(t, txn.Put(givenTxnKey, givenTxnVal1)) 62 | v4, err := txn.Get(ro, givenTxnKey) 63 | require.Nil(t, err) 64 | require.EqualValues(t, v4.Data(), givenTxnVal1) 65 | v4.Free() 66 | 67 | require.Nil(t, txn.Commit()) 68 | v5, err := db.Get(ro, givenTxnKey) 69 | require.Nil(t, err) 70 | require.EqualValues(t, v5.Data(), givenTxnVal1) 71 | v5.Free() 72 | 73 | // transaction 74 | txn2 := db.TransactionBegin(wo, to, nil) 75 | defer txn2.Destroy() 76 | // create 77 | require.Nil(t, txn2.Put(givenTxnKey2, givenTxnVal1)) 78 | // rollback 79 | require.Nil(t, txn2.Rollback()) 80 | v6, err := txn2.Get(ro, givenTxnKey2) 81 | require.Nil(t, err) 82 | require.True(t, v6.Data() == nil) 83 | v6.Free() 84 | 85 | // transaction 86 | txn3 := db.TransactionBegin(wo, to, nil) 87 | defer txn3.Destroy() 88 | require.NoError(t, txn3.SetName("fake_name")) 89 | require.Equal(t, "fake_name", txn3.GetName()) 90 | // delete 91 | require.Nil(t, txn3.Prepare()) 92 | require.Nil(t, txn3.Delete(givenTxnKey)) 93 | 94 | wi := txn3.GetWriteBatchWI() 95 | require.EqualValues(t, 1, wi.Count()) 96 | 97 | require.Nil(t, txn3.Commit()) 98 | 99 | v7, err := db.Get(ro, givenTxnKey) 100 | require.Nil(t, err) 101 | require.True(t, v7.Data() == nil) 102 | v7.Free() 103 | 104 | // transaction 105 | txn4 := db.TransactionBegin(wo, to, nil) 106 | defer txn4.Destroy() 107 | 108 | // mark delete 109 | require.Nil(t, txn4.Delete(givenTxnKey)) 110 | 111 | // rebuild with put op 112 | wi = NewWriteBatchWI(123, true) 113 | wi.Put(givenTxnKey, givenTxnVal1) 114 | require.Nil(t, txn4.RebuildFromWriteBatchWI(wi)) 115 | require.Nil(t, txn4.Commit()) 116 | 117 | v8, err := db.Get(ro, givenTxnKey) 118 | require.Nil(t, err) 119 | require.True(t, v8.Data() != nil) // due to rebuild -> put -> key exists 120 | v8.Free() 121 | 122 | // transaction 123 | txn5 := db.TransactionBegin(wo, to, nil) 124 | defer txn5.Destroy() 125 | 126 | // mark delete 127 | require.Nil(t, txn5.Delete(givenTxnKey2)) 128 | 129 | // rebuild with put op 130 | wb := NewWriteBatch() 131 | wb.Put(givenTxnKey2, givenTxnVal1) 132 | require.Nil(t, txn5.RebuildFromWriteBatch(wb)) 133 | 134 | v, err := txn5.GetPinned(ro, givenTxnKey2) 135 | require.Nil(t, err) 136 | require.Equal(t, []byte(givenTxnVal1), v.Data()) 137 | v.Destroy() 138 | 139 | require.Nil(t, txn5.Commit()) 140 | 141 | v9, err := db.Get(ro, givenTxnKey2) 142 | require.Nil(t, err) 143 | require.True(t, v9.Data() != nil) // due to rebuild -> put -> key exists 144 | v9.Free() 145 | } 146 | 147 | func TestTransactionDBGetForUpdate(t *testing.T) { 148 | t.Parallel() 149 | 150 | lockTimeoutMilliSec := int64(50) 151 | applyOpts := func(_ *Options, transactionDBOpts *TransactionDBOptions) { 152 | transactionDBOpts.SetTransactionLockTimeout(lockTimeoutMilliSec) 153 | } 154 | db := newTestTransactionDB(t, applyOpts) 155 | defer db.Close() 156 | 157 | var ( 158 | givenKey = []byte("hello") 159 | givenVal = []byte("world") 160 | wo = NewDefaultWriteOptions() 161 | ro = NewDefaultReadOptions() 162 | to = NewDefaultTransactionOptions() 163 | ) 164 | 165 | txn := db.TransactionBegin(wo, to, nil) 166 | defer txn.Destroy() 167 | 168 | v, err := txn.GetForUpdate(ro, givenKey) 169 | require.Nil(t, err) 170 | v.Free() 171 | 172 | // expect lock timeout error to be thrown 173 | if err := db.Put(wo, givenKey, givenVal); err == nil { 174 | t.Error("expect locktime out error, got nil error") 175 | } 176 | 177 | base := db.GetBaseDB() 178 | defer CloseBaseDBOfTransactionDB(base) 179 | } 180 | 181 | func newTestTransactionDB(t *testing.T, applyOpts func(opts *Options, transactionDBOpts *TransactionDBOptions)) *TransactionDB { 182 | dir := t.TempDir() 183 | 184 | opts := NewDefaultOptions() 185 | opts.SetCreateIfMissing(true) 186 | transactionDBOpts := NewDefaultTransactionDBOptions() 187 | if applyOpts != nil { 188 | applyOpts(opts, transactionDBOpts) 189 | } 190 | db, err := OpenTransactionDb(opts, transactionDBOpts, dir) 191 | require.Nil(t, err) 192 | 193 | return db 194 | } 195 | 196 | func TestTransactionDBColumnFamilyBatchPutGet(t *testing.T) { 197 | t.Parallel() 198 | 199 | dir := t.TempDir() 200 | 201 | givenNames := []string{"default", "guide"} 202 | 203 | opts := NewDefaultOptions() 204 | opts.SetCreateIfMissingColumnFamilies(true) 205 | opts.SetCreateIfMissing(true) 206 | 207 | db, cfh, err := OpenTransactionDbColumnFamilies(opts, NewDefaultTransactionDBOptions(), dir, givenNames, []*Options{opts, opts}) 208 | require.Nil(t, err) 209 | defer db.Close() 210 | 211 | require.EqualValues(t, len(cfh), 2) 212 | defer cfh[0].Destroy() 213 | defer cfh[1].Destroy() 214 | 215 | wo := NewDefaultWriteOptions() 216 | defer wo.Destroy() 217 | ro := NewDefaultReadOptions() 218 | defer ro.Destroy() 219 | 220 | givenKey0 := []byte("hello0") 221 | givenVal0 := []byte("world0") 222 | givenKey1 := []byte("hello1") 223 | givenVal1 := []byte("world1") 224 | givenKey2 := []byte("hello2") 225 | givenVal2 := []byte("world2") 226 | 227 | writeReadBatch := func(cf *ColumnFamilyHandle, keys, values [][]byte) { 228 | b := NewWriteBatch() 229 | defer b.Destroy() 230 | for i := range keys { 231 | b.PutCF(cf, keys[i], values[i]) 232 | } 233 | require.Nil(t, db.Write(wo, b)) 234 | 235 | for i := range keys { 236 | actualVal, err := db.GetCF(ro, cf, keys[i]) 237 | require.Nil(t, err) 238 | require.EqualValues(t, actualVal.Data(), values[i]) 239 | actualVal.Free() 240 | } 241 | } 242 | 243 | writeReadBatch(cfh[0], [][]byte{givenKey0}, [][]byte{givenVal0}) 244 | 245 | writeReadBatch(cfh[1], [][]byte{givenKey1, givenKey2}, [][]byte{givenVal1, givenVal2}) 246 | 247 | // check read from wrong CF returns nil 248 | actualVal, err := db.GetCF(ro, cfh[0], givenKey1) 249 | require.Nil(t, err) 250 | require.EqualValues(t, actualVal.Size(), 0) 251 | actualVal.Free() 252 | 253 | actualVal, err = db.GetCF(ro, cfh[1], givenKey0) 254 | require.Nil(t, err) 255 | require.EqualValues(t, actualVal.Size(), 0) 256 | actualVal.Free() 257 | 258 | // check batch read is correct 259 | actualVals, err := db.MultiGetWithCF(ro, cfh[1], givenKey1, givenKey2) 260 | require.Nil(t, err) 261 | require.EqualValues(t, len(actualVals), 2) 262 | require.EqualValues(t, actualVals[0].Data(), givenVal1) 263 | require.EqualValues(t, actualVals[1].Data(), givenVal2) 264 | actualVals.Destroy() 265 | 266 | // trigger flush 267 | require.Nil(t, db.FlushCF(cfh[0], NewDefaultFlushOptions())) 268 | require.Nil(t, db.FlushCFs(cfh, NewDefaultFlushOptions())) 269 | } 270 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | 7 | import ( 8 | "errors" 9 | "unsafe" 10 | ) 11 | 12 | // boolToChar converts a bool value to C.uchar. 13 | func boolToChar(b bool) C.uchar { 14 | if b { 15 | return 1 16 | } 17 | return 0 18 | } 19 | 20 | // charToBool converts C.uchar to bool value 21 | func charToBool(c C.uchar) bool { 22 | return c != 0 23 | } 24 | 25 | // refCBytes referencing *C.char. 26 | func refCBytes(data *C.char, len C.size_t) []byte { 27 | return unsafe.Slice((*byte)(unsafe.Pointer(data)), int(len)) 28 | } 29 | 30 | // refGoBytes returns *C.char from byte slice. 31 | func refGoBytes(b []byte) *C.char { 32 | var c *C.char 33 | if len(b) > 0 { 34 | c = (*C.char)(unsafe.Pointer(&b[0])) 35 | } 36 | return c 37 | } 38 | 39 | // Go []byte to C string 40 | // The C string is allocated in the C heap using malloc. 41 | func cByteSlice(b []byte) *C.char { 42 | var c *C.char 43 | if len(b) > 0 { 44 | c = (*C.char)(C.CBytes(b)) 45 | } 46 | return c 47 | } 48 | 49 | // charSlice converts a C array of *char to a []*C.char. 50 | func charSlice(data **C.char, len C.int) []*C.char { 51 | return unsafe.Slice(data, int(len)) 52 | } 53 | 54 | // charSliceIntoStringSlice converts a C array of *char to a []string. 55 | func charSliceIntoStringSlice(data **C.char, len C.int) []string { 56 | s := charSlice(data, len) 57 | 58 | result := make([]string, int(len)) 59 | for i := range s { 60 | result[i] = C.GoString(s[i]) 61 | } 62 | 63 | return result 64 | } 65 | 66 | // sizeSlice converts a C array of size_t to a []C.size_t. 67 | func sizeSlice(data *C.size_t, len C.int) []C.size_t { 68 | return unsafe.Slice(data, int(len)) 69 | } 70 | 71 | // fromCError returns go error and free c_err if need. 72 | func fromCError(cErr *C.char) (err error) { 73 | if cErr != nil { 74 | err = errors.New(C.GoString(cErr)) 75 | C.rocksdb_free(unsafe.Pointer(cErr)) 76 | } 77 | return 78 | } 79 | 80 | func toString(cVal *C.char, len C.int) string { 81 | s := C.GoStringN(cVal, C.int(len)) 82 | C.rocksdb_free(unsafe.Pointer(cVal)) 83 | return s 84 | } 85 | -------------------------------------------------------------------------------- /wal_iterator.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include 4 | // #include "rocksdb/c.h" 5 | import "C" 6 | import "unsafe" 7 | 8 | // WalIterator is iterator for WAL Files. 9 | type WalIterator struct { 10 | c *C.rocksdb_wal_iterator_t 11 | } 12 | 13 | // NewNativeWalIterator returns new WalIterator. 14 | func newNativeWalIterator(c unsafe.Pointer) *WalIterator { 15 | return &WalIterator{c: (*C.rocksdb_wal_iterator_t)(c)} 16 | } 17 | 18 | // Valid check if current WAL is valid. 19 | func (iter *WalIterator) Valid() bool { 20 | return C.rocksdb_wal_iter_valid(iter.c) != 0 21 | } 22 | 23 | // Next moves next. 24 | func (iter *WalIterator) Next() { 25 | C.rocksdb_wal_iter_next(iter.c) 26 | } 27 | 28 | // Err returns error happened during iteration. 29 | func (iter *WalIterator) Err() (err error) { 30 | var cErr *C.char 31 | C.rocksdb_wal_iter_status(iter.c, &cErr) 32 | err = fromCError(cErr) 33 | return 34 | } 35 | 36 | // Destroy free iterator. 37 | func (iter *WalIterator) Destroy() { 38 | C.rocksdb_wal_iter_destroy(iter.c) 39 | iter.c = nil 40 | } 41 | 42 | // GetBatch returns the current write_batch and the sequence number of the 43 | // earliest transaction contained in the batch. 44 | func (iter *WalIterator) GetBatch() (*WriteBatch, uint64) { 45 | var cSeq C.uint64_t 46 | cB := C.rocksdb_wal_iter_get_batch(iter.c, &cSeq) 47 | return newNativeWriteBatch(cB), uint64(cSeq) 48 | } 49 | -------------------------------------------------------------------------------- /write_batch_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestWriteBatch(t *testing.T) { 11 | t.Parallel() 12 | 13 | db := newTestDB(t, nil) 14 | defer db.Close() 15 | 16 | var ( 17 | givenKey1 = []byte("key1") 18 | givenVal1 = []byte("val1") 19 | givenKey2 = []byte("key2") 20 | ) 21 | wo := NewDefaultWriteOptions() 22 | require.Nil(t, db.Put(wo, givenKey2, []byte("foo"))) 23 | 24 | // create and fill the write batch 25 | wb := NewWriteBatch() 26 | defer wb.Destroy() 27 | wb.Put(givenKey1, givenVal1) 28 | wb.Delete(givenKey2) 29 | require.EqualValues(t, wb.Count(), 2) 30 | 31 | // perform the batch 32 | require.Nil(t, db.Write(wo, wb)) 33 | 34 | // check changes 35 | ro := NewDefaultReadOptions() 36 | v1, err := db.Get(ro, givenKey1) 37 | require.Nil(t, err) 38 | require.EqualValues(t, v1.Data(), givenVal1) 39 | v1.Free() 40 | 41 | v2, err := db.Get(ro, givenKey2) 42 | require.Nil(t, err) 43 | require.True(t, v2.Data() == nil) 44 | v2.Free() 45 | 46 | // DeleteRange test 47 | wb.Clear() 48 | wb.DeleteRange(givenKey1, givenKey2) 49 | 50 | // perform the batch 51 | require.Nil(t, db.Write(wo, wb)) 52 | 53 | v1, err = db.Get(ro, givenKey1) 54 | require.Nil(t, err) 55 | require.True(t, v1.Data() == nil) 56 | v1.Free() 57 | } 58 | 59 | func TestWriteBatchWithParams(t *testing.T) { 60 | t.Parallel() 61 | 62 | db := newTestDB(t, nil) 63 | defer db.Close() 64 | 65 | var ( 66 | givenKey1 = []byte("key1") 67 | givenVal1 = []byte("val1") 68 | givenKey2 = []byte("key2") 69 | ) 70 | wo := NewDefaultWriteOptions() 71 | require.Nil(t, db.Put(wo, givenKey2, []byte("foo"))) 72 | 73 | // create and fill the write batch 74 | wb := NewWriteBatchWithParams(10000, 200000, 10, 0) 75 | defer wb.Destroy() 76 | wb.Put(givenKey1, givenVal1) 77 | wb.Delete(givenKey2) 78 | require.EqualValues(t, wb.Count(), 2) 79 | 80 | // perform the batch 81 | require.Nil(t, db.Write(wo, wb)) 82 | 83 | // check changes 84 | ro := NewDefaultReadOptions() 85 | v1, err := db.Get(ro, givenKey1) 86 | require.Nil(t, err) 87 | require.EqualValues(t, v1.Data(), givenVal1) 88 | v1.Free() 89 | 90 | v2, err := db.Get(ro, givenKey2) 91 | require.Nil(t, err) 92 | require.True(t, v2.Data() == nil) 93 | v2.Free() 94 | 95 | // DeleteRange test 96 | wb.Clear() 97 | wb.DeleteRange(givenKey1, givenKey2) 98 | 99 | // perform the batch 100 | require.Nil(t, db.Write(wo, wb)) 101 | 102 | v1, err = db.Get(ro, givenKey1) 103 | require.Nil(t, err) 104 | require.True(t, v1.Data() == nil) 105 | v1.Free() 106 | } 107 | 108 | func TestWriteBatchIterator(t *testing.T) { 109 | t.Parallel() 110 | 111 | db := newTestDB(t, nil) 112 | defer db.Close() 113 | 114 | var ( 115 | givenKey1 = []byte("key1") 116 | givenVal1 = []byte("val1") 117 | givenKey2 = []byte("key2") 118 | ) 119 | // create and fill the write batch 120 | wb := NewWriteBatch() 121 | defer wb.Destroy() 122 | wb.Put(givenKey1, givenVal1) 123 | wb.Delete(givenKey2) 124 | require.EqualValues(t, wb.Count(), 2) 125 | 126 | // iterate over the batch 127 | iter := wb.NewIterator() 128 | require.True(t, iter.Next()) 129 | record := iter.Record() 130 | require.EqualValues(t, record.Type, WriteBatchValueRecord) 131 | require.EqualValues(t, record.Key, givenKey1) 132 | require.EqualValues(t, record.Value, givenVal1) 133 | 134 | require.True(t, iter.Next()) 135 | record = iter.Record() 136 | require.EqualValues(t, record.Type, WriteBatchDeletionRecord) 137 | require.EqualValues(t, record.Key, givenKey2) 138 | 139 | // there shouldn't be any left 140 | require.False(t, iter.Next()) 141 | } 142 | 143 | func TestDecodeVarint_ISSUE131(t *testing.T) { 144 | t.Parallel() 145 | 146 | tests := []struct { 147 | name string 148 | in []byte 149 | wantValue uint64 150 | expectErr bool 151 | }{ 152 | { 153 | name: "invalid: 10th byte", 154 | in: []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, 155 | wantValue: 0, 156 | expectErr: true, 157 | }, 158 | { 159 | name: "valid: math.MaxUint64-40", 160 | in: []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}, 161 | wantValue: math.MaxUint64 - 40, 162 | expectErr: false, 163 | }, 164 | { 165 | name: "invalid: with more than MaxVarintLen64 bytes", 166 | in: []byte{0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}, 167 | wantValue: 0, 168 | expectErr: true, 169 | }, 170 | { 171 | name: "invalid: 1000 bytes", 172 | in: func() []byte { 173 | b := make([]byte, 1000) 174 | for i := range b { 175 | b[i] = 0xff 176 | } 177 | b[999] = 0 178 | return b 179 | }(), 180 | wantValue: 0, 181 | expectErr: true, 182 | }, 183 | } 184 | 185 | for _, test := range tests { 186 | wbi := &WriteBatchIterator{data: test.in} 187 | require.EqualValues(t, test.wantValue, wbi.decodeVarint(), test.name) 188 | if test.expectErr { 189 | require.Error(t, wbi.err, test.name) 190 | } else { 191 | require.NoError(t, wbi.err, test.name) 192 | } 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /write_batch_ts_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestWriteBatchWithTS(t *testing.T) { 10 | t.Parallel() 11 | 12 | db, cfs, cleanup := newTestDBMultiCF(t, []string{"default"}, func(opts *Options) { 13 | opts.SetComparator(newDefaultComparatorWithTS()) 14 | }) 15 | defer cleanup() 16 | 17 | defaultCF := cfs[0] 18 | 19 | var ( 20 | givenKey1 = []byte("key1") 21 | givenVal1 = []byte("val1") 22 | givenKey2 = []byte("key2") 23 | 24 | givenTs1 = marshalTimestamp(1) 25 | givenTs2 = marshalTimestamp(2) 26 | ) 27 | wo := NewDefaultWriteOptions() 28 | require.Nil(t, db.PutWithTS(wo, givenKey2, givenTs1, []byte("foo"))) 29 | 30 | // create and fill the write batch 31 | wb := NewWriteBatch() 32 | defer wb.Destroy() 33 | wb.PutCFWithTS(defaultCF, givenKey1, givenTs2, givenVal1) 34 | wb.DeleteCFWithTS(defaultCF, givenKey2, givenTs2) 35 | require.EqualValues(t, wb.Count(), 2) 36 | 37 | // perform the batch 38 | require.Nil(t, db.Write(wo, wb)) 39 | 40 | // check changes 41 | ro := NewDefaultReadOptions() 42 | ro.SetTimestamp(givenTs2) 43 | v1, t1, err := db.GetWithTS(ro, givenKey1) 44 | require.Nil(t, err) 45 | require.EqualValues(t, v1.Data(), givenVal1) 46 | require.EqualValues(t, t1.Data(), givenTs2) 47 | v1.Free() 48 | 49 | v2, t2, err := db.GetWithTS(ro, givenKey2) 50 | require.Nil(t, err) 51 | require.True(t, v2.Data() == nil) 52 | require.True(t, t2.Data() == nil) 53 | v2.Free() 54 | 55 | wb.Clear() 56 | // DeleteRange not supported for timestamp 57 | } 58 | 59 | func TestWriteBatchIteratorWithTS(t *testing.T) { 60 | t.Parallel() 61 | 62 | _, cfs, cleanup := newTestDBMultiCF(t, []string{"default"}, func(opts *Options) { 63 | opts.SetComparator(newDefaultComparatorWithTS()) 64 | }) 65 | defer cleanup() 66 | 67 | defaultCF := cfs[0] 68 | 69 | var ( 70 | givenKey1 = []byte("key1") 71 | givenVal1 = []byte("val1") 72 | givenKey2 = []byte("key2") 73 | 74 | givenTs1 = marshalTimestamp(1) 75 | givenTs2 = marshalTimestamp(2) 76 | 77 | expectedKeyWithTS1 = append(givenKey1, givenTs1...) 78 | expectedKeyWithTS2 = append(givenKey2, givenTs2...) 79 | ) 80 | 81 | // create and fill the write batch 82 | wb := NewWriteBatch() 83 | defer wb.Destroy() 84 | wb.PutCFWithTS(defaultCF, givenKey1, givenTs1, givenVal1) 85 | wb.DeleteCFWithTS(defaultCF, givenKey2, givenTs2) 86 | require.EqualValues(t, wb.Count(), 2) 87 | 88 | // iterate over the batch 89 | iter := wb.NewIterator() 90 | require.True(t, iter.Next()) 91 | record := iter.Record() 92 | require.EqualValues(t, record.Type, WriteBatchValueRecord) 93 | require.EqualValues(t, record.Key, expectedKeyWithTS1) 94 | require.EqualValues(t, record.Value, givenVal1) 95 | 96 | require.True(t, iter.Next()) 97 | record = iter.Record() 98 | require.EqualValues(t, record.Type, WriteBatchDeletionRecord) 99 | require.EqualValues(t, record.Key, expectedKeyWithTS2) 100 | 101 | // there shouldn't be any left 102 | require.False(t, iter.Next()) 103 | } 104 | -------------------------------------------------------------------------------- /write_batch_wi_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestWriteBatchWI(t *testing.T) { 10 | t.Parallel() 11 | 12 | db := newTestDB(t, nil) 13 | defer db.Close() 14 | 15 | var ( 16 | givenKey1 = []byte("key1") 17 | givenVal1 = []byte("val1") 18 | givenKey2 = []byte("key2") 19 | givenVal2 = []byte("val2") 20 | givenKey3 = []byte("key3") 21 | givenVal3 = []byte("val3") 22 | givenKey4 = []byte("key4") 23 | givenVal4 = []byte("val4") 24 | givenVal2Updated = []byte("foo") 25 | ) 26 | 27 | wo := NewDefaultWriteOptions() 28 | require.Nil(t, db.Put(wo, givenKey1, givenVal1)) 29 | require.Nil(t, db.Put(wo, givenKey2, givenVal2)) 30 | require.Nil(t, db.Put(wo, givenKey3, givenVal3)) 31 | 32 | // create and fill the write batch 33 | wb := NewWriteBatchWI(0, true) 34 | defer wb.Destroy() 35 | wb.Put(givenKey2, givenVal2Updated) 36 | wb.Put(givenKey4, givenVal4) 37 | wb.Delete(givenKey3) 38 | require.EqualValues(t, wb.Count(), 3) 39 | 40 | // check before writing to db 41 | ro := NewDefaultReadOptions() 42 | v1, err := wb.GetFromDB(db, ro, givenKey1) 43 | require.Nil(t, err) 44 | require.EqualValues(t, v1.Data(), givenVal1) 45 | v1.Free() 46 | 47 | v2, err := wb.GetFromDB(db, ro, givenKey2) 48 | require.Nil(t, err) 49 | require.EqualValues(t, v2.Data(), givenVal2Updated) 50 | v2.Free() 51 | 52 | v3, err := wb.GetFromDB(db, ro, givenKey3) 53 | require.Nil(t, err) 54 | require.True(t, v3.Data() == nil) 55 | v3.Free() 56 | 57 | v4, err := wb.GetFromDB(db, ro, givenKey4) 58 | require.Nil(t, err) 59 | require.EqualValues(t, v4.Data(), givenVal4) 60 | v4.Free() 61 | 62 | // perform the batch 63 | require.Nil(t, db.WriteWI(wo, wb)) 64 | 65 | // check changes 66 | v1, err = db.Get(ro, givenKey1) 67 | require.Nil(t, err) 68 | require.EqualValues(t, v1.Data(), givenVal1) 69 | v1.Free() 70 | 71 | v2, err = db.Get(ro, givenKey2) 72 | require.Nil(t, err) 73 | require.EqualValues(t, v2.Data(), givenVal2Updated) 74 | v2.Free() 75 | 76 | v3, err = db.Get(ro, givenKey3) 77 | require.Nil(t, err) 78 | require.True(t, v3.Data() == nil) 79 | v3.Free() 80 | 81 | v4, err = db.Get(ro, givenKey4) 82 | require.Nil(t, err) 83 | require.EqualValues(t, v4.Data(), givenVal4) 84 | v4.Free() 85 | 86 | wb.Clear() 87 | // DeleteRange not supported 88 | } 89 | 90 | func TestWriteBatchWIIterator(t *testing.T) { 91 | t.Parallel() 92 | 93 | db := newTestDB(t, nil) 94 | defer db.Close() 95 | 96 | var ( 97 | givenKey1 = []byte("key1") 98 | givenVal1 = []byte("val1") 99 | givenKey2 = []byte("key2") 100 | ) 101 | // create and fill the write batch 102 | wb := NewWriteBatchWI(0, true) 103 | defer wb.Destroy() 104 | wb.Put(givenKey1, givenVal1) 105 | wb.Delete(givenKey2) 106 | require.EqualValues(t, wb.Count(), 2) 107 | 108 | // iterate over the batch 109 | iter := wb.NewIterator() 110 | require.True(t, iter.Next()) 111 | record := iter.Record() 112 | require.EqualValues(t, record.Type, WriteBatchValueRecord) 113 | require.EqualValues(t, record.Key, givenKey1) 114 | require.EqualValues(t, record.Value, givenVal1) 115 | 116 | require.True(t, iter.Next()) 117 | record = iter.Record() 118 | require.EqualValues(t, record.Type, WriteBatchDeletionRecord) 119 | require.EqualValues(t, record.Key, givenKey2) 120 | 121 | // there shouldn't be any left 122 | require.False(t, iter.Next()) 123 | } 124 | 125 | func TestWriteBatchWIIteratorWithBase(t *testing.T) { 126 | t.Parallel() 127 | 128 | db, cfs, closeup := newTestDBMultiCF(t, []string{"default", "custom"}, nil) 129 | defer closeup() 130 | 131 | defaultCF := cfs[0] 132 | customCF := cfs[1] 133 | 134 | var ( 135 | givenKey1 = []byte("key1") 136 | givenVal1 = []byte("val1") 137 | givenKey2 = []byte("key2") 138 | givenVal2 = []byte("val2") 139 | givenKey3 = []byte("key3") 140 | givenVal3 = []byte("val3") 141 | givenKey4 = []byte("key4") 142 | givenVal4 = []byte("val4") 143 | givenKey5 = []byte("key5") 144 | givenVal5 = []byte("val5") 145 | givenVal2Updated = []byte("foo") 146 | ) 147 | 148 | wo := NewDefaultWriteOptions() 149 | require.Nil(t, db.PutCF(wo, defaultCF, givenKey1, givenVal1)) 150 | require.Nil(t, db.PutCF(wo, defaultCF, givenKey2, givenVal2)) 151 | require.Nil(t, db.PutCF(wo, defaultCF, givenKey3, givenVal3)) 152 | require.Nil(t, db.PutCF(wo, customCF, givenKey2, givenVal2)) 153 | require.Nil(t, db.PutCF(wo, customCF, givenKey4, givenVal4)) 154 | 155 | // create and fill the write batch 156 | wb := NewWriteBatchWI(0, true) 157 | defer wb.Destroy() 158 | wb.PutCF(defaultCF, givenKey2, givenVal2Updated) 159 | wb.DeleteCF(defaultCF, givenKey3) 160 | wb.PutCF(defaultCF, givenKey4, givenVal4) 161 | wb.PutCF(customCF, givenKey5, givenVal5) 162 | wb.DeleteCF(customCF, givenKey2) 163 | require.EqualValues(t, wb.Count(), 5) 164 | 165 | // create base iterator for default 166 | ro := NewDefaultReadOptions() 167 | defBaseIter := db.NewIteratorCF(ro, defaultCF) 168 | 169 | iter1 := wb.NewIteratorWithBaseCF(db, defBaseIter, defaultCF) 170 | defer iter1.Close() 171 | 172 | givenKeys1 := [][]byte{givenKey1, givenKey2, givenKey4} 173 | givenValues1 := [][]byte{givenVal1, givenVal2Updated, givenVal4} 174 | 175 | var actualKeys, actualValues [][]byte 176 | 177 | for iter1.SeekToFirst(); iter1.Valid(); iter1.Next() { 178 | k := iter1.Key().Data() 179 | v := iter1.Value().Data() 180 | key := make([]byte, len(k)) 181 | value := make([]byte, len(v)) 182 | copy(key, k) 183 | copy(value, v) 184 | actualKeys = append(actualKeys, key) 185 | actualValues = append(actualValues, value) 186 | } 187 | 188 | require.Nil(t, iter1.Err()) 189 | require.EqualValues(t, actualKeys, givenKeys1) 190 | require.EqualValues(t, actualValues, givenValues1) 191 | 192 | // create base iterator for custom 193 | customBaseIter := db.NewIteratorCF(ro, customCF) 194 | 195 | iter2 := wb.NewIteratorWithBaseCF(db, customBaseIter, customCF) 196 | defer iter2.Close() 197 | 198 | givenKeys2 := [][]byte{givenKey4, givenKey5} 199 | givenValues2 := [][]byte{givenVal4, givenVal5} 200 | 201 | actualKeys = actualKeys[:0] 202 | actualValues = actualValues[:0] 203 | 204 | for iter2.SeekToFirst(); iter2.Valid(); iter2.Next() { 205 | k := iter2.Key().Data() 206 | v := iter2.Value().Data() 207 | key := make([]byte, len(k)) 208 | value := make([]byte, len(v)) 209 | copy(key, k) 210 | copy(value, v) 211 | actualKeys = append(actualKeys, key) 212 | actualValues = append(actualValues, value) 213 | } 214 | 215 | require.Nil(t, iter2.Err()) 216 | require.EqualValues(t, actualKeys, givenKeys2) 217 | require.EqualValues(t, actualValues, givenValues2) 218 | } 219 | -------------------------------------------------------------------------------- /write_buffer_manager.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | // #include "rocksdb/c.h" 4 | import "C" 5 | 6 | type WriteBufferManager struct { 7 | p *C.rocksdb_write_buffer_manager_t 8 | } 9 | 10 | // NewWriteBufferManager creates new WriteBufferManager object. 11 | // 12 | // @bufferSize: bufferSize = 0 indicates no limit. Memory won't be capped. 13 | // memory_usage() won't be valid and ShouldFlush() will always return true. 14 | // 15 | // @allowStall: if set true, it will enable stalling of writes when 16 | // memory_usage() exceeds buffer_size. It will wait for flush to complete and 17 | // memory usage to drop down. 18 | func NewWriteBufferManager(bufferSize int, allowStall bool) *WriteBufferManager { 19 | return &WriteBufferManager{ 20 | p: C.rocksdb_write_buffer_manager_create(C.size_t(bufferSize), C.bool(allowStall)), 21 | } 22 | } 23 | 24 | // NewWriteBufferManagerWithCache similars to NewWriteBufferManager, we'll put dummy entries in the cache and 25 | // cost the memory allocated to the cache. It can be used even if bufferSize = 0. 26 | func NewWriteBufferManagerWithCache(bufferSize int, cache *Cache, allowStall bool) *WriteBufferManager { 27 | return &WriteBufferManager{ 28 | p: C.rocksdb_write_buffer_manager_create_with_cache(C.size_t(bufferSize), cache.c, C.bool(allowStall)), 29 | } 30 | } 31 | 32 | // Enabled returns true if buffer_limit is passed to limit the total memory usage and 33 | // is greater than 0. 34 | func (w *WriteBufferManager) Enabled() bool { 35 | return bool(C.rocksdb_write_buffer_manager_enabled(w.p)) 36 | } 37 | 38 | // CostToCache returns true if pointer to cache is passed. 39 | func (w *WriteBufferManager) CostToCache() bool { 40 | return bool(C.rocksdb_write_buffer_manager_cost_to_cache(w.p)) 41 | } 42 | 43 | // MemoryUsage returns the total memory used by memtables. 44 | // Only valid if enabled(). 45 | func (w *WriteBufferManager) MemoryUsage() int { 46 | return int(C.rocksdb_write_buffer_manager_memory_usage(w.p)) 47 | } 48 | 49 | // MemtableMemoryUsage returns the total memory used by active memtables. 50 | func (w *WriteBufferManager) MemtableMemoryUsage() int { 51 | return int(C.rocksdb_write_buffer_manager_mutable_memtable_memory_usage(w.p)) 52 | } 53 | 54 | // DummyEntriesInCacheUsage returns number of dummy entries in cache. 55 | func (w *WriteBufferManager) DummyEntriesInCacheUsage() int { 56 | return int(C.rocksdb_write_buffer_manager_dummy_entries_in_cache_usage(w.p)) 57 | } 58 | 59 | // BufferSize returns the buffer size. 60 | func (w *WriteBufferManager) BufferSize() int { 61 | return int(C.rocksdb_write_buffer_manager_buffer_size(w.p)) 62 | } 63 | 64 | // SetBufferSize sets buffer size. 65 | func (w *WriteBufferManager) SetBufferSize(size int) { 66 | C.rocksdb_write_buffer_manager_set_buffer_size(w.p, C.size_t(size)) 67 | } 68 | 69 | // ToggleAllowStall sets allow stall option. 70 | func (w *WriteBufferManager) ToggleAllowStall(v bool) { 71 | C.rocksdb_write_buffer_manager_set_allow_stall(w.p, C.bool(v)) 72 | } 73 | 74 | // Destroy the object. 75 | func (w *WriteBufferManager) Destroy() { 76 | C.rocksdb_write_buffer_manager_destroy(w.p) 77 | w.p = nil 78 | } 79 | -------------------------------------------------------------------------------- /write_buffer_manager_test.go: -------------------------------------------------------------------------------- 1 | package grocksdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestWriteBufferManager(t *testing.T) { 10 | t.Parallel() 11 | 12 | wbm := NewWriteBufferManager(12345, false) 13 | defer wbm.Destroy() 14 | 15 | require.True(t, wbm.Enabled()) 16 | require.False(t, wbm.CostToCache()) 17 | require.EqualValues(t, 0, wbm.MemoryUsage()) 18 | require.EqualValues(t, 0, wbm.MemtableMemoryUsage()) 19 | require.EqualValues(t, 0, wbm.DummyEntriesInCacheUsage()) 20 | 21 | wbm.ToggleAllowStall(true) 22 | 23 | require.Equal(t, 12345, wbm.BufferSize()) 24 | wbm.SetBufferSize(123456) 25 | require.Equal(t, 123456, wbm.BufferSize()) 26 | } 27 | 28 | func TestWriteBufferManagerWithCache(t *testing.T) { 29 | t.Parallel() 30 | 31 | cache := NewLRUCache(123) 32 | defer cache.Destroy() 33 | 34 | wbm := NewWriteBufferManagerWithCache(12345, cache, false) 35 | defer wbm.Destroy() 36 | 37 | require.True(t, wbm.Enabled()) 38 | require.True(t, wbm.CostToCache()) 39 | require.EqualValues(t, 0, wbm.MemoryUsage()) 40 | require.EqualValues(t, 0, wbm.MemtableMemoryUsage()) 41 | require.EqualValues(t, 0, wbm.DummyEntriesInCacheUsage()) 42 | 43 | wbm.ToggleAllowStall(true) 44 | 45 | require.Equal(t, 12345, wbm.BufferSize()) 46 | wbm.SetBufferSize(123456) 47 | require.Equal(t, 123456, wbm.BufferSize()) 48 | } 49 | --------------------------------------------------------------------------------