├── docs ├── _static │ └── .empty ├── api │ ├── index.rst │ ├── backup.rst │ ├── interfaces.rst │ └── database.rst ├── index.rst ├── installation.rst ├── Makefile ├── changelog.rst ├── conf.py └── tutorial │ └── index.rst ├── rocksdb ├── tests │ ├── __init__.py │ ├── test_memtable.py │ ├── test_transaction_db.py │ ├── test_transactiondb_options.py │ ├── test_options.py │ └── test_db.py ├── __init__.py ├── snapshot.pxd ├── env.pxd ├── logger.pxd ├── cpp │ ├── utils.hpp │ ├── memtable_factories.hpp │ ├── comparator_wrapper.hpp │ ├── write_batch_iter_helper.hpp │ ├── filter_policy_wrapper.hpp │ ├── slice_transform_wrapper.hpp │ └── merge_operator_wrapper.hpp ├── cache.pxd ├── std_memory.pxd ├── stackable_db.pxd ├── errors.py ├── concurrent_task_limiter.pxd ├── memtablerep.pxd ├── universal_compaction.pxd ├── status.pxd ├── iterator.pxd ├── slice_.pxd ├── comparator.pxd ├── merge_operators.py ├── types.pxd ├── filter_policy.pxd ├── slice_transform.pxd ├── merge_operator.pxd ├── backup.pxd ├── interfaces.py ├── statistics.pxd ├── table_factory.pxd ├── transaction_db.pxd ├── compaction_filter.pxd ├── table_properties.pxd ├── advanced_options.pxd ├── db.pxd └── options.pxd ├── MANIFEST.in ├── pyproject.toml ├── .gitignore ├── .travis.yml ├── tox.ini ├── Dockerfile ├── README.rst ├── setup.py ├── setup.cfg ├── .github └── workflows │ ├── debian.yml │ ├── dist.yml │ └── build.yml └── LICENSE /docs/_static/.empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rocksdb/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rocksdb/__init__.py: -------------------------------------------------------------------------------- 1 | from ._rocksdb import * 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include rocksdb/cpp/*.hpp 2 | recursive-include rocksdb *.pxd 3 | recursive-include rocksdb *.pyx 4 | -------------------------------------------------------------------------------- /rocksdb/snapshot.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/db.h" namespace "rocksdb": 2 | cdef cppclass Snapshot: 3 | pass 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "cython", "pkgconfig"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | docs/_build 3 | .pytest_cache 4 | .eggs/ 5 | .tox/ 6 | *.egg-info/ 7 | *.pyc 8 | *.so 9 | __pycache__ 10 | rocksdb/_rocksdb.cpp 11 | 12 | -------------------------------------------------------------------------------- /rocksdb/env.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/env.h" namespace "rocksdb": 2 | cdef cppclass Env: 3 | Env() 4 | 5 | cdef Env* Env_Default "rocksdb::Env::Default"() 6 | -------------------------------------------------------------------------------- /rocksdb/logger.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/env.h" namespace "rocksdb": 2 | cdef cppclass Logger: 3 | pass 4 | 5 | void Log(Logger*, const char*, ...) nogil except+ 6 | -------------------------------------------------------------------------------- /rocksdb/cpp/utils.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace py_rocks { 4 | template 5 | const T* vector_data(std::vector& v) { 6 | return v.data(); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | Python driver for RocksDB 2 | ========================= 3 | 4 | .. py:module:: rocksdb 5 | 6 | .. toctree:: 7 | 8 | Options 9 | Database 10 | Interfaces 11 | Backup 12 | -------------------------------------------------------------------------------- /rocksdb/cache.pxd: -------------------------------------------------------------------------------- 1 | from .std_memory cimport shared_ptr 2 | 3 | cdef extern from "rocksdb/cache.h" namespace "rocksdb": 4 | cdef cppclass Cache: 5 | pass 6 | 7 | cdef extern shared_ptr[Cache] NewLRUCache(size_t) 8 | cdef extern shared_ptr[Cache] NewLRUCache(size_t, int) 9 | -------------------------------------------------------------------------------- /rocksdb/std_memory.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "" namespace "std": 2 | cdef cppclass shared_ptr[T]: 3 | shared_ptr() nogil except+ 4 | shared_ptr(T*) nogil except+ 5 | void reset() nogil except+ 6 | void reset(T*) nogil except+ 7 | T* get() nogil except+ 8 | -------------------------------------------------------------------------------- /rocksdb/stackable_db.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.memory cimport shared_ptr 2 | from .db cimport DB 3 | 4 | cdef extern from "rocksdb/utilities/stackable_db.h" namespace "rocksdb": 5 | cdef cppclass StackableDB(DB): 6 | StackableDB(DB*) nogil except+ 7 | StackableDB(shared_ptr[DB] db) nogil except+ 8 | DB* GetBaseDB() nogil except+ 9 | -------------------------------------------------------------------------------- /rocksdb/errors.py: -------------------------------------------------------------------------------- 1 | class Error(Exception): 2 | pass 3 | 4 | class NotFound(Error): 5 | pass 6 | 7 | class Corruption(Error): 8 | pass 9 | 10 | class NotSupported(Error): 11 | pass 12 | 13 | class InvalidArgument(Error): 14 | pass 15 | 16 | class RocksIOError(Error): 17 | pass 18 | 19 | class MergeInProgress(Error): 20 | pass 21 | 22 | class Incomplete(Error): 23 | pass 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: generic 4 | services: 5 | - docker 6 | 7 | cache: 8 | directories: 9 | - ~/.cache/pip 10 | 11 | install: 12 | docker build . -t ci-image; 13 | script: 14 | docker run -v ~/.cache/pip:/home/tester/.cache/pip -v $(pwd):/home/tester/src ci-image:latest tox -e ${TOXENV} ; 15 | env: 16 | - TOXENV=py27 17 | - TOXENV=py36 18 | - TOXENV=docs 19 | -------------------------------------------------------------------------------- /rocksdb/cpp/memtable_factories.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/memtablerep.h" 2 | 3 | using rocksdb::MemTableRepFactory; 4 | using rocksdb::VectorRepFactory; 5 | using rocksdb::SkipListFactory; 6 | 7 | namespace py_rocks { 8 | MemTableRepFactory* NewVectorRepFactory(size_t count = 0) { 9 | return new VectorRepFactory(count); 10 | } 11 | 12 | MemTableRepFactory* NewSkipListFactory() { 13 | return new SkipListFactory(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /rocksdb/concurrent_task_limiter.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from libc.stdint cimport int32_t 3 | 4 | cdef extern from "rocksdb/concurrent_task_limiter.h" namespace "rocksdb": 5 | cdef cppclass ConcurrentTaskLimiter: 6 | const string& GetName() 7 | void SetMaxOutstandingTask(int32_t) 8 | void ResetMaxOutstandingTask() 9 | int32_t GetOutstandingTask() 10 | 11 | ConcurrentTaskLimiter* NewConcurrentTaskLimiter(const string&, int32_t) 12 | -------------------------------------------------------------------------------- /rocksdb/memtablerep.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport int32_t 2 | 3 | cdef extern from "rocksdb/memtablerep.h" namespace "rocksdb": 4 | cdef cppclass MemTableRepFactory: 5 | MemTableRepFactory() 6 | 7 | cdef MemTableRepFactory* NewHashSkipListRepFactory(size_t, int32_t, int32_t) 8 | cdef MemTableRepFactory* NewHashLinkListRepFactory(size_t) 9 | 10 | cdef extern from "cpp/memtable_factories.hpp" namespace "py_rocks": 11 | cdef MemTableRepFactory* NewVectorRepFactory(size_t) 12 | cdef MemTableRepFactory* NewSkipListFactory() 13 | -------------------------------------------------------------------------------- /rocksdb/universal_compaction.pxd: -------------------------------------------------------------------------------- 1 | cdef extern from "rocksdb/universal_compaction.h" namespace "rocksdb": 2 | 3 | ctypedef enum CompactionStopStyle: 4 | kCompactionStopStyleSimilarSize 5 | kCompactionStopStyleTotalSize 6 | 7 | cdef cppclass CompactionOptionsUniversal: 8 | CompactionOptionsUniversal() 9 | 10 | unsigned int size_ratio 11 | unsigned int min_merge_width 12 | unsigned int max_merge_width 13 | unsigned int max_size_amplification_percent 14 | int compression_size_percent 15 | CompactionStopStyle stop_style 16 | -------------------------------------------------------------------------------- /rocksdb/status.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | 4 | cdef extern from "rocksdb/status.h" namespace "rocksdb": 5 | cdef cppclass Status: 6 | Status() 7 | cpp_bool ok() nogil 8 | cpp_bool IsNotFound() nogil const 9 | cpp_bool IsCorruption() nogil const 10 | cpp_bool IsNotSupported() nogil const 11 | cpp_bool IsInvalidArgument() nogil const 12 | cpp_bool IsIOError() nogil const 13 | cpp_bool IsMergeInProgress() nogil const 14 | cpp_bool IsIncomplete() nogil const 15 | string ToString() nogil except+ 16 | -------------------------------------------------------------------------------- /rocksdb/iterator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from .slice_ cimport Slice 3 | from .status cimport Status 4 | 5 | cdef extern from "rocksdb/iterator.h" namespace "rocksdb": 6 | cdef cppclass Iterator: 7 | cpp_bool Valid() nogil except+ 8 | void SeekToFirst() nogil except+ 9 | void SeekToLast() nogil except+ 10 | void Seek(const Slice&) nogil except+ 11 | void Next() nogil except+ 12 | void Prev() nogil except+ 13 | void SeekForPrev(const Slice&) nogil except+ 14 | Slice key() nogil except+ 15 | Slice value() nogil except+ 16 | Status status() nogil except+ 17 | -------------------------------------------------------------------------------- /rocksdb/slice_.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from libcpp cimport bool as cpp_bool 3 | 4 | cdef extern from "rocksdb/slice.h" namespace "rocksdb": 5 | cdef cppclass Slice: 6 | Slice() nogil 7 | Slice(const char*, size_t) nogil 8 | Slice(const string&) nogil 9 | Slice(const char*) nogil 10 | 11 | const char* data() nogil 12 | size_t size() nogil 13 | cpp_bool empty() nogil 14 | char operator[](int) nogil 15 | void clear() nogil 16 | void remove_prefix(size_t) nogil 17 | string ToString() nogil 18 | string ToString(cpp_bool) nogil 19 | int compare(const Slice&) nogil 20 | cpp_bool starts_with(const Slice&) nogil 21 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35,py36,py37,py38,py39 3 | minversion = 2.0 4 | isolated_build = True 5 | skipsdist = True 6 | 7 | [testenv] 8 | # Install the module in `.` and its `test` extra dependencies from 9 | # setup.cfg. 10 | deps = 11 | .[test] 12 | changedir = / 13 | # Point directly to the installed package, and do not use `python3 -m pytest`. 14 | # This way we prevent importing the module from the current directory instead 15 | # of the installed package, and failing when it cannot find the shared library. 16 | commands = pytest {envsitepackagesdir}/rocksdb 17 | 18 | [testenv:docs] 19 | deps = .[doc] 20 | commands = python3 setup.py build_sphinx -W 21 | 22 | [pytest] 23 | addopts = --verbose --pyargs 24 | norecursedirs = .tox 25 | -------------------------------------------------------------------------------- /rocksdb/comparator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from .slice_ cimport Slice 3 | from .logger cimport Logger 4 | from .std_memory cimport shared_ptr 5 | 6 | cdef extern from "rocksdb/comparator.h" namespace "rocksdb": 7 | cdef cppclass Comparator: 8 | const char* Name() 9 | int Compare(const Slice&, const Slice&) const 10 | 11 | cdef extern const Comparator* BytewiseComparator() nogil except + 12 | 13 | ctypedef int (*compare_func)( 14 | void*, 15 | Logger*, 16 | string&, 17 | const Slice&, 18 | const Slice&) 19 | 20 | cdef extern from "cpp/comparator_wrapper.hpp" namespace "py_rocks": 21 | cdef cppclass ComparatorWrapper: 22 | ComparatorWrapper(string, void*, compare_func) nogil except + 23 | void set_info_log(shared_ptr[Logger]) nogil except+ 24 | -------------------------------------------------------------------------------- /rocksdb/merge_operators.py: -------------------------------------------------------------------------------- 1 | import struct as py_struct 2 | from rocksdb.interfaces import AssociativeMergeOperator 3 | 4 | class UintAddOperator(AssociativeMergeOperator): 5 | def merge(self, key, existing_value, value): 6 | if existing_value: 7 | s = py_struct.unpack('Q', existing_value)[0] + py_struct.unpack('Q', value)[0] 8 | return (True, py_struct.pack('Q', s)) 9 | return (True, value) 10 | 11 | def name(self): 12 | return b'uint64add' 13 | 14 | class StringAppendOperator(AssociativeMergeOperator): 15 | def merge(self, key, existing_value, value): 16 | if existing_value: 17 | s = existing_value + b',' + value 18 | return (True, s) 19 | return (True, value) 20 | 21 | def name(self): 22 | return b'StringAppendOperator' 23 | -------------------------------------------------------------------------------- /rocksdb/types.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport uint64_t, uint32_t 2 | from .slice_ cimport Slice 3 | from libcpp.string cimport string 4 | from libcpp cimport bool as cpp_bool 5 | 6 | cdef extern from "rocksdb/types.h" namespace "rocksdb": 7 | ctypedef uint64_t SequenceNumber 8 | 9 | cdef enum EntryType: 10 | kEntryPut 11 | kEntryDelete 12 | kEntrySingleDelete 13 | kEntryMerge 14 | kEntryRangeDeletion 15 | kEntryBlobIndex 16 | kEntryOther 17 | 18 | cdef cppclass FullKey: 19 | Slice user_key 20 | SequenceNumber sequence 21 | EntryType type 22 | 23 | FullKey() except+ 24 | FullKey(const Slice&, const SequenceNumber&, EntryType) except+ 25 | string DebugString(cpp_bool hex) nogil except+ 26 | void clear() nogil except+ 27 | 28 | cpp_bool ParseFullKey(const Slice&, FullKey*) 29 | -------------------------------------------------------------------------------- /rocksdb/tests/test_memtable.py: -------------------------------------------------------------------------------- 1 | # content of test_sample.py 2 | import rocksdb 3 | import pytest 4 | import shutil 5 | import os 6 | import tempfile 7 | 8 | def test_open_skiplist_memtable_factory(): 9 | opts = rocksdb.Options() 10 | opts.memtable_factory = rocksdb.SkipListMemtableFactory() 11 | opts.create_if_missing = True 12 | 13 | loc = tempfile.mkdtemp() 14 | try: 15 | test_db = rocksdb.DB(os.path.join(loc, "test"), opts) 16 | finally: 17 | shutil.rmtree(loc) 18 | 19 | 20 | def test_open_vector_memtable_factory(): 21 | opts = rocksdb.Options() 22 | opts.allow_concurrent_memtable_write = False 23 | opts.memtable_factory = rocksdb.VectorMemtableFactory() 24 | opts.create_if_missing = True 25 | loc = tempfile.mkdtemp() 26 | try: 27 | test_db = rocksdb.DB(os.path.join(loc, "test"), opts) 28 | finally: 29 | shutil.rmtree(loc) 30 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to python-rocksdb's documentation! 2 | ========================================== 3 | 4 | Overview 5 | -------- 6 | 7 | Python bindings to the C++ interface of http://rocksdb.org/ using cython:: 8 | 9 | import rocksdb 10 | db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True)) 11 | db.put(b"a", b"b") 12 | print db.get(b"a") 13 | 14 | Tested with python3.8 and python3.9 and RocksDB version 6.11.4. 15 | 16 | .. toctree:: 17 | :maxdepth: 2 18 | 19 | Instructions how to install 20 | Tutorial 21 | API 22 | Changelog 23 | 24 | 25 | Contributing 26 | ------------ 27 | 28 | Source can be found on `github `_. 29 | Feel free to fork and send pull-requests or create issues on the 30 | `github issue tracker `_ 31 | 32 | RoadMap/TODO 33 | ------------ 34 | 35 | No plans so far. Please submit wishes to the github issues. 36 | 37 | Indices and tables 38 | ================== 39 | 40 | * :ref:`genindex` 41 | * :ref:`modindex` 42 | * :ref:`search` 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ENV SRC /home/tester/src 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update -y && apt-get install -qy \ 6 | locales \ 7 | git \ 8 | wget \ 9 | python \ 10 | python3 \ 11 | python-dev \ 12 | python3-dev \ 13 | python3-pip \ 14 | librocksdb-dev \ 15 | libsnappy-dev \ 16 | zlib1g-dev \ 17 | libbz2-dev \ 18 | liblz4-dev \ 19 | && rm -rf /var/lib/apt/lists/* 20 | 21 | #NOTE(sileht): really no utf-8 in 2017 !? 22 | ENV LANG en_US.UTF-8 23 | RUN update-locale 24 | RUN locale-gen $LANG 25 | 26 | #NOTE(sileht): Upgrade python dev tools 27 | RUN pip3 install -U pip tox virtualenv setuptools pytest Cython 28 | 29 | # Set username same as generic default username. Allows output build to be available to same user 30 | ENV USER_NAME ubuntu 31 | 32 | ARG host_uid=1001 33 | ARG host_gid=1001 34 | RUN groupadd -g $host_gid $USER_NAME && \ 35 | useradd -g $host_gid -m -s /bin/bash -u $host_uid $USER_NAME 36 | 37 | USER $USER_NAME 38 | 39 | ENV BUILD_INPUT_DIR /home/$USER_NAME/workspace 40 | RUN mkdir -p $BUILD_INPUT_DIR 41 | 42 | WORKDIR $BUILD_INPUT_DIR 43 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | python-rocksdb 2 | ============== 3 | 4 | Python bindings for RocksDB. 5 | 6 | See https://rocksdb-tina.readthedocs.io/ for a more comprehensive install and 7 | usage description. 8 | 9 | 10 | Quick install 11 | ------------- 12 | 13 | .. code-block:: bash 14 | 15 | $ pip install rocksdb 16 | 17 | 18 | Quick usage guide 19 | ----------------- 20 | 21 | .. code-block:: python 22 | 23 | >>> import rocksdb 24 | >>> db = rocksdb.DB('test.db', rocksdb.Options(create_if_missing=True)) 25 | >>> db.put(b'a', b'data') 26 | >>> print(db.get(b'a')) 27 | b'data' 28 | 29 | 30 | Acknowledgements 31 | ---------------- 32 | 33 | This project attempts to collect the efforts put into different forks of the 34 | `pyrocksdb`_ project that was originally written by `stephan-hof`_, as sadly 35 | none seems to be actively maintained. In particular, the `python-rocksdb`_ fork 36 | created by `twmht`_, but it also incorporates changes from other forks and 37 | unfinished pull requests. 38 | 39 | .. _python-rocksdb: https://github.com/twmht/python-rocksdb 40 | .. _twmht: https://github.com/twmht 41 | .. _pyrocksdb: https://github.com/stephan-hof/pyrocksdb 42 | .. _stephan-hof: https://github.com/stephan-hof 43 | -------------------------------------------------------------------------------- /rocksdb/filter_policy.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libc.string cimport const_char 4 | from .slice_ cimport Slice 5 | from .std_memory cimport shared_ptr 6 | from .logger cimport Logger 7 | 8 | cdef extern from "rocksdb/filter_policy.h" namespace "rocksdb": 9 | cdef cppclass FilterPolicy: 10 | void CreateFilter(const Slice*, int, string*) nogil except+ 11 | cpp_bool KeyMayMatch(const Slice&, const Slice&) nogil except+ 12 | const_char* Name() nogil except+ 13 | 14 | cdef extern const FilterPolicy* NewBloomFilterPolicy(int) nogil except+ 15 | 16 | ctypedef void (*create_filter_func)( 17 | void*, 18 | Logger*, 19 | string&, 20 | const Slice*, 21 | int, 22 | string*) 23 | 24 | ctypedef cpp_bool (*key_may_match_func)( 25 | void*, 26 | Logger*, 27 | string&, 28 | const Slice&, 29 | const Slice&) 30 | 31 | cdef extern from "cpp/filter_policy_wrapper.hpp" namespace "py_rocks": 32 | cdef cppclass FilterPolicyWrapper: 33 | FilterPolicyWrapper( 34 | string, 35 | void*, 36 | create_filter_func, 37 | key_may_match_func) nogil except+ 38 | 39 | void set_info_log(shared_ptr[Logger]) nogil except+ 40 | -------------------------------------------------------------------------------- /rocksdb/slice_transform.pxd: -------------------------------------------------------------------------------- 1 | from .slice_ cimport Slice 2 | from libcpp.string cimport string 3 | from libcpp cimport bool as cpp_bool 4 | from .logger cimport Logger 5 | from .std_memory cimport shared_ptr 6 | 7 | cdef extern from "rocksdb/slice_transform.h" namespace "rocksdb": 8 | cdef cppclass SliceTransform: 9 | pass 10 | 11 | cdef const SliceTransform* ST_NewCappedPrefixTransform "rocksdb::NewCappedPrefixTransform"( 12 | size_t) nogil except+ 13 | 14 | cdef const SliceTransform* ST_NewFixedPrefixTransform "rocksdb::NewFixedPrefixTransform"( 15 | size_t) nogil except+ 16 | 17 | ctypedef Slice (*transform_func)( 18 | void*, 19 | Logger*, 20 | string&, 21 | const Slice&) 22 | 23 | ctypedef cpp_bool (*in_domain_func)( 24 | void*, 25 | Logger*, 26 | string&, 27 | const Slice&) 28 | 29 | ctypedef cpp_bool (*in_range_func)( 30 | void*, 31 | Logger*, 32 | string&, 33 | const Slice&) 34 | 35 | cdef extern from "cpp/slice_transform_wrapper.hpp" namespace "py_rocks": 36 | cdef cppclass SliceTransformWrapper: 37 | SliceTransformWrapper( 38 | string name, 39 | void*, 40 | transform_func, 41 | in_domain_func, 42 | in_range_func) nogil except+ 43 | void set_info_log(shared_ptr[Logger]) nogil except+ 44 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import platform 5 | import sys 6 | 7 | import pkgconfig 8 | from Cython.Build import cythonize 9 | from setuptools import Extension, setup 10 | 11 | extra_compile_args = [ 12 | '-std=c++11', 13 | '-O3', 14 | '-Wall', 15 | '-Wextra', 16 | '-Wconversion', 17 | '-fno-strict-aliasing', 18 | '-fno-rtti', 19 | ] 20 | 21 | if platform.system() == 'Darwin': 22 | extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] 23 | 24 | if sys.version_info < (3 , 0): 25 | raise Exception('python-rocksdb requires Python 3.x') 26 | 27 | try: 28 | ext_args = pkgconfig.parse('rocksdb') 29 | except pkgconfig.PackageNotFoundError: 30 | include_path = os.environ.get('INCLUDE_PATH') 31 | library_path = os.environ.get('LIBRARY_PATH') 32 | 33 | ext_args = { 34 | 'include_dirs': include_path.split(os.pathsep) if include_path else [], 35 | 'library_dirs': library_path.split(os.pathsep) if library_path else [], 36 | 'libraries': ['rocksdb', 'snappy', 'bz2', 'z', 'lz4'], 37 | } 38 | 39 | rocksdb_extension = Extension( 40 | 'rocksdb._rocksdb', 41 | [ 42 | 'rocksdb/_rocksdb.pyx', 43 | ], 44 | extra_compile_args=extra_compile_args, 45 | language='c++', 46 | **ext_args, 47 | ) 48 | 49 | setup( 50 | ext_modules=cythonize([rocksdb_extension]), 51 | ) 52 | -------------------------------------------------------------------------------- /rocksdb/merge_operator.pxd: -------------------------------------------------------------------------------- 1 | from libcpp.string cimport string 2 | from libcpp cimport bool as cpp_bool 3 | from libcpp.deque cimport deque 4 | from .slice_ cimport Slice 5 | from .logger cimport Logger 6 | from .std_memory cimport shared_ptr 7 | 8 | cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb": 9 | cdef cppclass MergeOperator: 10 | pass 11 | 12 | ctypedef cpp_bool (*merge_func)( 13 | void*, 14 | const Slice&, 15 | const Slice*, 16 | const Slice&, 17 | string*, 18 | Logger*) 19 | 20 | ctypedef cpp_bool (*full_merge_func)( 21 | void* ctx, 22 | const Slice& key, 23 | const Slice* existing_value, 24 | const deque[string]& operand_list, 25 | string* new_value, 26 | Logger* logger) 27 | 28 | ctypedef cpp_bool (*partial_merge_func)( 29 | void* ctx, 30 | const Slice& key, 31 | const Slice& left_op, 32 | const Slice& right_op, 33 | string* new_value, 34 | Logger* logger) 35 | 36 | cdef extern from "cpp/merge_operator_wrapper.hpp" namespace "py_rocks": 37 | cdef cppclass AssociativeMergeOperatorWrapper: 38 | AssociativeMergeOperatorWrapper(string, void*, merge_func) nogil except+ 39 | 40 | cdef cppclass MergeOperatorWrapper: 41 | MergeOperatorWrapper( 42 | string, 43 | void*, 44 | void*, 45 | full_merge_func, 46 | partial_merge_func) nogil except+ 47 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = rocksdb 3 | version = 0.8.0rc3 4 | description = Python bindings for RocksDB 5 | long_description = file: README.rst 6 | long_description_content_type = text/x-rst 7 | author = Martina Ferrari 8 | author_email = tina@tina.pm 9 | url = https://github.com/NightTsarina/python-rocksdb 10 | project_urls = 11 | Bug Reports = https://github.com/NightTsarina/python-rocksdb/issues 12 | Repository = https://github.com/NightTsarina/python-rocksdb 13 | license = BSD 3-Clause License 14 | license_file = LICENSE 15 | keywords = rocksdb bindings 16 | classifiers = 17 | Development Status :: 3 - Alpha 18 | Intended Audience :: Developers 19 | License :: OSI Approved :: BSD License 20 | Programming Language :: Python :: 3 21 | Programming Language :: Python :: 3.7 22 | Programming Language :: Python :: 3.8 23 | Programming Language :: Python :: 3.9 24 | Topic :: Database 25 | 26 | [options] 27 | packages = find: 28 | package_dir = 29 | rocksdb = rocksdb 30 | include_package_data = True 31 | zip_safe = False 32 | install_requires = 33 | setuptools >= 25 34 | test_require = 35 | pytest 36 | 37 | [options.packages.find] 38 | where = . 39 | 40 | [options.extras_require] 41 | doc = 42 | sphinx 43 | sphinx_rtd_theme 44 | test = 45 | pytest 46 | 47 | [build_sphinx] 48 | source-dir = docs 49 | build-dir = docs/_build 50 | all_files = 1 51 | 52 | [upload_sphinx] 53 | upload-dir = docs/_build/html 54 | -------------------------------------------------------------------------------- /rocksdb/backup.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libc.stdint cimport uint32_t 5 | from libc.stdint cimport int64_t 6 | from libc.stdint cimport uint64_t 7 | 8 | from .status cimport Status 9 | from .db cimport DB 10 | from .env cimport Env 11 | 12 | # TODO: For rocksdb >= 6.21.0, change to `rocksdb/utilities/backup_engine.h`. 13 | cdef extern from "rocksdb/utilities/backupable_db.h" namespace "rocksdb": 14 | ctypedef uint32_t BackupID 15 | 16 | # TODO: For rocksdb >= 6.21.0, rename to `BackupEngineOptions`. 17 | cdef cppclass BackupableDBOptions: 18 | BackupableDBOptions(const string& backup_dir) 19 | 20 | cdef struct BackupInfo: 21 | BackupID backup_id 22 | int64_t timestamp 23 | uint64_t size 24 | 25 | cdef cppclass BackupEngine: 26 | Status CreateNewBackup(DB*, cpp_bool) nogil except+ 27 | Status PurgeOldBackups(uint32_t) nogil except+ 28 | Status DeleteBackup(BackupID) nogil except+ 29 | void StopBackup() nogil except+ 30 | void GetBackupInfo(vector[BackupInfo]*) nogil except+ 31 | Status RestoreDBFromBackup(BackupID, string&, string&) nogil except+ 32 | Status RestoreDBFromLatestBackup(string&, string&) nogil except+ 33 | 34 | # TODO: For rocksdb >= 6.21.0, swap order of first two parameters. 35 | cdef Status BackupEngine_Open "rocksdb::BackupEngine::Open"( 36 | Env*, 37 | BackupableDBOptions&, 38 | BackupEngine**) 39 | -------------------------------------------------------------------------------- /.github/workflows/debian.yml: -------------------------------------------------------------------------------- 1 | name: debian 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - 'main' 8 | - 'force_ci/all/**' # For development, forcing all workflows to run. 9 | - 'force_ci/debian/**' # For debugging and/or only forcing this workflow. 10 | 11 | jobs: 12 | debian-build: 13 | name: ${{ matrix.dist }} 14 | runs-on: ubuntu-latest 15 | container: debian:${{ matrix.dist }}-slim 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | dist: [bullseye, bookworm] 20 | steps: 21 | - uses: actions/checkout@v2 22 | 23 | - name: Install build-dependencies 24 | # TODO(dato): find out why setup.py links to compression libraries 25 | # by hand (and hence their development packages needed here). 26 | run: | 27 | apt-get update 28 | apt-get install --no-install-recommends -y \ 29 | build-essential librocksdb-dev cython3 python3-pkgconfig \ 30 | python3-dev python3-pip python3-pytest \ 31 | libsnappy-dev libbz2-dev liblz4-dev libz-dev 32 | 33 | - name: Build pyrocksdb 34 | # TODO(dato): consider using pypa/build --no-isolaiton, to 35 | # build the package using a tool specifically designed for 36 | # that, rather than trusting it to a tool that does a lot 37 | # more (pip). 38 | run: | 39 | python3 -m pip install --no-build-isolation -v '.[test]' 40 | 41 | - name: Run tests 42 | run: | 43 | pytest-3 --pyargs rocksdb 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Stephan Hofmockel 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | * Neither the name of the author nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /rocksdb/interfaces.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | 4 | 5 | class Comparator: 6 | __metaclass__ = ABCMeta 7 | 8 | @abstractmethod 9 | def compare(self, a, b): 10 | pass 11 | 12 | @abstractmethod 13 | def name(self): 14 | pass 15 | 16 | 17 | class AssociativeMergeOperator: 18 | __metaclass__ = ABCMeta 19 | 20 | @abstractmethod 21 | def merge(self, key, existing_value, value): 22 | pass 23 | 24 | @abstractmethod 25 | def name(self): 26 | pass 27 | 28 | 29 | class MergeOperator: 30 | __metaclass__ = ABCMeta 31 | 32 | @abstractmethod 33 | def full_merge(self, key, existing_value, operand_list): 34 | pass 35 | 36 | @abstractmethod 37 | def partial_merge(self, key, left_operand, right_operand): 38 | pass 39 | 40 | @abstractmethod 41 | def name(self): 42 | pass 43 | 44 | 45 | class FilterPolicy: 46 | __metaclass__ = ABCMeta 47 | 48 | @abstractmethod 49 | def name(self): 50 | pass 51 | 52 | @abstractmethod 53 | def create_filter(self, keys): 54 | pass 55 | 56 | @abstractmethod 57 | def key_may_match(self, key, filter_): 58 | pass 59 | 60 | class SliceTransform: 61 | __metaclass__ = ABCMeta 62 | 63 | @abstractmethod 64 | def name(self): 65 | pass 66 | 67 | @abstractmethod 68 | def transform(self, src): 69 | pass 70 | 71 | @abstractmethod 72 | def in_domain(self, src): 73 | pass 74 | 75 | @abstractmethod 76 | def in_range(self, dst): 77 | pass 78 | -------------------------------------------------------------------------------- /rocksdb/tests/test_transaction_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import gc 5 | import unittest 6 | import rocksdb 7 | from itertools import takewhile 8 | import struct 9 | import tempfile 10 | from rocksdb.merge_operators import UintAddOperator, StringAppendOperator 11 | 12 | from .test_db import TestHelper, TestDB 13 | 14 | class TestTransactionDB(TestDB): 15 | """Re-run TestDB with the TransactionDB object""" 16 | 17 | def setUp(self): 18 | TestHelper.setUp(self) 19 | opts = rocksdb.Options(create_if_missing=True) 20 | tdb_opts = rocksdb.TransactionDBOptions() 21 | self.db = rocksdb.TransactionDB( 22 | os.path.join(self.db_loc, "test"), 23 | opts, 24 | tdb_opts=tdb_opts) 25 | 26 | def test_options_used_twice(self): 27 | expected = "Options object is already used by another DB" 28 | tdb_opts = rocksdb.TransactionDBOptions() 29 | with self.assertRaisesRegex(rocksdb.InvalidArgument, expected): 30 | rocksdb.TransactionDB(os.path.join(self.db_loc, "test2"), 31 | self.db.options, 32 | tdb_opts=tdb_opts) 33 | 34 | def test_transaction_options_used_twice(self): 35 | expected = "Transaction Options object is already used by another DB" 36 | opts = rocksdb.Options(create_if_missing=True) 37 | with self.assertRaisesRegex(rocksdb.InvalidArgument, expected): 38 | rocksdb.TransactionDB(os.path.join(self.db_loc, "test2"), 39 | opts, 40 | tdb_opts=self.db.transaction_options) 41 | -------------------------------------------------------------------------------- /rocksdb/statistics.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport uint32_t, uint8_t, uint64_t 2 | from libcpp.memory cimport shared_ptr 3 | from libcpp.string cimport string 4 | from libcpp cimport bool as cpp_bool 5 | from libcpp.map cimport map 6 | from .status cimport Status 7 | 8 | cdef extern from "rocksdb/statistics.h" namespace "rocksdb": 9 | ctypedef enum StatsLevel: 10 | kExceptHistogramOrTimers 11 | kExceptTimers 12 | kExceptDetailedTimers 13 | kExceptTimeForMutex 14 | kAll 15 | 16 | cdef cppclass HistogramData: 17 | double median 18 | double percentile95 19 | double percentile99 20 | double average 21 | double standard_deviation 22 | double max 23 | uint64_t count 24 | uint64_t sum 25 | double min 26 | 27 | cdef cppclass Statistics: 28 | const char* Type() nogil except+ 29 | uint64_t getTickerCount(uint32_t) nogil except+ 30 | void histogramData(uint32_t type, 31 | HistogramData* const) nogil except+ 32 | string getHistogramString(uint32_t) nogil except+ 33 | void recordTick(uint32_t, uint64_t) nogil except+ 34 | void setTickerCount(uint32_t tickerType, uint64_t count) nogil except+ 35 | uint64_t getAndResetTickerCount(uint32_t) nogil except+ 36 | void reportTimeToHistogram(uint32_t, uint64_t) nogil except+ 37 | void measureTime(uint32_t, uint64_t) nogil except+ 38 | void recordInHistogram(uint32_t, uint64_t) nogil except+ 39 | Status Reset() nogil except+ 40 | string ToString() nogil except+ 41 | cpp_bool getTickerMap(map[string, uint64_t]*) nogil except+ 42 | cpp_bool HistEnabledForType(uint32_t type) nogil except+ 43 | void set_stats_level(StatsLevel) nogil except+ 44 | StatsLevel get_stats_level() nogil except+ 45 | -------------------------------------------------------------------------------- /rocksdb/table_factory.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport uint32_t 2 | from libcpp cimport bool as cpp_bool 3 | from .std_memory cimport shared_ptr 4 | 5 | from .cache cimport Cache 6 | from .filter_policy cimport FilterPolicy 7 | 8 | cdef extern from "rocksdb/table.h" namespace "rocksdb": 9 | cdef cppclass TableFactory: 10 | TableFactory() 11 | 12 | ctypedef enum BlockBasedTableIndexType: 13 | kBinarySearch "rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch" 14 | kHashSearch "rocksdb::BlockBasedTableOptions::IndexType::kHashSearch" 15 | 16 | ctypedef enum ChecksumType: 17 | kCRC32c 18 | kxxHash 19 | 20 | cdef cppclass BlockBasedTableOptions: 21 | BlockBasedTableOptions() 22 | BlockBasedTableIndexType index_type 23 | cpp_bool hash_index_allow_collision 24 | ChecksumType checksum 25 | cpp_bool no_block_cache 26 | size_t block_size 27 | int block_size_deviation 28 | int block_restart_interval 29 | cpp_bool whole_key_filtering 30 | shared_ptr[Cache] block_cache 31 | shared_ptr[Cache] block_cache_compressed 32 | shared_ptr[FilterPolicy] filter_policy 33 | cpp_bool enable_index_compression 34 | cpp_bool cache_index_and_filter_blocks 35 | int format_version 36 | 37 | cdef TableFactory* NewBlockBasedTableFactory(const BlockBasedTableOptions&) 38 | 39 | ctypedef enum EncodingType: 40 | kPlain 41 | kPrefix 42 | 43 | cdef cppclass PlainTableOptions: 44 | uint32_t user_key_len 45 | int bloom_bits_per_key 46 | double hash_table_ratio 47 | size_t index_sparseness 48 | size_t huge_page_tlb_size 49 | EncodingType encoding_type 50 | cpp_bool full_scan_mode 51 | cpp_bool store_index_in_file 52 | 53 | cdef TableFactory* NewPlainTableFactory(const PlainTableOptions&) 54 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installing 2 | ========== 3 | .. highlight:: bash 4 | 5 | 6 | With distro package and pypi 7 | **************************** 8 | 9 | This requires librocksdb-dev>=5.0 10 | 11 | .. code-block:: bash 12 | 13 | apt-get install python-virtualenv python-dev librocksdb-dev 14 | virtualenv venv 15 | source venv/bin/activate 16 | pip install python-rocksdb 17 | 18 | From source 19 | *********** 20 | 21 | Building rocksdb 22 | ---------------- 23 | 24 | Briefly describes how to build rocksdb under an ordinary debian/ubuntu. 25 | For more details consider https://github.com/facebook/rocksdb/blob/master/INSTALL.md 26 | 27 | .. code-block:: bash 28 | 29 | apt-get install build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev 30 | git clone https://github.com/facebook/rocksdb.git 31 | cd rocksdb 32 | mkdir build && cd build 33 | cmake .. 34 | make 35 | 36 | Systemwide rocksdb 37 | ^^^^^^^^^^^^^^^^^^ 38 | The following command installs the shared library in ``/usr/lib/`` and the 39 | header files in ``/usr/include/rocksdb/``:: 40 | 41 | make install-shared INSTALL_PATH=/usr 42 | 43 | To uninstall use:: 44 | 45 | make uninstall INSTALL_PATH=/usr 46 | 47 | Local rocksdb 48 | ^^^^^^^^^^^^^ 49 | If you don't like the system wide installation, or you don't have the 50 | permissions, it is possible to set the following environment variables. 51 | These varialbes are picked up by the compiler, linker and loader 52 | 53 | .. code-block:: bash 54 | 55 | export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:`pwd`/../include 56 | export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:`pwd` 57 | export LIBRARY_PATH=${LIBRARY_PATH}:`pwd` 58 | 59 | Building python-rocksdb 60 | ----------------------- 61 | 62 | .. code-block:: bash 63 | 64 | apt-get install python-virtualenv python-dev 65 | virtualenv venv 66 | source venv/bin/activate 67 | pip install git+git://github.com/NightTsarina/python-rocksdb.git#egg=python-rocksdb 68 | -------------------------------------------------------------------------------- /rocksdb/tests/test_transactiondb_options.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import rocksdb 4 | 5 | class TestTransactionDBOptions(unittest.TestCase): 6 | def test_default_getter_setter(self): 7 | NOTNONE = object() 8 | UNSETTABLE = object() 9 | for option, def_value, new_value in ( 10 | ('max_num_locks', -1, 100), 11 | ('max_num_deadlocks', 5, 10), 12 | ('num_stripes', 16, 10), 13 | ('transaction_lock_timeout', 1000, 100), 14 | ('default_lock_timeout', 1000, 10000), 15 | ('rollback_merge_operands', False, True), 16 | ('skip_concurrency_control', False, True), 17 | ('default_write_batch_flush_threshold', 0, 10) 18 | ): 19 | with self.subTest(option=option): 20 | opts = rocksdb.TransactionDBOptions() 21 | if def_value is NOTNONE: 22 | self.assertIsNotNone(getattr(opts, option)) 23 | else: 24 | self.assertEqual(def_value, getattr(opts, option)) 25 | if new_value is UNSETTABLE: 26 | self.assertRaises( 27 | Exception, setattr, opts, option, new_value) 28 | else: 29 | setattr(opts, option, new_value) 30 | self.assertEqual(getattr(opts, option), new_value) 31 | 32 | def test_write_policy(self): 33 | opts = rocksdb.TransactionDBOptions() 34 | self.assertEqual(opts.write_policy, 'write_committed') 35 | accepted_values = ('write_committed', 36 | 'write_prepared', 37 | 'write_unprepared') 38 | for value in accepted_values: 39 | opts.write_policy = value 40 | self.assertEqual(opts.write_policy, value) 41 | 42 | error_value = 'write_dummy' 43 | self.assertRaises( 44 | rocksdb.InvalidArgument, setattr, opts, 'write_policy', error_value) 45 | -------------------------------------------------------------------------------- /rocksdb/cpp/comparator_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/comparator.h" 2 | #include "rocksdb/env.h" 3 | #include 4 | 5 | using std::string; 6 | using rocksdb::Comparator; 7 | using rocksdb::Slice; 8 | using rocksdb::Logger; 9 | 10 | namespace py_rocks { 11 | class ComparatorWrapper: public Comparator { 12 | public: 13 | typedef int (*compare_func)( 14 | void*, 15 | Logger*, 16 | string&, 17 | const Slice&, 18 | const Slice&); 19 | 20 | ComparatorWrapper( 21 | string name, 22 | void* compare_context, 23 | compare_func compare_callback): 24 | name(name), 25 | compare_context(compare_context), 26 | compare_callback(compare_callback) 27 | {} 28 | 29 | virtual int Compare(const Slice& a, const Slice& b) const { 30 | string error_msg; 31 | int val; 32 | 33 | val = this->compare_callback( 34 | this->compare_context, 35 | this->info_log.get(), 36 | error_msg, 37 | a, 38 | b); 39 | 40 | if (error_msg.size()) { 41 | throw std::runtime_error(error_msg.c_str()); 42 | } 43 | return val; 44 | } 45 | 46 | virtual const char* Name() const { 47 | return this->name.c_str(); 48 | } 49 | 50 | virtual void FindShortestSeparator(string*, const Slice&) const {} 51 | virtual void FindShortSuccessor(string*) const {} 52 | 53 | void set_info_log(std::shared_ptr info_log) { 54 | this->info_log = info_log; 55 | } 56 | 57 | private: 58 | string name; 59 | void* compare_context; 60 | compare_func compare_callback; 61 | std::shared_ptr info_log; 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /rocksdb/cpp/write_batch_iter_helper.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "rocksdb/write_batch.h" 5 | 6 | namespace py_rocks { 7 | 8 | class RecordItemsHandler: public rocksdb::WriteBatch::Handler { 9 | public: 10 | enum Optype {PutRecord, MergeRecord, DeleteRecord}; 11 | 12 | class BatchItem { 13 | public: 14 | BatchItem( 15 | const Optype& op, 16 | uint32_t column_family_id, 17 | const rocksdb::Slice& key, 18 | const rocksdb::Slice& value): 19 | op(op), 20 | column_family_id(column_family_id), 21 | key(key), 22 | value(value) 23 | {} 24 | 25 | const Optype op; 26 | uint32_t column_family_id; 27 | const rocksdb::Slice key; 28 | const rocksdb::Slice value; 29 | }; 30 | 31 | typedef std::vector BatchItems; 32 | 33 | public: 34 | /* Items is filled during iteration. */ 35 | RecordItemsHandler(BatchItems* items): items(items) {} 36 | 37 | virtual rocksdb::Status PutCF( 38 | uint32_t column_family_id, const Slice& key, const Slice& value) { 39 | this->items->emplace_back(PutRecord, column_family_id, key, value); 40 | return rocksdb::Status::OK(); 41 | } 42 | 43 | virtual rocksdb::Status MergeCF( 44 | uint32_t column_family_id, const Slice& key, const Slice& value) { 45 | this->items->emplace_back(MergeRecord, column_family_id, key, value); 46 | return rocksdb::Status::OK(); 47 | } 48 | 49 | virtual rocksdb::Status DeleteCF( 50 | uint32_t column_family_id, const Slice& key) { 51 | this->items->emplace_back(DeleteRecord, column_family_id, key, rocksdb::Slice()); 52 | return rocksdb::Status::OK(); 53 | } 54 | 55 | private: 56 | BatchItems* items; 57 | }; 58 | 59 | rocksdb::Status 60 | get_batch_items(const rocksdb::WriteBatch* batch, RecordItemsHandler::BatchItems* items) { 61 | RecordItemsHandler handler(items); 62 | return batch->Iterate(&handler); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /docs/api/backup.rst: -------------------------------------------------------------------------------- 1 | Backup and Restore 2 | ****************** 3 | 4 | BackupEngine 5 | ============ 6 | 7 | .. py:class:: rocksdb.BackupEngine 8 | 9 | .. py:method:: __init__(backup_dir) 10 | 11 | Creates a object to manage backup of a single database. 12 | 13 | :param unicode backup_dir: Where to keep the backup files. 14 | Has to be different than db.db_name. 15 | For example db.db_name + '/backups'. 16 | 17 | .. py:method:: create_backup(db, flush_before_backup=False) 18 | 19 | Triggers the creation of a backup. 20 | 21 | :param db: Database object to backup. 22 | :type db: :py:class:`rocksdb.DB` 23 | 24 | :param bool flush_before_backup: If ``True`` the current memtable is flushed. 25 | 26 | .. py:method:: restore_backup(backup_id, db_dir, wal_dir) 27 | 28 | Restores the backup from the given id. 29 | 30 | :param int backup_id: id of the backup to restore. 31 | :param unicode db_dir: Target directory to restore backup. 32 | :param unicode wal_dir: Target directory to restore backuped WAL files. 33 | 34 | .. py:method:: restore_latest_backup(db_dir, wal_dir) 35 | 36 | Restores the latest backup. 37 | 38 | :param unicode db_dir: see :py:meth:`restore_backup` 39 | :param unicode wal_dir: see :py:meth:`restore_backup` 40 | 41 | .. py:method:: stop_backup() 42 | 43 | Can be called from another thread to stop the current backup process. 44 | 45 | .. py:method:: purge_old_backups(num_backups_to_keep) 46 | 47 | Deletes all backups (oldest first) until "num_backups_to_keep" are left. 48 | 49 | :param int num_backups_to_keep: Number of backupfiles to keep. 50 | 51 | .. py:method:: delete_backup(backup_id) 52 | 53 | :param int backup_id: Delete the backup with the given id. 54 | 55 | .. py:method:: get_backup_info() 56 | 57 | Returns information about all backups. 58 | 59 | It returns a list of dict's where each dict as the following keys. 60 | 61 | ``backup_id`` 62 | (int): id of this backup. 63 | 64 | ``timestamp`` 65 | (int): Seconds since epoch, when the backup was created. 66 | 67 | ``size`` 68 | (int): Size in bytes of the backup. 69 | -------------------------------------------------------------------------------- /rocksdb/transaction_db.pxd: -------------------------------------------------------------------------------- 1 | from . cimport options 2 | from libc.stdint cimport uint64_t, uint32_t, int64_t 3 | from .status cimport Status 4 | from libcpp cimport bool as cpp_bool 5 | from libcpp.string cimport string 6 | from libcpp.vector cimport vector 7 | from libcpp.memory cimport shared_ptr 8 | from .types cimport SequenceNumber 9 | from .db cimport DB, WriteBatch, ColumnFamilyDescriptor, ColumnFamilyHandle 10 | from .stackable_db cimport StackableDB 11 | 12 | cdef extern from "rocksdb/utilities/transaction_db.h" namespace "rocksdb": 13 | cpdef enum TxnDBWritePolicy: 14 | WRITE_COMMITTED 15 | WRITE_PREPARED 16 | WRITE_UNPREPARED 17 | 18 | cdef cppclass TransactionDBOptions: 19 | int64_t max_num_locks 20 | uint32_t max_num_deadlocks 21 | size_t num_stripes 22 | int64_t transaction_lock_timeout 23 | int64_t default_lock_timeout 24 | # TODO shared_ptr[TransactionDBMutexFactory] custom_mutex_factory 25 | TxnDBWritePolicy write_policy 26 | cpp_bool rollback_merge_operands 27 | cpp_bool skip_concurrency_control 28 | int64_t default_write_batch_flush_threshold 29 | 30 | cdef cppclass TransactionOptions: 31 | cpp_bool set_snapshot 32 | cpp_bool deadlock_detect 33 | cpp_bool use_only_the_last_commit_time_batch_for_recovery 34 | int64_t lock_timeout 35 | int64_t expiration 36 | int64_t deadlock_detect_depth 37 | size_t max_write_batch_size 38 | cpp_bool skip_concurrency_control 39 | cpp_bool skip_prepare 40 | int64_t write_batch_flush_threshold 41 | 42 | cdef cppclass TransactionDBWriteOptimizations: 43 | cpp_bool skip_concurrency_control 44 | cpp_bool skip_duplicate_key_check 45 | 46 | cdef cppclass TransactionDB(StackableDB): 47 | Status Write(const options.WriteOptions&, 48 | const TransactionDBWriteOptimizations&, 49 | WriteBatch*) nogil except+ 50 | 51 | cdef Status TransactionDB_Open "rocksdb::TransactionDB::Open"( 52 | const options.Options&, 53 | const TransactionDBOptions&, 54 | const string&, 55 | TransactionDB**) nogil except+ 56 | 57 | cdef Status TransactionDB_Open_ColumnFamilies "rocksdb::TransactionDB::Open"( 58 | const options.DBOptions&, 59 | const TransactionDBOptions&, 60 | const string&, 61 | const vector[ColumnFamilyDescriptor]&, 62 | vector[ColumnFamilyHandle*]*, 63 | TransactionDB**) nogil except+ 64 | -------------------------------------------------------------------------------- /rocksdb/compaction_filter.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libcpp.memory cimport unique_ptr 5 | from libc.stdint cimport uint32_t 6 | from .slice_ cimport Slice 7 | 8 | cdef extern from "rocksdb/compaction_filter.h" namespace "rocksdb": 9 | cdef cppclass CompactionFilterContext: 10 | cpp_bool is_full_compaction 11 | cpp_bool is_manual_compaction 12 | 13 | cdef enum ValueType "rocksdb::CompactionFilter::ValueType": 14 | ValueType_kValue "rocksdb::CompactionFilter::ValueType::kValue" 15 | ValueType_kMergeOperand "rocksdb::CompactionFilter::ValueType::kMergeOperand" 16 | ValueType_kBlobIndex "rocksdb::CompactionFilter::ValueType::kBlobIndex" 17 | 18 | cdef enum Decision "rocksdb::CompactionFilter::Decision": 19 | Decision_kKeep "rocksdb::CompactionFilter::Decision::kKeep" 20 | Decision_kRemove "rocksdb::CompactionFilter::Decision::kRemove" 21 | Decision_kChangeValue "rocksdb::CompactionFilter::Decision::kChangeValue" 22 | Decision_kRemoveAndSkipUntil "rocksdb::CompactionFilter::Decision::kRemoveAndSkipUntil" 23 | 24 | cdef enum BlobDecision "rocksdb::CompactionFilter::BlobDecision": 25 | BlobDecision_kKeep "rocksdb::CompactionFilter::BlobDecision::kKeep" 26 | BlobDecision_kChangeValue "rocksdb::CompactionFilter::BlobDecision::kChangeValue" 27 | BlobDecision_kCorruption "rocksdb::CompactionFilter::BlobDecision::kCorruption" 28 | BlobDecision_kIOError "rocksdb::CompactionFilter::BlobDecision::kIOError" 29 | 30 | cdef cppclass Context "rocksdb::CompactionFilter::Context": 31 | cpp_bool is_full_compaction 32 | cpp_bool is_manual_compaction 33 | uint32_t column_family_id 34 | 35 | cdef cppclass CompactionFilter: 36 | cpp_bool Filter(int, const Slice&, 37 | const Slice&, 38 | string*, 39 | cpp_bool*) nogil except+ 40 | cpp_bool FilterMergeOperand(int, const Slice&, 41 | const Slice&) nogil except+ 42 | Decision FilterV2(int level, const Slice&, ValueType, 43 | const Slice&, string*, 44 | string*) 45 | BlobDecision PrepareBlobOutput(const Slice&, 46 | const Slice&, 47 | string*) 48 | cpp_bool IgnoreSnapshots() 49 | const char* Name() 50 | 51 | cdef cppclass CompactionFilterFactory: 52 | unique_ptr[CompactionFilter] CreateCompactionFilter( 53 | const Context&) 54 | const char* Name() 55 | -------------------------------------------------------------------------------- /rocksdb/table_properties.pxd: -------------------------------------------------------------------------------- 1 | from libc.stdint cimport uint32_t, uint64_t 2 | from libcpp cimport bool as cpp_bool 3 | from libcpp.string cimport string 4 | from libcpp.vector cimport vector 5 | from libcpp.map cimport map 6 | 7 | from .slice_ cimport Slice 8 | from .status cimport Status 9 | from .types cimport SequenceNumber, EntryType 10 | 11 | 12 | cdef extern from "rocksdb/table_properties.h" namespace "rocksdb": 13 | ctypedef map[string, string] UserCollectedProperties 14 | 15 | cdef cppclass TablePropertiesCollector: 16 | Status Add(const Slice&, const Slice&) 17 | Status AddUserKey(const Slice&, const Slice&, 18 | EntryType, SequenceNumber, 19 | uint64_t) 20 | void BlockAdd(uint64_t, 21 | uint64_t, 22 | uint64_t) 23 | 24 | Status Finish(UserCollectedProperties*) 25 | UserCollectedProperties GetReadableProperties() 26 | const char* Name() 27 | cpp_bool NeedCompact() 28 | 29 | cdef cppclass TablePropertiesCollectorFactory_Context "rocksdb::TablePropertiesCollectorFactory::Context": 30 | uint32_t column_family_id 31 | uint32_t kUnknownColumnFamily 32 | 33 | cdef cppclass TablePropertiesCollectorFactory: 34 | TablePropertiesCollector* CreateTablePropertiesCollector( 35 | TablePropertiesCollectorFactory_Context context) 36 | const char* Name() 37 | string ToString() 38 | 39 | cdef cppclass TableProperties: 40 | uint64_t data_size 41 | uint64_t index_size 42 | uint64_t index_partitions 43 | uint64_t top_level_index_size 44 | uint64_t index_key_is_user_key 45 | uint64_t index_value_is_delta_encoded 46 | uint64_t filter_size 47 | uint64_t raw_key_size 48 | uint64_t raw_value_size 49 | uint64_t num_data_blocks 50 | uint64_t num_entries 51 | uint64_t num_deletions 52 | uint64_t num_merge_operands 53 | uint64_t num_range_deletions 54 | uint64_t format_version 55 | uint64_t fixed_key_len 56 | uint64_t column_family_id 57 | uint64_t creation_time 58 | uint64_t oldest_key_time 59 | uint64_t file_creation_time 60 | 61 | string column_family_name 62 | string filter_policy_name 63 | string comparator_name 64 | string merge_operator_name 65 | string prefix_extractor_name 66 | string property_collectors_names 67 | string compression_name 68 | string compression_options 69 | UserCollectedProperties user_collected_properties 70 | UserCollectedProperties readable_properties 71 | map[string, uint64_t] properties_offsets 72 | string ToString(const string&, 73 | const string&) 74 | void Add(const TableProperties&) 75 | 76 | -------------------------------------------------------------------------------- /rocksdb/cpp/filter_policy_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/filter_policy.h" 2 | #include "rocksdb/env.h" 3 | #include 4 | 5 | using std::string; 6 | using rocksdb::FilterPolicy; 7 | using rocksdb::Slice; 8 | using rocksdb::Logger; 9 | 10 | namespace py_rocks { 11 | class FilterPolicyWrapper: public FilterPolicy { 12 | public: 13 | typedef void (*create_filter_func)( 14 | void* ctx, 15 | Logger*, 16 | string&, 17 | const Slice* keys, 18 | int n, 19 | string* dst); 20 | 21 | typedef bool (*key_may_match_func)( 22 | void* ctx, 23 | Logger*, 24 | string&, 25 | const Slice& key, 26 | const Slice& filter); 27 | 28 | FilterPolicyWrapper( 29 | string name, 30 | void* ctx, 31 | create_filter_func create_filter_callback, 32 | key_may_match_func key_may_match_callback): 33 | name(name), 34 | ctx(ctx), 35 | create_filter_callback(create_filter_callback), 36 | key_may_match_callback(key_may_match_callback) 37 | {} 38 | 39 | virtual void 40 | CreateFilter(const Slice* keys, int n, std::string* dst) const { 41 | string error_msg; 42 | 43 | this->create_filter_callback( 44 | this->ctx, 45 | this->info_log.get(), 46 | error_msg, 47 | keys, 48 | n, 49 | dst); 50 | 51 | if (error_msg.size()) { 52 | throw std::runtime_error(error_msg.c_str()); 53 | } 54 | } 55 | 56 | virtual bool 57 | KeyMayMatch(const Slice& key, const Slice& filter) const { 58 | string error_msg; 59 | bool val; 60 | 61 | val = this->key_may_match_callback( 62 | this->ctx, 63 | this->info_log.get(), 64 | error_msg, 65 | key, 66 | filter); 67 | 68 | if (error_msg.size()) { 69 | throw std::runtime_error(error_msg.c_str()); 70 | } 71 | return val; 72 | } 73 | 74 | virtual const char* Name() const { 75 | return this->name.c_str(); 76 | } 77 | 78 | void set_info_log(std::shared_ptr info_log) { 79 | this->info_log = info_log; 80 | } 81 | 82 | private: 83 | string name; 84 | void* ctx; 85 | create_filter_func create_filter_callback; 86 | key_may_match_func key_may_match_callback; 87 | std::shared_ptr info_log; 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /rocksdb/cpp/slice_transform_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "rocksdb/slice_transform.h" 3 | #include "rocksdb/env.h" 4 | #include 5 | 6 | using std::string; 7 | using rocksdb::SliceTransform; 8 | using rocksdb::Slice; 9 | using rocksdb::Logger; 10 | 11 | namespace py_rocks { 12 | class SliceTransformWrapper: public SliceTransform { 13 | public: 14 | typedef Slice (*transform_func)( 15 | void*, 16 | Logger*, 17 | string&, 18 | const Slice&); 19 | 20 | typedef bool (*in_domain_func)( 21 | void*, 22 | Logger*, 23 | string&, 24 | const Slice&); 25 | 26 | typedef bool (*in_range_func)( 27 | void*, 28 | Logger*, 29 | string&, 30 | const Slice&); 31 | 32 | SliceTransformWrapper( 33 | string name, 34 | void* ctx, 35 | transform_func transform_callback, 36 | in_domain_func in_domain_callback, 37 | in_range_func in_range_callback): 38 | name(name), 39 | ctx(ctx), 40 | transform_callback(transform_callback), 41 | in_domain_callback(in_domain_callback), 42 | in_range_callback(in_range_callback) 43 | {} 44 | 45 | virtual const char* Name() const { 46 | return this->name.c_str(); 47 | } 48 | 49 | virtual Slice Transform(const Slice& src) const { 50 | string error_msg; 51 | Slice val; 52 | 53 | val = this->transform_callback( 54 | this->ctx, 55 | this->info_log.get(), 56 | error_msg, 57 | src); 58 | 59 | if (error_msg.size()) { 60 | throw std::runtime_error(error_msg.c_str()); 61 | } 62 | return val; 63 | } 64 | 65 | virtual bool InDomain(const Slice& src) const { 66 | string error_msg; 67 | bool val; 68 | 69 | val = this->in_domain_callback( 70 | this->ctx, 71 | this->info_log.get(), 72 | error_msg, 73 | src); 74 | 75 | if (error_msg.size()) { 76 | throw std::runtime_error(error_msg.c_str()); 77 | } 78 | return val; 79 | } 80 | 81 | virtual bool InRange(const Slice& dst) const { 82 | string error_msg; 83 | bool val; 84 | 85 | val = this->in_range_callback( 86 | this->ctx, 87 | this->info_log.get(), 88 | error_msg, 89 | dst); 90 | 91 | if (error_msg.size()) { 92 | throw std::runtime_error(error_msg.c_str()); 93 | } 94 | return val; 95 | } 96 | 97 | void set_info_log(std::shared_ptr info_log) { 98 | this->info_log = info_log; 99 | } 100 | 101 | private: 102 | string name; 103 | void* ctx; 104 | transform_func transform_callback; 105 | in_domain_func in_domain_callback; 106 | in_range_func in_range_callback; 107 | std::shared_ptr info_log; 108 | }; 109 | } 110 | -------------------------------------------------------------------------------- /.github/workflows/dist.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sw=2:et:ai:sts=2 2 | name: 'Build distribution' 3 | 4 | on: 5 | # Only run when pushing to main/merging PRs. 6 | push: 7 | branches: 8 | - main 9 | - 'force_ci/all/**' # For development, forcing all workflows to run. 10 | - 'force_ci/dist/**' # For debugging and/or only forcing this workflow. 11 | 12 | jobs: 13 | build_wheels: 14 | name: 'Build wheels' 15 | runs-on: ${{ matrix.os }} 16 | env: 17 | LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }} 18 | strategy: 19 | matrix: 20 | os: [ubuntu-latest] 21 | rocksdb_ver: ['v6.14.6'] 22 | 23 | steps: 24 | - uses: actions/checkout@v2 25 | name: 'Checkout source repository' 26 | 27 | - uses: actions/setup-python@v2 28 | name: 'Set up Python 3.9' 29 | with: 30 | python-version: '3.9' 31 | 32 | - name: 'Install cibuildwheel' 33 | run: | 34 | python3 -m pip install cibuildwheel==1.7.1 35 | 36 | - name: 'Build wheels' 37 | run: | 38 | python3 -m cibuildwheel --output-dir dist 39 | env: 40 | CIBW_MANYLINUX_X86_64_IMAGE: 'manylinux2014' 41 | CIBW_BUILD: 'cp3*' 42 | CIBW_SKIP: '*-manylinux_i686' 43 | # Install python package and test-deps. 44 | CIBW_TEST_REQUIRES: '.[test] pytest' 45 | # Use `--pyargs` to interpret parameter as module to import, not as a 46 | # path, and do not use `python3 -m pytest`. This way we prevent 47 | # importing the module from the current directory instead of the 48 | # installed package, and failing when it cannot find the shared 49 | # library. 50 | CIBW_TEST_COMMAND: 'pytest --pyargs rocksdb' 51 | # Avoid re-building the C library in every iteration by testing for 52 | # the build directory. 53 | CIBW_BEFORE_BUILD: > 54 | yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel 55 | python3-Cython && ( 56 | test -d ${{ env.LIBROCKSDB_PATH }} || ( 57 | git clone https://github.com/facebook/rocksdb --depth 1 58 | --branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} && 59 | cd ${{ env.LIBROCKSDB_PATH }} && 60 | CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4 61 | )) && 62 | pushd ${{ env.LIBROCKSDB_PATH }} && 63 | make install-shared && 64 | ldconfig && 65 | popd 66 | 67 | - uses: actions/upload-artifact@v2 68 | name: 'Upload build artifacts' 69 | with: 70 | path: 'dist/*.whl' 71 | 72 | build_sdist: 73 | name: 'Build source distribution' 74 | runs-on: 'ubuntu-latest' 75 | steps: 76 | - uses: actions/checkout@v2 77 | name: 'Checkout source repository' 78 | 79 | - uses: actions/setup-python@v2 80 | name: 'Set up Python 3.9' 81 | with: 82 | python-version: '3.9' 83 | 84 | - name: 'Build sdist' 85 | run: | 86 | python3 setup.py sdist 87 | 88 | - uses: actions/upload-artifact@v2 89 | name: 'Upload build artifacts' 90 | with: 91 | path: 'dist/*.tar.gz' 92 | 93 | 94 | # upload_pypi: 95 | # name: 'Upload packages' 96 | # needs: ['build_wheels', 'build_sdist'] 97 | # runs-on: 'ubuntu-latest' 98 | # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') 99 | # steps: 100 | # - uses: actions/download-artifact@v2 101 | # name: 'Download artifacts' 102 | # with: 103 | # name: 'artifact' 104 | # path: 'dist' 105 | # 106 | # - uses: pypa/gh-action-pypi-publish@master 107 | # name: 'Publish built packages' 108 | # with: 109 | # user: '__token__' 110 | # password: '${{ secrets.PYPI_API_TOKEN }}' 111 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sw=2:et:ai:sts=2 2 | name: 'Build' 3 | 4 | on: 5 | pull_request: 6 | push: 7 | branches: 8 | - 'main' 9 | - 'force_ci/all/**' # For development, forcing all workflows to run. 10 | - 'force_ci/build/**' # For debugging and/or only forcing this workflow. 11 | 12 | jobs: 13 | # Build the RocksDB C library and cache the result. 14 | librocksdb: 15 | name: 'Build librocksdb' 16 | runs-on: ${{ matrix.os }} 17 | env: 18 | LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }} 19 | strategy: 20 | matrix: 21 | os: [ubuntu-latest] 22 | rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.11.4'] 23 | 24 | steps: 25 | - uses: actions/cache@v2 26 | id: cache-librocksdb 27 | with: 28 | key: ${{ matrix.os }}-librocksdb-${{ matrix.rocksdb_ver }} 29 | path: ${{ env.LIBROCKSDB_PATH }} 30 | 31 | - name: 'Install dependencies' 32 | if: steps.cache-librocksdb.outputs.cache-hit != 'true' 33 | run: > 34 | sudo apt install -y libsnappy-dev libbz2-dev liblz4-dev libz-dev 35 | libgflags-dev libzstd-dev 36 | 37 | - name: 'Clone & build RocksDB ${{ matrix.rocksdb_ver }}' 38 | if: steps.cache-librocksdb.outputs.cache-hit != 'true' 39 | run: > 40 | git clone https://github.com/facebook/rocksdb --depth 1 41 | --branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} && 42 | pushd ${{ env.LIBROCKSDB_PATH }} && 43 | CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4 && 44 | popd 45 | 46 | test: 47 | name: 'Build and test python-rocksdb' 48 | needs: librocksdb 49 | runs-on: ${{ matrix.os }} 50 | env: 51 | LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }} 52 | strategy: 53 | matrix: 54 | os: [ubuntu-latest] 55 | py_ver: ['3.7', '3.8', '3.9'] 56 | rocksdb_ver: ['v6.29.3', 'v6.25.3', 'v6.11.4'] 57 | 58 | steps: 59 | - uses: actions/checkout@v2 60 | name: 'Checkout source repository' 61 | 62 | - uses: actions/setup-python@v2 63 | name: 'Set up Python ${{ matrix.py_ver }}' 64 | with: 65 | python-version: ${{ matrix.py_ver }} 66 | 67 | - name: 'Install C libraries' 68 | # XXX(Tina): The non-development versions are sufficient, but package 69 | # names are difficult to predict. 70 | run: > 71 | sudo apt install -y libsnappy-dev libbz2-dev liblz4-dev libz-dev 72 | libgflags-dev libzstd-dev 73 | 74 | # Recover the pre-built C library. 75 | - uses: actions/cache@v2 76 | id: cache-librocksdb 77 | with: 78 | key: ${{ matrix.os }}-librocksdb-${{ matrix.rocksdb_ver }} 79 | path: ${{ env.LIBROCKSDB_PATH }} 80 | 81 | - name: 'Install RocksDB ${{ matrix.rocksdb_ver }}' 82 | if: steps.cache-librocksdb.outputs.cache-hit == 'true' 83 | # DO NOT FORGET to call `ldconfig`! 84 | run: | 85 | pushd ${{ env.LIBROCKSDB_PATH }} && 86 | sudo make install-shared && 87 | sudo ldconfig && 88 | popd 89 | 90 | - name: Build and install python-rocksdb 91 | # Use `pip install` instead of `setup.py` so build-dependencies from 92 | # `pyproject.toml` are installed, in particular `Cython`, without which 93 | # the build fails in confusing ways. 94 | run: | 95 | python3 -m pip install '.[test]' 96 | 97 | - name: Run tests 98 | # Use `--pyargs` to interpret parameter as module to import, not as a 99 | # path, and do not use `python3 -m pytest`. This way we prevent 100 | # importing the module from the current directory instead of the 101 | # installed package, and failing when it cannot find the shared 102 | # library. 103 | run: | 104 | pytest --pyargs rocksdb 105 | -------------------------------------------------------------------------------- /rocksdb/cpp/merge_operator_wrapper.hpp: -------------------------------------------------------------------------------- 1 | #include "rocksdb/merge_operator.h" 2 | 3 | using std::string; 4 | using std::deque; 5 | using rocksdb::Slice; 6 | using rocksdb::Logger; 7 | using rocksdb::MergeOperator; 8 | using rocksdb::AssociativeMergeOperator; 9 | 10 | namespace py_rocks { 11 | class AssociativeMergeOperatorWrapper: public AssociativeMergeOperator { 12 | public: 13 | typedef bool (*merge_func)( 14 | void*, 15 | const Slice& key, 16 | const Slice* existing_value, 17 | const Slice& value, 18 | std::string* new_value, 19 | Logger* logger); 20 | 21 | 22 | AssociativeMergeOperatorWrapper( 23 | string name, 24 | void* merge_context, 25 | merge_func merge_callback): 26 | name(name), 27 | merge_context(merge_context), 28 | merge_callback(merge_callback) 29 | {} 30 | 31 | virtual bool Merge( 32 | const Slice& key, 33 | const Slice* existing_value, 34 | const Slice& value, 35 | std::string* new_value, 36 | Logger* logger) const 37 | { 38 | return this->merge_callback( 39 | this->merge_context, 40 | key, 41 | existing_value, 42 | value, 43 | new_value, 44 | logger); 45 | } 46 | 47 | virtual const char* Name() const { 48 | return this->name.c_str(); 49 | } 50 | 51 | private: 52 | string name; 53 | void* merge_context; 54 | merge_func merge_callback; 55 | }; 56 | 57 | class MergeOperatorWrapper: public MergeOperator { 58 | public: 59 | typedef bool (*full_merge_func)( 60 | void* ctx, 61 | const Slice& key, 62 | const Slice* existing_value, 63 | const deque& operand_list, 64 | string* new_value, 65 | Logger* logger); 66 | 67 | typedef bool (*partial_merge_func)( 68 | void* ctx, 69 | const Slice& key, 70 | const Slice& left_op, 71 | const Slice& right_op, 72 | string* new_value, 73 | Logger* logger); 74 | 75 | MergeOperatorWrapper( 76 | string name, 77 | void* full_merge_context, 78 | void* partial_merge_context, 79 | full_merge_func full_merge_callback, 80 | partial_merge_func partial_merge_callback): 81 | name(name), 82 | full_merge_context(full_merge_context), 83 | partial_merge_context(partial_merge_context), 84 | full_merge_callback(full_merge_callback), 85 | partial_merge_callback(partial_merge_callback) 86 | {} 87 | 88 | virtual bool FullMerge( 89 | const Slice& key, 90 | const Slice* existing_value, 91 | const deque& operand_list, 92 | string* new_value, 93 | Logger* logger) const 94 | { 95 | return this->full_merge_callback( 96 | this->full_merge_context, 97 | key, 98 | existing_value, 99 | operand_list, 100 | new_value, 101 | logger); 102 | } 103 | 104 | virtual bool PartialMerge ( 105 | const Slice& key, 106 | const Slice& left_operand, 107 | const Slice& right_operand, 108 | string* new_value, 109 | Logger* logger) const 110 | { 111 | return this->partial_merge_callback( 112 | this->partial_merge_context, 113 | key, 114 | left_operand, 115 | right_operand, 116 | new_value, 117 | logger); 118 | } 119 | 120 | virtual const char* Name() const { 121 | return this->name.c_str(); 122 | } 123 | 124 | private: 125 | string name; 126 | void* full_merge_context; 127 | void* partial_merge_context; 128 | full_merge_func full_merge_callback; 129 | partial_merge_func partial_merge_callback; 130 | 131 | }; 132 | } 133 | -------------------------------------------------------------------------------- /rocksdb/advanced_options.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libc.stdint cimport uint64_t 5 | from libc.stdint cimport uint32_t 6 | from libc.stdint cimport int64_t 7 | from libc.stdint cimport int32_t 8 | from .std_memory cimport shared_ptr 9 | from .comparator cimport Comparator 10 | from .merge_operator cimport MergeOperator 11 | from .logger cimport Logger 12 | from .slice_ cimport Slice 13 | from .snapshot cimport Snapshot 14 | from .slice_transform cimport SliceTransform 15 | from .table_factory cimport TableFactory 16 | from .memtablerep cimport MemTableRepFactory 17 | from .universal_compaction cimport CompactionOptionsUniversal 18 | from .cache cimport Cache 19 | from .options cimport Options 20 | from .options cimport CompressionType 21 | from .table_properties cimport TablePropertiesCollectorFactory 22 | 23 | cdef extern from "rocksdb/advanced_options.h" namespace "rocksdb": 24 | ctypedef enum CompactionStyle: 25 | kCompactionStyleLevel 26 | kCompactionStyleUniversal 27 | kCompactionStyleFIFO 28 | kCompactionStyleNone 29 | 30 | ctypedef enum CompactionPri: 31 | kByCompensatedSize 32 | kOldestLargestSeqFirst 33 | kOldestSmallestSeqFirst 34 | kMinOverlappingRatio 35 | 36 | cdef cppclass CompactionOptionsFIFO: 37 | uint64_t max_table_files_size 38 | cpp_bool allow_compaction 39 | CompactionOptionsFIFO() 40 | CompactionOptionsFIFO(uint64_t, cpp_bool) 41 | 42 | cdef cppclass CompressionOptions: 43 | int window_bits; 44 | int level; 45 | int strategy; 46 | uint32_t max_dict_bytes 47 | uint32_t zstd_max_train_bytes 48 | uint32_t parallel_threads 49 | cpp_bool enabled 50 | CompressionOptions() except + 51 | CompressionOptions(int, int, int, int, 52 | int, int, cpp_bool) except + 53 | 54 | cdef enum UpdateStatus: 55 | UPDATE_FAILED 56 | UPDATED_INPLACE 57 | UPDATED 58 | 59 | cdef cppclass AdvancedColumnFamilyOptions: 60 | int max_write_buffer_number 61 | int min_write_buffer_number_to_merge 62 | int max_write_buffer_number_to_maintain 63 | int64_t max_write_buffer_size_to_maintain 64 | cpp_bool inplace_update_support 65 | size_t inplace_update_num_locks 66 | 67 | UpdateStatus (*inplace_callback)(char*, 68 | uint32_t*, 69 | Slice, 70 | string*) 71 | double memtable_prefix_bloom_size_ratio 72 | cpp_bool memtable_whole_key_filtering 73 | size_t memtable_huge_page_size 74 | shared_ptr[const SliceTransform] memtable_insert_with_hint_prefix_extractor 75 | uint32_t bloom_locality 76 | size_t arena_block_size 77 | vector[CompressionType] compression_per_level 78 | int num_levels 79 | int level0_slowdown_writes_trigger 80 | int level0_stop_writes_trigger 81 | uint64_t target_file_size_base 82 | int target_file_size_multiplier 83 | cpp_bool level_compaction_dynamic_level_bytes 84 | double max_bytes_for_level_multiplier 85 | 86 | vector[int] max_bytes_for_level_multiplier_additional 87 | uint64_t max_compaction_bytes 88 | uint64_t soft_pending_compaction_bytes_limit 89 | uint64_t hard_pending_compaction_bytes_limit 90 | 91 | CompactionStyle compaction_style 92 | CompactionPri compaction_pri 93 | CompactionOptionsUniversal compaction_options_universal 94 | 95 | CompactionOptionsFIFO compaction_options_fifo 96 | 97 | uint64_t max_sequential_skip_in_iterations 98 | shared_ptr[MemTableRepFactory] memtable_factory 99 | vector[shared_ptr[TablePropertiesCollectorFactory]] table_properties_collector_factories 100 | 101 | size_t max_successive_merges 102 | cpp_bool optimize_filters_for_hits 103 | 104 | cpp_bool paranoid_file_checks 105 | cpp_bool force_consistency_checks 106 | 107 | cpp_bool report_bg_io_stats 108 | uint64_t ttl 109 | 110 | uint64_t periodic_compaction_seconds 111 | uint64_t sample_for_compression 112 | 113 | AdvancedColumnFamilyOptions(); 114 | AdvancedColumnFamilyOptions(const Options&); 115 | # ---------------- OPTIONS NOT SUPPORTED ANYMORE ---------------- 116 | # But kept for compatibality as they are still in the header files. 117 | int max_mem_compaction_level 118 | double soft_rate_limit 119 | double hard_rate_limit 120 | unsigned int rate_limit_delay_max_milliseconds 121 | cpp_bool purge_redundant_kvs_while_flush 122 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rocksdb.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rocksdb.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/rocksdb" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rocksdb" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /rocksdb/db.pxd: -------------------------------------------------------------------------------- 1 | from . cimport options 2 | from libc.stdint cimport uint64_t, uint32_t 3 | from .status cimport Status 4 | from libcpp cimport bool as cpp_bool 5 | from libcpp.string cimport string 6 | from libcpp.vector cimport vector 7 | from .types cimport SequenceNumber 8 | from .slice_ cimport Slice 9 | from .snapshot cimport Snapshot 10 | from .iterator cimport Iterator 11 | 12 | cdef extern from "rocksdb/write_batch.h" namespace "rocksdb": 13 | cdef cppclass WriteBatch: 14 | WriteBatch() nogil except+ 15 | WriteBatch(string) nogil except+ 16 | void Put(const Slice&, const Slice&) nogil except+ 17 | void Put(ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ 18 | void Merge(const Slice&, const Slice&) nogil except+ 19 | void Merge(ColumnFamilyHandle*, const Slice&, const Slice&) nogil except+ 20 | void Delete(const Slice&) nogil except+ 21 | void Delete(ColumnFamilyHandle*, const Slice&) nogil except+ 22 | void PutLogData(const Slice&) nogil except+ 23 | void Clear() nogil except+ 24 | const string& Data() nogil except+ 25 | int Count() nogil except+ 26 | 27 | cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks": 28 | cdef enum BatchItemOp "RecordItemsHandler::Optype": 29 | BatchItemOpPut "py_rocks::RecordItemsHandler::Optype::PutRecord" 30 | BatchItemOpMerge "py_rocks::RecordItemsHandler::Optype::MergeRecord" 31 | BatchItemOpDelte "py_rocks::RecordItemsHandler::Optype::DeleteRecord" 32 | 33 | cdef cppclass BatchItem "py_rocks::RecordItemsHandler::BatchItem": 34 | BatchItemOp op 35 | uint32_t column_family_id 36 | Slice key 37 | Slice value 38 | 39 | Status get_batch_items(WriteBatch* batch, vector[BatchItem]* items) 40 | 41 | 42 | cdef extern from "rocksdb/db.h" namespace "rocksdb": 43 | string kDefaultColumnFamilyName 44 | 45 | cdef struct LiveFileMetaData: 46 | string name 47 | int level 48 | uint64_t size 49 | string smallestkey 50 | string largestkey 51 | SequenceNumber smallest_seqno 52 | SequenceNumber largest_seqno 53 | 54 | # cdef struct SstFileMetaData: 55 | # uint64_t size 56 | # string name 57 | # uint64_t file_number 58 | # string db_path 59 | # string smallestkey 60 | # string largestkey 61 | # SequenceNumber smallest_seqno 62 | # SequenceNumber largest_seqno 63 | 64 | # cdef struct LevelMetaData: 65 | # int level 66 | # uint64_t size 67 | # string largestkey 68 | # LiveFileMetaData files 69 | 70 | cdef struct ColumnFamilyMetaData: 71 | uint64_t size 72 | uint64_t file_count 73 | # string largestkey 74 | # LevelMetaData levels 75 | 76 | cdef cppclass Range: 77 | Range(const Slice&, const Slice&) 78 | 79 | cdef cppclass DB: 80 | Status Put( 81 | const options.WriteOptions&, 82 | ColumnFamilyHandle*, 83 | const Slice&, 84 | const Slice&) nogil except+ 85 | 86 | Status Delete( 87 | const options.WriteOptions&, 88 | ColumnFamilyHandle*, 89 | const Slice&) nogil except+ 90 | 91 | Status Merge( 92 | const options.WriteOptions&, 93 | ColumnFamilyHandle*, 94 | const Slice&, 95 | const Slice&) nogil except+ 96 | 97 | Status Write( 98 | const options.WriteOptions&, 99 | WriteBatch*) nogil except+ 100 | 101 | Status Get( 102 | const options.ReadOptions&, 103 | ColumnFamilyHandle*, 104 | const Slice&, 105 | string*) nogil except+ 106 | 107 | vector[Status] MultiGet( 108 | const options.ReadOptions&, 109 | const vector[ColumnFamilyHandle*]&, 110 | const vector[Slice]&, 111 | vector[string]*) nogil except+ 112 | 113 | cpp_bool KeyMayExist( 114 | const options.ReadOptions&, 115 | ColumnFamilyHandle*, 116 | Slice&, 117 | string*, 118 | cpp_bool*) nogil except+ 119 | 120 | cpp_bool KeyMayExist( 121 | const options.ReadOptions&, 122 | ColumnFamilyHandle*, 123 | Slice&, 124 | string*) nogil except+ 125 | 126 | Iterator* NewIterator( 127 | const options.ReadOptions&, 128 | ColumnFamilyHandle*) nogil except+ 129 | 130 | void NewIterators( 131 | const options.ReadOptions&, 132 | vector[ColumnFamilyHandle*]&, 133 | vector[Iterator*]*) nogil except+ 134 | 135 | const Snapshot* GetSnapshot() nogil except+ 136 | 137 | void ReleaseSnapshot(const Snapshot*) nogil except+ 138 | 139 | cpp_bool GetProperty( 140 | ColumnFamilyHandle*, 141 | const Slice&, 142 | string*) nogil except+ 143 | 144 | void GetApproximateSizes( 145 | ColumnFamilyHandle*, 146 | const Range* 147 | int, 148 | uint64_t*) nogil except+ 149 | 150 | Status CompactRange( 151 | const options.CompactRangeOptions&, 152 | ColumnFamilyHandle*, 153 | const Slice*, 154 | const Slice*) nogil except+ 155 | 156 | Status CreateColumnFamily( 157 | const options.ColumnFamilyOptions&, 158 | const string&, 159 | ColumnFamilyHandle**) nogil except+ 160 | 161 | Status DropColumnFamily( 162 | ColumnFamilyHandle*) nogil except+ 163 | 164 | int NumberLevels(ColumnFamilyHandle*) nogil except+ 165 | int MaxMemCompactionLevel(ColumnFamilyHandle*) nogil except+ 166 | int Level0StopWriteTrigger(ColumnFamilyHandle*) nogil except+ 167 | const string& GetName() nogil except+ 168 | const options.Options& GetOptions(ColumnFamilyHandle*) nogil except+ 169 | Status Flush(const options.FlushOptions&, ColumnFamilyHandle*) nogil except+ 170 | Status DisableFileDeletions() nogil except+ 171 | Status EnableFileDeletions() nogil except+ 172 | Status Close() nogil except+ 173 | 174 | # TODO: Status GetSortedWalFiles(VectorLogPtr& files) 175 | # TODO: SequenceNumber GetLatestSequenceNumber() 176 | # TODO: Status GetUpdatesSince( 177 | # SequenceNumber seq_number, 178 | # unique_ptr[TransactionLogIterator]*) 179 | 180 | Status DeleteFile(string) nogil except+ 181 | void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+ 182 | void GetColumnFamilyMetaData(ColumnFamilyHandle*, ColumnFamilyMetaData*) nogil except+ 183 | ColumnFamilyHandle* DefaultColumnFamily() 184 | 185 | 186 | cdef Status DB_Open "rocksdb::DB::Open"( 187 | const options.Options&, 188 | const string&, 189 | DB**) nogil except+ 190 | 191 | cdef Status DB_Open_ColumnFamilies "rocksdb::DB::Open"( 192 | const options.Options&, 193 | const string&, 194 | const vector[ColumnFamilyDescriptor]&, 195 | vector[ColumnFamilyHandle*]*, 196 | DB**) nogil except+ 197 | 198 | cdef Status DB_OpenForReadOnly "rocksdb::DB::OpenForReadOnly"( 199 | const options.Options&, 200 | const string&, 201 | DB**, 202 | cpp_bool) nogil except+ 203 | 204 | cdef Status DB_OpenForReadOnly_ColumnFamilies "rocksdb::DB::OpenForReadOnly"( 205 | const options.Options&, 206 | const string&, 207 | const vector[ColumnFamilyDescriptor]&, 208 | vector[ColumnFamilyHandle*]*, 209 | DB**, 210 | cpp_bool) nogil except+ 211 | 212 | cdef Status RepairDB(const string& dbname, const options.Options&) 213 | 214 | cdef Status ListColumnFamilies "rocksdb::DB::ListColumnFamilies" ( 215 | const options.Options&, 216 | const string&, 217 | vector[string]*) nogil except+ 218 | 219 | cdef cppclass ColumnFamilyHandle: 220 | const string& GetName() nogil except+ 221 | int GetID() nogil except+ 222 | 223 | cdef cppclass ColumnFamilyDescriptor: 224 | ColumnFamilyDescriptor() nogil except+ 225 | ColumnFamilyDescriptor( 226 | const string&, 227 | const options.ColumnFamilyOptions&) nogil except+ 228 | string name 229 | options.ColumnFamilyOptions options 230 | 231 | cdef extern from "rocksdb/convenience.h" namespace "rocksdb": 232 | void CancelAllBackgroundWork(DB*, cpp_bool) nogil except+ 233 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ********* 3 | 4 | Version 0.8 5 | ----------- 6 | 7 | Yet Another Fork, started by `Martina Ferrari`_, collecting loose commits from the 8 | many forks of the original project. Summary of commits: 9 | 10 | * Alexander Böhn 11 | 12 | * Allow :py:class:`rocksdb.DB` instances to be manually closed. 13 | 14 | * `@iFA88`_ 15 | 16 | * Many tidying changes. 17 | * Added support for many parameters in different interfaces. 18 | * Create statistics.pxd 19 | * Fixing closing 20 | 21 | * `Andrey Martyanov`_ 22 | 23 | * Build wheel packages 24 | * Update README with simplified installation procedure 25 | 26 | * `Martina Ferrari`_ 27 | 28 | * Fix a few typos. 29 | * Add as_dict option to multi_get. 30 | * Update README, set myself as current author/maintainer, and move most 31 | of setup.py to the configuration file. 32 | 33 | Version 0.7 34 | ----------- 35 | 36 | Version released by `Ming-Hsuan-Tu`_; summary of commits: 37 | 38 | * `Ming-Hsuan-Tu`_ 39 | 40 | * remove full_scan_mode 41 | * change default compaction_pri 42 | 43 | * meridianz 44 | 45 | * Docs: fix typo in installation command line 46 | 47 | * Roman Zeyde 48 | 49 | * Remove `fetch=False` unsupported keyword from 50 | db.iter{items,keys,values} documentation 51 | 52 | * Abhiram R 53 | 54 | * Modified docs to export CPLUS_INCLUDE_PATH, LD_LIBRARY_PATH and 55 | LIBRARY_PATH correctly even if they weren't originally assigned 56 | * Added liblz4-dev as a package to be installed 57 | 58 | * Jason Fried 59 | 60 | * Column Family Support. Add support for Column Families in a runtime 61 | safe way. Add unittests to test functionality Insure all unittests are 62 | passing. Cleaned up unittests to not use a fixed directory in tmp, but 63 | use tempfile 64 | 65 | Version 0.6 66 | ----------- 67 | 68 | Version released by `Ming-Hsuan-Tu`_; summary of commits: 69 | 70 | * `Ming-Hsuan-Tu`_ 71 | 72 | * now support rocksdb 5.3.0 73 | * Merge options source_compaction_factor, max_grandparent_overlap_bytes 74 | and expanded_compaction_factor into max_compaction_bytes 75 | * add default merge operator 76 | * add compaction_pri 77 | * add seekForPrev 78 | * update the usage of default operators 79 | * fix memtable_factory crash 80 | * add testcase for memtable 81 | 82 | * George Mossessian 83 | 84 | * allow snappy_compression as a default option in 85 | test_options.py::TestOptions::test_simple 86 | 87 | * RIMPY BHAROT 88 | 89 | * Update installation.rst. Missing steps need to be added for clean 90 | installation. 91 | 92 | * Chris Hager 93 | 94 | * OSX support for 'pip install' 95 | 96 | * Mehdi Abaakouk 97 | 98 | * Allow to compile the extension everywhere. 99 | 100 | 101 | Version 0.5 102 | ----------- 103 | 104 | Last version released by `Stephan Hofmockel`_; summary of commits: 105 | 106 | * Remove prints from the tests. 107 | * Use another compiler flag wich works for clang and gcc. 108 | * Wrap the `RepairDB` function. 109 | * Get rid of this `extension_defaults` variable. 110 | * Only `cythonize` if Cython is installed. 111 | * Add the `.hpp` `.pxd` `.pyx` files for the sdist. 112 | * Rename `README.md` to `README.rst` so `setup.py` can pick it up. 113 | * Update the installation page by mentioning a "system wide" rocksdb 114 | installation. 115 | * Improve the `README.rst` by adding a quick install/using guide. 116 | * Don't set a theme explicitly. Let `readthedocs` decide itself. 117 | * Change API of `compact_range` to be compatible with the change of 118 | rocksdb. 119 | * No need for the `get_ob` methods on PyCache. 120 | * Add `row_cache` to options. 121 | * Document the new `row_cache` option. 122 | * Update the versions (python, rocksdb) `pyrocksdb` 0.4 was tested with. 123 | * Mention in the changelog that this version is avaialable on pypi. 124 | 125 | Version 0.4 126 | ----------- 127 | This version works with RocksDB v3.12. 128 | 129 | * Added :py:func:`repair_db`. 130 | * Added :py:meth:`rocksdb.Options.row_cache` 131 | * Publish to pypi. 132 | 133 | Backward Incompatible Changes: 134 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 135 | 136 | * Changed API of :py:meth:`rocksdb.DB.compact_range`. 137 | 138 | * Only allow keyword arguments. 139 | * Changed ``reduce_level`` to ``change_level``. 140 | * Add new argument called ``bottommost_level_compaction``. 141 | 142 | 143 | Version 0.3 144 | ----------- 145 | This version works with RocksDB version v3.11. 146 | 147 | Backward Incompatible Changes: 148 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 149 | 150 | **Prefix Seeks:** 151 | 152 | According to this page https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes, 153 | all the prefix related parameters on ``ReadOptions`` are removed. 154 | Rocksdb realizes now if ``Options.prefix_extractor`` is set and uses then 155 | prefix-seeks automatically. This means the following changes on pyrocksdb. 156 | 157 | * DB.iterkeys, DB.itervalues, DB.iteritems have *no* ``prefix`` parameter anymore. 158 | * DB.get, DB.multi_get, DB.key_may_exist, DB.iterkeys, DB.itervalues, DB.iteritems 159 | have *no* ``prefix_seek`` parameter anymore. 160 | 161 | Which means all the iterators walk now always to the *end* of the database. 162 | So if you need to stay within a prefix, write your own code to ensure that. 163 | For DB.iterkeys and DB.iteritems ``itertools.takewhile`` is a possible solution. :: 164 | 165 | from itertools import takewhile 166 | 167 | it = self.db.iterkeys() 168 | it.seek(b'00002') 169 | print list(takewhile(lambda key: key.startswith(b'00002'), it)) 170 | 171 | it = self.db.iteritems() 172 | it.seek(b'00002') 173 | print dict(takewhile(lambda item: item[0].startswith(b'00002'), it)) 174 | 175 | **SST Table Builders:** 176 | 177 | * Removed ``NewTotalOrderPlainTableFactory``, because rocksdb drops it too. 178 | 179 | **Changed Options:** 180 | 181 | In newer versions of rocksdb a bunch of options were moved or removed. 182 | 183 | * Rename ``bloom_bits_per_prefix`` of :py:class:`rocksdb.PlainTableFactory` to ``bloom_bits_per_key`` 184 | * Removed ``Options.db_stats_log_interval``. 185 | * Removed ``Options.disable_seek_compaction`` 186 | * Moved ``Options.no_block_cache`` to ``BlockBasedTableFactory`` 187 | * Moved ``Options.block_size`` to ``BlockBasedTableFactory`` 188 | * Moved ``Options.block_size_deviation`` to ``BlockBasedTableFactory`` 189 | * Moved ``Options.block_restart_interval`` to ``BlockBasedTableFactory`` 190 | * Moved ``Options.whole_key_filtering`` to ``BlockBasedTableFactory`` 191 | * Removed ``Options.table_cache_remove_scan_count_limit`` 192 | * Removed rm_scan_count_limit from ``LRUCache`` 193 | 194 | 195 | New: 196 | ^^^^ 197 | * Make CompactRange available: :py:meth:`rocksdb.DB.compact_range` 198 | * Add init options to :py:class:`rocksdb.BlockBasedTableFactory` 199 | * Add more option to :py:class:`rocksdb.PlainTableFactory` 200 | * Add :py:class:`rocksdb.WriteBatchIterator` 201 | * add :py:attr:`rocksdb.CompressionType.lz4_compression` 202 | * add :py:attr:`rocksdb.CompressionType.lz4hc_compression` 203 | 204 | 205 | Version 0.2 206 | ----------- 207 | 208 | This version works with RocksDB version 2.8.fb. Now you have access to the more 209 | advanced options of rocksdb. Like changing the memtable or SST representation. 210 | It is also possible now to enable *Universal Style Compaction*. 211 | 212 | * Fixed `issue 3 `_. 213 | Which fixed the change of prefix_extractor from raw-pointer to smart-pointer. 214 | 215 | * Support the new :py:attr:`rocksdb.Options.verify_checksums_in_compaction` option. 216 | 217 | * Add :py:attr:`rocksdb.Options.table_factory` option. So you could use the new 218 | 'PlainTableFactories' which are optimized for in-memory-databases. 219 | 220 | * https://github.com/facebook/rocksdb/wiki/PlainTable-Format 221 | * https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database%3F 222 | 223 | * Add :py:attr:`rocksdb.Options.memtable_factory` option. 224 | 225 | * Add options :py:attr:`rocksdb.Options.compaction_style` and 226 | :py:attr:`rocksdb.Options.compaction_options_universal` to change the 227 | compaction style. 228 | 229 | * Update documentation to the new default values 230 | 231 | * allow_mmap_reads=true 232 | * allow_mmap_writes=false 233 | * max_background_flushes=1 234 | * max_open_files=5000 235 | * paranoid_checks=true 236 | * disable_seek_compaction=true 237 | * level0_stop_writes_trigger=24 238 | * level0_slowdown_writes_trigger=20 239 | 240 | * Document new property names for :py:meth:`rocksdb.DB.get_property`. 241 | 242 | Version 0.1 243 | ----------- 244 | 245 | Initial version. Works with rocksdb version 2.7.fb. 246 | 247 | .. _`Martina Ferrari`: https://github.com/NightTsarina/ 248 | .. _`Andrey Martyanov`: https://github.com/martyanov/ 249 | .. _`@iFA88`: https://github.com/iFA88/ 250 | .. _`Ming-Hsuan-Tu`: https://github.com/twmht/ 251 | .. _`Stephan Hofmockel`: https://github.com/stephan-hof/ 252 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # python-rocksdb documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Dec 31 12:50:54 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.todo', 34 | 'sphinx.ext.viewcode', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix of source filenames. 41 | source_suffix = '.rst' 42 | 43 | # The encoding of source files. 44 | #source_encoding = 'utf-8-sig' 45 | 46 | # The master toctree document. 47 | master_doc = 'index' 48 | 49 | # General information about the project. 50 | project = u'python-rocksdb' 51 | copyright = u'2014, sh' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '0.6' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '0.6.7' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = ['_build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | # html_theme = 'default' 106 | 107 | # Theme options are theme-specific and customize the look and feel of a theme 108 | # further. For a list of options available for each theme, see the 109 | # documentation. 110 | #html_theme_options = {} 111 | 112 | # Add any paths that contain custom themes here, relative to this directory. 113 | #html_theme_path = [] 114 | 115 | # The name for this set of Sphinx documents. If None, it defaults to 116 | # " v documentation". 117 | #html_title = None 118 | 119 | # A shorter title for the navigation bar. Default is the same as html_title. 120 | #html_short_title = None 121 | 122 | # The name of an image file (relative to this directory) to place at the top 123 | # of the sidebar. 124 | #html_logo = None 125 | 126 | # The name of an image file (within the static path) to use as favicon of the 127 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 128 | # pixels large. 129 | #html_favicon = None 130 | 131 | # Add any paths that contain custom static files (such as style sheets) here, 132 | # relative to this directory. They are copied after the builtin static files, 133 | # so a file named "default.css" will overwrite the builtin "default.css". 134 | html_static_path = ['_static'] 135 | 136 | # Add any extra paths that contain custom files (such as robots.txt or 137 | # .htaccess) here, relative to this directory. These files are copied 138 | # directly to the root of the documentation. 139 | #html_extra_path = [] 140 | 141 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 142 | # using the given strftime format. 143 | #html_last_updated_fmt = '%b %d, %Y' 144 | 145 | # If true, SmartyPants will be used to convert quotes and dashes to 146 | # typographically correct entities. 147 | #html_use_smartypants = True 148 | 149 | # Custom sidebar templates, maps document names to template names. 150 | #html_sidebars = {} 151 | 152 | # Additional templates that should be rendered to pages, maps page names to 153 | # template names. 154 | #html_additional_pages = {} 155 | 156 | # If false, no module index is generated. 157 | #html_domain_indices = True 158 | 159 | # If false, no index is generated. 160 | #html_use_index = True 161 | 162 | # If true, the index is split into individual pages for each letter. 163 | #html_split_index = False 164 | 165 | # If true, links to the reST sources are added to the pages. 166 | #html_show_sourcelink = True 167 | 168 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 169 | #html_show_sphinx = True 170 | 171 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 172 | #html_show_copyright = True 173 | 174 | # If true, an OpenSearch description file will be output, and all pages will 175 | # contain a tag referring to it. The value of this option must be the 176 | # base URL from which the finished HTML is served. 177 | #html_use_opensearch = '' 178 | 179 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 180 | #html_file_suffix = None 181 | 182 | # Output file base name for HTML help builder. 183 | htmlhelp_basename = 'python-rocksdbdoc' 184 | 185 | 186 | # -- Options for LaTeX output --------------------------------------------- 187 | 188 | latex_elements = { 189 | # The paper size ('letterpaper' or 'a4paper'). 190 | #'papersize': 'letterpaper', 191 | 192 | # The font size ('10pt', '11pt' or '12pt'). 193 | #'pointsize': '10pt', 194 | 195 | # Additional stuff for the LaTeX preamble. 196 | #'preamble': '', 197 | } 198 | 199 | # Grouping the document tree into LaTeX files. List of tuples 200 | # (source start file, target name, title, 201 | # author, documentclass [howto, manual, or own class]). 202 | latex_documents = [ 203 | ('index', 'python-rocksdb.tex', u'python-rocksdb Documentation', 204 | u'sh', 'manual'), 205 | ] 206 | 207 | # The name of an image file (relative to this directory) to place at the top of 208 | # the title page. 209 | #latex_logo = None 210 | 211 | # For "manual" documents, if this is true, then toplevel headings are parts, 212 | # not chapters. 213 | #latex_use_parts = False 214 | 215 | # If true, show page references after internal links. 216 | #latex_show_pagerefs = False 217 | 218 | # If true, show URL addresses after external links. 219 | #latex_show_urls = False 220 | 221 | # Documents to append as an appendix to all manuals. 222 | #latex_appendices = [] 223 | 224 | # If false, no module index is generated. 225 | #latex_domain_indices = True 226 | 227 | 228 | # -- Options for manual page output --------------------------------------- 229 | 230 | # One entry per manual page. List of tuples 231 | # (source start file, name, description, authors, manual section). 232 | man_pages = [ 233 | ('index', 'python-rocksdb', u'python-rocksdb Documentation', 234 | [u'sh'], 1) 235 | ] 236 | 237 | # If true, show URL addresses after external links. 238 | #man_show_urls = False 239 | 240 | 241 | # -- Options for Texinfo output ------------------------------------------- 242 | 243 | # Grouping the document tree into Texinfo files. List of tuples 244 | # (source start file, target name, title, author, 245 | # dir menu entry, description, category) 246 | texinfo_documents = [ 247 | ('index', 'python-rocksdb', u'python-rocksdb Documentation', 248 | u'sh', 'python-rocksdb', 'One line description of project.', 249 | 'Miscellaneous'), 250 | ] 251 | 252 | # Documents to append as an appendix to all manuals. 253 | #texinfo_appendices = [] 254 | 255 | # If false, no module index is generated. 256 | #texinfo_domain_indices = True 257 | 258 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 259 | #texinfo_show_urls = 'footnote' 260 | 261 | # If true, do not generate a @detailmenu in the "Top" node's menu. 262 | #texinfo_no_detailmenu = False 263 | -------------------------------------------------------------------------------- /docs/api/interfaces.rst: -------------------------------------------------------------------------------- 1 | Interfaces 2 | ********** 3 | 4 | Comparator 5 | ========== 6 | 7 | .. py:class:: rocksdb.interfaces.Comparator 8 | 9 | A Comparator object provides a total order across slices that are 10 | used as keys in an sstable or a database. A Comparator implementation 11 | must be thread-safe since rocksdb may invoke its methods concurrently 12 | from multiple threads. 13 | 14 | .. py:method:: compare(a, b) 15 | 16 | Three-way comparison. 17 | 18 | :param bytes a: First field to compare 19 | :param bytes b: Second field to compare 20 | :returns: * -1 if a < b 21 | * 0 if a == b 22 | * 1 if a > b 23 | :rtype: ``int`` 24 | 25 | .. py:method:: name() 26 | 27 | The name of the comparator. Used to check for comparator 28 | mismatches (i.e., a DB created with one comparator is 29 | accessed using a different comparator). 30 | 31 | The client of this package should switch to a new name whenever 32 | the comparator implementation changes in a way that will cause 33 | the relative ordering of any two keys to change. 34 | 35 | Names starting with "rocksdb." are reserved and should not be used 36 | by any clients of this package. 37 | 38 | :rtype: ``bytes`` 39 | 40 | Merge Operator 41 | ============== 42 | 43 | Essentially, a MergeOperator specifies the SEMANTICS of a merge, which only 44 | client knows. It could be numeric addition, list append, string 45 | concatenation, edit data structure, whatever. 46 | The library, on the other hand, is concerned with the exercise of this 47 | interface, at the right time (during get, iteration, compaction...) 48 | 49 | To use merge, the client needs to provide an object implementing one of 50 | the following interfaces: 51 | 52 | * AssociativeMergeOperator - for most simple semantics (always take 53 | two values, and merge them into one value, which is then put back 54 | into rocksdb). 55 | numeric addition and string concatenation are examples. 56 | 57 | * MergeOperator - the generic class for all the more complex operations. 58 | One method (FullMerge) to merge a Put/Delete value with a merge operand. 59 | Another method (PartialMerge) that merges two operands together. 60 | This is especially useful if your key values have a complex structure but 61 | you would still like to support client-specific incremental updates. 62 | 63 | AssociativeMergeOperator is simpler to implement. 64 | MergeOperator is simply more powerful. 65 | 66 | See this page for more details 67 | https://github.com/facebook/rocksdb/wiki/Merge-Operator 68 | 69 | AssociativeMergeOperator 70 | ------------------------ 71 | 72 | .. py:class:: rocksdb.interfaces.AssociativeMergeOperator 73 | 74 | .. py:method:: merge(key, existing_value, value) 75 | 76 | Gives the client a way to express the read -> modify -> write semantics 77 | 78 | :param bytes key: The key that's associated with this merge operation 79 | :param bytes existing_value: The current value in the db. 80 | ``None`` indicates the key does not exist 81 | before this op 82 | :param bytes value: The value to update/merge the existing_value with 83 | 84 | :returns: ``True`` and the new value on success. 85 | All values passed in will be client-specific values. 86 | So if this method returns false, it is because client 87 | specified bad data or there was internal corruption. 88 | The client should assume that this will be treated as an 89 | error by the library. 90 | 91 | :rtype: ``(bool, bytes)`` 92 | 93 | .. py:method:: name() 94 | 95 | The name of the MergeOperator. Used to check for MergeOperator mismatches. 96 | For example a DB created with one MergeOperator is accessed using a 97 | different MergeOperator. 98 | 99 | :rtype: ``bytes`` 100 | 101 | MergeOperator 102 | ------------- 103 | 104 | .. py:class:: rocksdb.interfaces.MergeOperator 105 | 106 | .. py:method:: full_merge(key, existing_value, operand_list) 107 | 108 | Gives the client a way to express the read -> modify -> write semantics 109 | 110 | :param bytes key: The key that's associated with this merge operation. 111 | Client could multiplex the merge operator based on it 112 | if the key space is partitioned and different subspaces 113 | refer to different types of data which have different 114 | merge operation semantics 115 | 116 | :param bytes existing_value: The current value in the db. 117 | ``None`` indicates the key does not exist 118 | before this op 119 | 120 | :param operand_list: The sequence of merge operations to apply. 121 | :type operand_list: list of bytes 122 | 123 | :returns: ``True`` and the new value on success. 124 | All values passed in will be client-specific values. 125 | So if this method returns false, it is because client 126 | specified bad data or there was internal corruption. 127 | The client should assume that this will be treated as an 128 | error by the library. 129 | 130 | :rtype: ``(bool, bytes)`` 131 | 132 | .. py:method:: partial_merge(key, left_operand, right_operand) 133 | 134 | This function performs merge(left_op, right_op) 135 | when both the operands are themselves merge operation types 136 | that you would have passed to a DB::Merge() call in the same order. 137 | For example DB::Merge(key,left_op), followed by DB::Merge(key,right_op)). 138 | 139 | PartialMerge should combine them into a single merge operation that is 140 | returned together with ``True`` 141 | This new value should be constructed such that a call to 142 | DB::Merge(key, new_value) would yield the same result as a call 143 | to DB::Merge(key, left_op) followed by DB::Merge(key, right_op). 144 | 145 | If it is impossible or infeasible to combine the two operations, 146 | return ``(False, None)`` The library will internally keep track of the 147 | operations, and apply them in the correct order once a base-value 148 | (a Put/Delete/End-of-Database) is seen. 149 | 150 | :param bytes key: the key that is associated with this merge operation. 151 | :param bytes left_operand: First operand to merge 152 | :param bytes right_operand: Second operand to merge 153 | :rtype: ``(bool, bytes)`` 154 | 155 | .. note:: 156 | 157 | Presently there is no way to differentiate between error/corruption 158 | and simply "return false". For now, the client should simply return 159 | false in any case it cannot perform partial-merge, regardless of reason. 160 | If there is corruption in the data, handle it in the FullMerge() function, 161 | and return false there. 162 | 163 | .. py:method:: name() 164 | 165 | The name of the MergeOperator. Used to check for MergeOperator mismatches. 166 | For example a DB created with one MergeOperator is accessed using a 167 | different MergeOperator. 168 | 169 | :rtype: ``bytes`` 170 | 171 | FilterPolicy 172 | ============ 173 | 174 | .. py:class:: rocksdb.interfaces.FilterPolicy 175 | 176 | .. py:method:: create_filter(keys) 177 | 178 | Create a bytestring which can act as a filter for keys. 179 | 180 | :param keys: list of keys (potentially with duplicates) 181 | that are ordered according to the user supplied 182 | comparator. 183 | :type keys: list of bytes 184 | 185 | :returns: A filter that summarizes keys 186 | :rtype: ``bytes`` 187 | 188 | .. py:method:: key_may_match(key, filter) 189 | 190 | Check if the key is maybe in the filter. 191 | 192 | :param bytes key: Key for a single entry inside the database 193 | :param bytes filter: Contains the data returned by a preceding call 194 | to create_filter on this class 195 | :returns: This method must return ``True`` if the key was in the list 196 | of keys passed to create_filter(). 197 | This method may return ``True`` or ``False`` if the key was 198 | not on the list, but it should aim to return ``False`` with 199 | a high probability. 200 | :rtype: ``bool`` 201 | 202 | 203 | .. py:method:: name() 204 | 205 | Return the name of this policy. Note that if the filter encoding 206 | changes in an incompatible way, the name returned by this method 207 | must be changed. Otherwise, old incompatible filters may be 208 | passed to methods of this type. 209 | 210 | :rtype: ``bytes`` 211 | 212 | 213 | SliceTransform 214 | ============== 215 | 216 | .. py:class:: rocksdb.interfaces.SliceTransform 217 | 218 | SliceTransform is currently used to implement the 'prefix-API' of rocksdb. 219 | https://github.com/facebook/rocksdb/wiki/Proposal-for-prefix-API 220 | 221 | .. py:method:: transform(src) 222 | 223 | :param bytes src: Full key to extract the prefix from. 224 | 225 | :returns: A tuple of two interges ``(offset, size)``. 226 | Where the first integer is the offset within the ``src`` 227 | and the second the size of the prefix after the offset. 228 | Which means the prefix is generted by ``src[offset:offset+size]`` 229 | 230 | :rtype: ``(int, int)`` 231 | 232 | 233 | .. py:method:: in_domain(src) 234 | 235 | Decide if a prefix can be extraced from ``src``. 236 | Only if this method returns ``True`` :py:meth:`transform` will be 237 | called. 238 | 239 | :param bytes src: Full key to check. 240 | :rtype: ``bool`` 241 | 242 | .. py:method:: in_range(prefix) 243 | 244 | Checks if prefix is a valid prefix 245 | 246 | :param bytes prefix: Prefix to check. 247 | :returns: ``True`` if ``prefix`` is a valid prefix. 248 | :rtype: ``bool`` 249 | 250 | .. py:method:: name() 251 | 252 | Return the name of this transformation. 253 | 254 | :rtype: ``bytes`` 255 | -------------------------------------------------------------------------------- /rocksdb/options.pxd: -------------------------------------------------------------------------------- 1 | from libcpp cimport bool as cpp_bool 2 | from libcpp.string cimport string 3 | from libcpp.vector cimport vector 4 | from libc.stdint cimport uint64_t 5 | from libc.stdint cimport uint32_t 6 | from .std_memory cimport shared_ptr 7 | from .comparator cimport Comparator 8 | from .merge_operator cimport MergeOperator 9 | from .logger cimport Logger 10 | from .slice_ cimport Slice 11 | from .snapshot cimport Snapshot 12 | from .slice_transform cimport SliceTransform 13 | from .table_factory cimport TableFactory 14 | from .statistics cimport Statistics 15 | from .memtablerep cimport MemTableRepFactory 16 | from .universal_compaction cimport CompactionOptionsUniversal 17 | from .cache cimport Cache 18 | from . cimport advanced_options 19 | from .advanced_options cimport CompressionOptions 20 | from .advanced_options cimport AdvancedColumnFamilyOptions 21 | from .env cimport Env 22 | from .types cimport SequenceNumber 23 | from .compaction_filter cimport CompactionFilter, CompactionFilterFactory 24 | from .concurrent_task_limiter cimport ConcurrentTaskLimiter 25 | 26 | cdef extern from "rocksdb/options.h" namespace "rocksdb": 27 | ctypedef enum CpuPriority: 28 | kIdle 29 | kLow 30 | kNormal 31 | kHigh 32 | 33 | ctypedef enum CompressionType: 34 | kNoCompression 35 | kSnappyCompression 36 | kZlibCompression 37 | kBZip2Compression 38 | kLZ4Compression 39 | kLZ4HCCompression 40 | kXpressCompression 41 | kZSTD 42 | kZSTDNotFinalCompression 43 | kDisableCompressionOption 44 | 45 | cdef cppclass ColumnFamilyOptions(AdvancedColumnFamilyOptions): 46 | ColumnFamilyOptions* OldDefaults(int, int) 47 | ColumnFamilyOptions* OptimizeForSmallDb(shared_ptr[Cache]*) 48 | ColumnFamilyOptions* OptimizeForPointLookup(uint64_t) 49 | ColumnFamilyOptions* OptimizeLevelStyleCompaction(uint64_t) 50 | ColumnFamilyOptions* OptimizeUniversalStyleCompaction(uint64_t) 51 | const Comparator* comparator 52 | shared_ptr[MergeOperator] merge_operator 53 | CompactionFilter* compaction_filter 54 | shared_ptr[CompactionFilterFactory] compaction_filter_factory 55 | size_t write_buffer_size 56 | advanced_options.CompressionType compression 57 | advanced_options.CompressionType bottommost_compression 58 | CompressionOptions bottommost_compression_opts 59 | advanced_options.CompactionPri compaction_pri 60 | CompressionOptions compression_opts 61 | int level0_file_num_compaction_trigger 62 | shared_ptr[SliceTransform] prefix_extractor 63 | uint64_t max_bytes_for_level_base 64 | # Deprecated but kept here since it is in the header. 65 | uint64_t snap_refresh_nanos 66 | cpp_bool disable_auto_compactions 67 | shared_ptr[TableFactory] table_factory 68 | 69 | vector[DbPath] cf_paths 70 | shared_ptr[ConcurrentTaskLimiter] compaction_thread_limiter 71 | ColumnFamilyOptions() 72 | ColumnFamilyOptions(const Options& options) 73 | void Dump(Logger*) 74 | 75 | # This needs to be in _rocksdb.pxd so it will export into python 76 | cpdef enum AccessHint "rocksdb::DBOptions::AccessHint": 77 | NONE, 78 | NORMAL, 79 | SEQUENTIAL, 80 | WILLNEED 81 | 82 | cpdef enum WALRecoveryMode: 83 | kTolerateCorruptedTailRecords 84 | kAbsoluteConsistency 85 | kPointInTimeRecovery 86 | kSkipAnyCorruptedRecords 87 | 88 | cdef cppclass DbPath: 89 | string path 90 | uint64_t target_size 91 | 92 | DbPath() except + 93 | DbPath(const string&, uint64_t) except + 94 | 95 | cdef cppclass DBOptions: 96 | DBOptions* OldDefaults(int, int) nogil except+ 97 | DBOptions* OptimizeForSmallDb(shared_ptr[Cache]*) nogil except+ 98 | void IncreaseParallelism(int) nogil except+ 99 | cpp_bool create_if_missing 100 | cpp_bool create_missing_column_families 101 | cpp_bool error_if_exists 102 | cpp_bool paranoid_checks 103 | Env* env 104 | # TODO shared_ptr[RateLimiter] rate_limiter 105 | # TODO shared_ptr[SstFileManager] sst_file_manager 106 | shared_ptr[Logger] info_log 107 | # TODO InfoLogLevel info_log_level 108 | int max_open_files 109 | int max_file_opening_threads 110 | uint64_t max_total_wal_size 111 | shared_ptr[Statistics] statistics 112 | cpp_bool use_fsync 113 | vector[DbPath] db_paths 114 | string db_log_dir 115 | string wal_dir 116 | uint64_t delete_obsolete_files_period_micros 117 | int max_background_jobs 118 | int base_background_compactions 119 | int max_background_compactions 120 | uint32_t max_subcompactions 121 | int max_background_flushes 122 | size_t max_log_file_size 123 | size_t log_file_time_to_roll 124 | size_t keep_log_file_num 125 | size_t recycle_log_file_num 126 | uint64_t max_manifest_file_size 127 | int table_cache_numshardbits 128 | uint64_t WAL_ttl_seconds 129 | uint64_t WAL_size_limit_MB 130 | size_t manifest_preallocation_size 131 | cpp_bool allow_mmap_reads 132 | cpp_bool allow_mmap_writes 133 | cpp_bool use_direct_reads 134 | cpp_bool use_direct_io_for_flush_and_compaction 135 | cpp_bool allow_fallocate 136 | cpp_bool is_fd_close_on_exec 137 | unsigned int stats_dump_period_sec 138 | unsigned int stats_persist_period_sec 139 | cpp_bool persist_stats_to_disk 140 | size_t stats_history_buffer_size 141 | cpp_bool advise_random_on_open 142 | size_t db_write_buffer_size 143 | # TODO shared_ptr[WriteBufferManager] write_buffer_manager 144 | AccessHint access_hint_on_compaction_start 145 | cpp_bool new_table_reader_for_compaction_inputs 146 | size_t compaction_readahead_size 147 | size_t random_access_max_buffer_size 148 | size_t writable_file_max_buffer_size 149 | cpp_bool use_adaptive_mutex 150 | DBOptions() nogil except+ 151 | DBOptions(const Options&) nogil except+ 152 | void Dump(Logger*) nogil except+ 153 | 154 | uint64_t bytes_per_sync 155 | uint64_t wal_bytes_per_sync 156 | cpp_bool strict_bytes_per_sync 157 | # TODO vector[shared_ptr[EventListener]] listeners 158 | cpp_bool enable_thread_tracking 159 | uint64_t delayed_write_rate 160 | cpp_bool enable_pipelined_write 161 | cpp_bool unordered_write 162 | cpp_bool allow_concurrent_memtable_write 163 | cpp_bool enable_write_thread_adaptive_yield 164 | uint64_t max_write_batch_group_size_bytes 165 | uint64_t write_thread_max_yield_usec 166 | uint64_t write_thread_slow_yield_usec 167 | cpp_bool skip_stats_update_on_db_open 168 | cpp_bool skip_checking_sst_file_sizes_on_db_open 169 | WALRecoveryMode wal_recovery_mode 170 | cpp_bool allow_2pc 171 | shared_ptr[Cache] row_cache 172 | # TODO WalFilter* wal_filter 173 | cpp_bool fail_if_options_file_error 174 | cpp_bool dump_malloc_stats 175 | cpp_bool avoid_flush_during_recovery 176 | cpp_bool avoid_flush_during_shutdown 177 | cpp_bool allow_ingest_behind 178 | cpp_bool preserve_deletes 179 | cpp_bool two_write_queues 180 | cpp_bool manual_wal_flush 181 | cpp_bool atomic_flush 182 | cpp_bool avoid_unnecessary_blocking_io 183 | cpp_bool write_dbid_to_manifest 184 | size_t log_readahead_size 185 | # TODO shared_ptr[FileChecksumGenFactory] file_checksum_gen_factory 186 | cpp_bool best_efforts_recovery 187 | 188 | cdef cppclass Options(DBOptions, ColumnFamilyOptions): 189 | Options() except+ 190 | Options(const DBOptions&, const ColumnFamilyOptions&) except+ 191 | Options* OldDefaults(int, int) nogil except+ 192 | void Dump(Logger*) nogil except+ 193 | void DumpCFOptions(Logger*) nogil except+ 194 | Options* PrepareForBulkLoad() nogil except+ 195 | Options* OptimizeForSmallDb() nogil except+ 196 | 197 | ctypedef enum ReadTier: 198 | kReadAllTier 199 | kBlockCacheTier 200 | kPersistedTier 201 | kMemtableTier 202 | 203 | cdef cppclass ReadOptions: 204 | const Snapshot* snapshot 205 | const Slice* iterate_lower_bound 206 | const Slice* iterate_upper_bound 207 | size_t readahead_size 208 | uint64_t max_skippable_internal_keys 209 | ReadTier read_tier 210 | cpp_bool verify_checksums 211 | cpp_bool fill_cache 212 | cpp_bool tailing 213 | cpp_bool managed 214 | cpp_bool total_order_seek 215 | cpp_bool auto_prefix_mode 216 | cpp_bool prefix_same_as_start 217 | cpp_bool pin_data 218 | cpp_bool background_purge_on_iterator_cleanup 219 | cpp_bool ignore_range_deletions 220 | # TODO std::function table_filter 221 | SequenceNumber iter_start_seqnum 222 | const Slice* timestamp 223 | const Slice* iter_start_ts 224 | # TODO std::chrono::microseconds deadline 225 | uint64_t value_size_soft_limit 226 | ReadOptions() nogil except+ 227 | ReadOptions(cpp_bool, cpp_bool) nogil except+ 228 | 229 | cdef cppclass WriteOptions: 230 | cpp_bool sync 231 | cpp_bool disableWAL 232 | cpp_bool ignore_missing_column_families 233 | cpp_bool no_slowdown 234 | cpp_bool low_pri 235 | cpp_bool memtable_insert_hint_per_batch 236 | const Slice* timestamp 237 | WriteOptions() nogil except+ 238 | 239 | cdef cppclass FlushOptions: 240 | cpp_bool wait 241 | cpp_bool allow_write_stall 242 | FlushOptions() nogil except+ 243 | 244 | cdef cppclass CompactionOptions: 245 | CompressionType compression 246 | uint64_t output_file_size_limit 247 | uint32_t max_subcompactions 248 | CompactionOptions() nogil except+ 249 | 250 | ctypedef enum BottommostLevelCompaction: 251 | blc_skip "rocksdb::BottommostLevelCompaction::kSkip" 252 | blc_is_filter "rocksdb::BottommostLevelCompaction::kIfHaveCompactionFilter" 253 | blc_force "rocksdb::BottommostLevelCompaction::kForce" 254 | blc_force_optimized "rocksdb::BottommostLevelCompaction::kForceOptimized" 255 | 256 | cdef cppclass CompactRangeOptions: 257 | cpp_bool exclusive_manual_compaction 258 | cpp_bool change_level 259 | int target_level 260 | uint32_t target_path_id 261 | BottommostLevelCompaction bottommost_level_compaction 262 | cpp_bool allow_write_stall 263 | uint32_t max_subcompactions 264 | 265 | cdef cppclass IngestExternalFileOptions: 266 | cpp_bool move_files 267 | cpp_bool failed_move_fall_back_to_copy 268 | cpp_bool snapshot_consistency 269 | cpp_bool allow_global_seqno 270 | cpp_bool allow_blocking_flush 271 | cpp_bool ingest_behind 272 | cpp_bool write_global_seqno 273 | cpp_bool verify_checksums_before_ingest 274 | size_t verify_checksums_readahead_size 275 | cpp_bool verify_file_checksum 276 | 277 | ctypedef enum TraceFilterType: 278 | kTraceFilterNone 279 | kTraceFilterGet 280 | kTraceFilterWrite 281 | 282 | cdef cppclass TraceOptions: 283 | uint64_t max_trace_file_size 284 | uint64_t sampling_frequency 285 | uint64_t filter 286 | 287 | cdef cppclass ImportColumnFamilyOptions: 288 | cpp_bool move_files 289 | 290 | cdef cppclass SizeApproximationOptions: 291 | cpp_bool include_memtabtles 292 | cpp_bool include_files 293 | double files_size_error_margin 294 | -------------------------------------------------------------------------------- /docs/tutorial/index.rst: -------------------------------------------------------------------------------- 1 | Basic Usage of python-rocksdb 2 | ***************************** 3 | 4 | Open 5 | ==== 6 | 7 | The most basic open call is :: 8 | 9 | import rocksdb 10 | 11 | db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True)) 12 | 13 | A more production ready open can look like this :: 14 | 15 | import rocksdb 16 | 17 | opts = rocksdb.Options() 18 | opts.create_if_missing = True 19 | opts.max_open_files = 300000 20 | opts.write_buffer_size = 67108864 21 | opts.max_write_buffer_number = 3 22 | opts.target_file_size_base = 67108864 23 | 24 | opts.table_factory = rocksdb.BlockBasedTableFactory( 25 | filter_policy=rocksdb.BloomFilterPolicy(10), 26 | block_cache=rocksdb.LRUCache(2 * (1024 ** 3)), 27 | block_cache_compressed=rocksdb.LRUCache(500 * (1024 ** 2))) 28 | 29 | db = rocksdb.DB("test.db", opts) 30 | 31 | It assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps 32 | more data (64 MB) in memory before writting a .sst file. 33 | 34 | About Bytes And Unicode 35 | ======================= 36 | 37 | RocksDB stores all data as uninterpreted *byte strings*. 38 | pyrocksdb behaves the same and uses nearly everywhere byte strings too. 39 | In python2 this is the ``str`` type. In python3 the ``bytes`` type. 40 | Since the default string type for string literals differs between python 2 and 3, 41 | it is strongly recommended to use an explicit ``b`` prefix for all byte string 42 | literals in both python2 and python3 code. 43 | For example ``b'this is a byte string'``. This avoids ambiguity and ensures 44 | that your code keeps working as intended if you switch between python2 and python3. 45 | 46 | The only place where you can pass unicode objects are filesytem paths like 47 | 48 | * Directory name of the database itself :py:meth:`rocksdb.DB.__init__` 49 | 50 | * :py:attr:`rocksdb.Options.wal_dir` 51 | 52 | * :py:attr:`rocksdb.Options.db_log_dir` 53 | 54 | To encode this path name, `sys.getfilesystemencoding()` encoding is used. 55 | 56 | Access 57 | ====== 58 | 59 | Store, Get, Delete is straight forward :: 60 | 61 | # Store 62 | db.put(b"key", b"value") 63 | 64 | # Get 65 | db.get(b"key") 66 | 67 | # Delete 68 | db.delete(b"key") 69 | 70 | It is also possible to gather modifications and 71 | apply them in a single operation :: 72 | 73 | batch = rocksdb.WriteBatch() 74 | batch.put(b"key", b"v1") 75 | batch.delete(b"key") 76 | batch.put(b"key", b"v2") 77 | batch.put(b"key", b"v3") 78 | 79 | db.write(batch) 80 | 81 | Fetch of multiple values at once :: 82 | 83 | db.put(b"key1", b"v1") 84 | db.put(b"key2", b"v2") 85 | 86 | ret = db.multi_get([b"key1", b"key2", b"key3"]) 87 | 88 | # prints b"v1" 89 | print ret[b"key1"] 90 | 91 | # prints None 92 | print ret[b"key3"] 93 | 94 | Iteration 95 | ========= 96 | 97 | Iterators behave slightly different than expected. Per default they are not 98 | valid. So you have to call one of its seek methods first :: 99 | 100 | db.put(b"key1", b"v1") 101 | db.put(b"key2", b"v2") 102 | db.put(b"key3", b"v3") 103 | 104 | it = db.iterkeys() 105 | it.seek_to_first() 106 | 107 | # prints [b'key1', b'key2', b'key3'] 108 | print list(it) 109 | 110 | it.seek_to_last() 111 | # prints [b'key3'] 112 | print list(it) 113 | 114 | it.seek(b'key2') 115 | # prints [b'key2', b'key3'] 116 | print list(it) 117 | 118 | There are also methods to iterate over values/items :: 119 | 120 | it = db.itervalues() 121 | it.seek_to_first() 122 | 123 | # prints [b'v1', b'v2', b'v3'] 124 | print list(it) 125 | 126 | it = db.iteritems() 127 | it.seek_to_first() 128 | 129 | # prints [(b'key1', b'v1'), (b'key2, b'v2'), (b'key3', b'v3')] 130 | print list(it) 131 | 132 | Reversed iteration :: 133 | 134 | it = db.iteritems() 135 | it.seek_to_last() 136 | 137 | # prints [(b'key3', b'v3'), (b'key2', b'v2'), (b'key1', b'v1')] 138 | print list(reversed(it)) 139 | 140 | SeekForPrev (Take the example from `https://github.com/facebook/rocksdb/wiki/SeekForPrev`):: 141 | 142 | db.put(b'a1', b'a1_value') 143 | db.put(b'a3', b'a3_value') 144 | db.put(b'b1', b'b1_value') 145 | db.put(b'b2', b'b2_value') 146 | db.put(b'c2', b'c2_value') 147 | db.put(b'c4', b'c4_value') 148 | 149 | it = db.iteritems() 150 | it.seek(b'a1') 151 | assertEqual(it.get(), (b'a1', b'a1_value')) 152 | it.seek(b'a3') 153 | assertEqual(it.get(), (b'a3', b'a3_value')) 154 | it.seek_for_prev(b'c4') 155 | assertEqual(it.get(), (b'c4', b'c4_value')) 156 | it.seek_for_prev(b'c3') 157 | assertEqual(it.get(), (b'c2', b'c2_value')) 158 | 159 | 160 | Snapshots 161 | ========= 162 | 163 | Snapshots are nice to get a consistent view on the database :: 164 | 165 | self.db.put(b"a", b"1") 166 | self.db.put(b"b", b"2") 167 | 168 | snapshot = self.db.snapshot() 169 | self.db.put(b"a", b"2") 170 | self.db.delete(b"b") 171 | 172 | it = self.db.iteritems() 173 | it.seek_to_first() 174 | 175 | # prints {b'a': b'2'} 176 | print dict(it) 177 | 178 | it = self.db.iteritems(snapshot=snapshot) 179 | it.seek_to_first() 180 | 181 | # prints {b'a': b'1', b'b': b'2'} 182 | print dict(it) 183 | 184 | 185 | MergeOperator 186 | ============= 187 | 188 | Merge operators are useful for efficient read-modify-write operations. 189 | For more details see `Merge Operator `_ 190 | 191 | A python merge operator must either implement the 192 | :py:class:`rocksdb.interfaces.AssociativeMergeOperator` or 193 | :py:class:`rocksdb.interfaces.MergeOperator` interface. 194 | 195 | The following example python merge operator implements a counter :: 196 | 197 | class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator): 198 | def merge(self, key, existing_value, value): 199 | if existing_value: 200 | s = int(existing_value) + int(value) 201 | return (True, str(s).encode('ascii')) 202 | return (True, value) 203 | 204 | def name(self): 205 | return b'AssocCounter' 206 | 207 | 208 | opts = rocksdb.Options() 209 | opts.create_if_missing = True 210 | opts.merge_operator = AssocCounter() 211 | db = rocksdb.DB('test.db', opts) 212 | 213 | db.merge(b"a", b"1") 214 | db.merge(b"a", b"1") 215 | 216 | # prints b'2' 217 | print db.get(b"a") 218 | 219 | We provide a set of default operators ``rocksdb.merge_operators.UintAddOperator`` and ``rocksdb.merge_operators.StringAppendOperator``:: 220 | 221 | from rocksdb.merge_operators import UintAddOperator, StringAppendOperator 222 | opts = rocksdb.Options() 223 | opts.create_if_missing = True 224 | # you should also play with StringAppendOperator 225 | opts.merge_operator = UintAddOperator() 226 | db = rocksdb.DB('/tmp/test', opts) 227 | self.db.put(b'a', struct.pack('Q', 5566)) 228 | for x in range(1000): 229 | self.db.merge(b"a", struct.pack('Q', x)) 230 | self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) 231 | 232 | 233 | 234 | PrefixExtractor 235 | =============== 236 | 237 | According to `Prefix API `_ 238 | a prefix_extractor can reduce IO for scans within a prefix range. 239 | A python prefix extractor must implement the :py:class:`rocksdb.interfaces.SliceTransform` interface. 240 | 241 | The following example presents a prefix extractor of a static size. 242 | So always the first 5 bytes are used as the prefix :: 243 | 244 | class StaticPrefix(rocksdb.interfaces.SliceTransform): 245 | def name(self): 246 | return b'static' 247 | 248 | def transform(self, src): 249 | return (0, 5) 250 | 251 | def in_domain(self, src): 252 | return len(src) >= 5 253 | 254 | def in_range(self, dst): 255 | return len(dst) == 5 256 | 257 | opts = rocksdb.Options() 258 | opts.create_if_missing=True 259 | opts.prefix_extractor = StaticPrefix() 260 | 261 | db = rocksdb.DB('test.db', opts) 262 | 263 | db.put(b'00001.x', b'x') 264 | db.put(b'00001.y', b'y') 265 | db.put(b'00001.z', b'z') 266 | 267 | db.put(b'00002.x', b'x') 268 | db.put(b'00002.y', b'y') 269 | db.put(b'00002.z', b'z') 270 | 271 | db.put(b'00003.x', b'x') 272 | db.put(b'00003.y', b'y') 273 | db.put(b'00003.z', b'z') 274 | 275 | prefix = b'00002' 276 | 277 | it = db.iteritems() 278 | it.seek(prefix) 279 | 280 | # prints {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'} 281 | print dict(itertools.takewhile(lambda item: item[0].startswith(prefix), it)) 282 | 283 | 284 | Backup And Restore 285 | ================== 286 | 287 | Backup and Restore is done with a separate :py:class:`rocksdb.BackupEngine` object. 288 | 289 | A backup can only be created on a living database object. :: 290 | 291 | import rocksdb 292 | 293 | db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True)) 294 | db.put(b'a', b'v1') 295 | db.put(b'b', b'v2') 296 | db.put(b'c', b'v3') 297 | 298 | Backup is created like this. 299 | You can choose any path for the backup destination except the db path itself. 300 | If ``flush_before_backup`` is ``True`` the current memtable is flushed to disk 301 | before backup. :: 302 | 303 | backup = rocksdb.BackupEngine("test.db/backups") 304 | backup.create_backup(db, flush_before_backup=True) 305 | 306 | Restore is done like this. 307 | The two arguments are the db_dir and wal_dir, which are mostly the same. :: 308 | 309 | backup = rocksdb.BackupEngine("test.db/backups") 310 | backup.restore_latest_backup("test.db", "test.db") 311 | 312 | 313 | Change Memtable Or SST Implementations 314 | ====================================== 315 | 316 | As noted here :ref:`memtable_factories_label`, RocksDB offers different implementations for the memtable 317 | representation. Per default :py:class:`rocksdb.SkipListMemtableFactory` is used, 318 | but changing it to a different one is veary easy. 319 | 320 | Here is an example for HashSkipList-MemtableFactory. 321 | Keep in mind: To use the hashed based MemtableFactories you must set 322 | :py:attr:`rocksdb.Options.prefix_extractor`. 323 | In this example all keys have a static prefix of len 5. :: 324 | 325 | class StaticPrefix(rocksdb.interfaces.SliceTransform): 326 | def name(self): 327 | return b'static' 328 | 329 | def transform(self, src): 330 | return (0, 5) 331 | 332 | def in_domain(self, src): 333 | return len(src) >= 5 334 | 335 | def in_range(self, dst): 336 | return len(dst) == 5 337 | 338 | 339 | opts = rocksdb.Options() 340 | opts.prefix_extractor = StaticPrefix() 341 | opts.allow_concurrent_memtable_write = False 342 | opts.memtable_factory = rocksdb.HashSkipListMemtableFactory() 343 | opts.create_if_missing = True 344 | 345 | db = rocksdb.DB("test.db", opts) 346 | db.put(b'00001.x', b'x') 347 | db.put(b'00001.y', b'y') 348 | db.put(b'00002.x', b'x') 349 | 350 | For initial bulk loads the Vector-MemtableFactory makes sense. :: 351 | 352 | opts = rocksdb.Options() 353 | opts.allow_concurrent_memtable_write = False 354 | opts.memtable_factory = rocksdb.VectorMemtableFactory() 355 | opts.create_if_missing = True 356 | 357 | db = rocksdb.DB("test.db", opts) 358 | 359 | As noted here :ref:`table_factories_label`, it is also possible to change the 360 | representation of the final data files. 361 | Here is an example how to use a 'PlainTable'. :: 362 | 363 | opts = rocksdb.Options() 364 | opts.table_factory = rocksdb.PlainTableFactory() 365 | opts.create_if_missing = True 366 | 367 | db = rocksdb.DB("test.db", opts) 368 | 369 | Change Compaction Style 370 | ======================= 371 | 372 | RocksDB has a compaction algorithm called *universal*. This style typically 373 | results in lower write amplification but higher space amplification than 374 | Level Style Compaction. See here for more details, 375 | https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide#multi-threaded-compactions 376 | 377 | Here is an example to switch to *universal style compaction*. :: 378 | 379 | opts = rocksdb.Options() 380 | opts.compaction_style = "universal" 381 | opts.compaction_options_universal = {"min_merge_width": 3} 382 | 383 | See here for more options on *universal style compaction*, 384 | :py:attr:`rocksdb.Options.compaction_options_universal` 385 | 386 | Iterate Over WriteBatch 387 | ======================= 388 | 389 | In same cases you need to know, what operations happened on a WriteBatch. 390 | The pyrocksdb WriteBatch supports the iterator protocol, see this example. :: 391 | 392 | batch = rocksdb.WriteBatch() 393 | batch.put(b"key1", b"v1") 394 | batch.delete(b'a') 395 | batch.merge(b'xxx', b'value') 396 | 397 | for op, key, value in batch: 398 | print op, key, value 399 | 400 | # prints the following three lines 401 | # Put key1 v1 402 | # Delete a 403 | # Merge xxx value 404 | -------------------------------------------------------------------------------- /rocksdb/tests/test_options.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import unittest 3 | 4 | import rocksdb 5 | 6 | 7 | class TestFilterPolicy(rocksdb.interfaces.FilterPolicy): 8 | def create_filter(self, keys): 9 | return b'nix' 10 | 11 | def key_may_match(self, key, fil): 12 | return True 13 | 14 | def name(self): 15 | return b'testfilter' 16 | 17 | class TestMergeOperator(rocksdb.interfaces.MergeOperator): 18 | def full_merge(self, *args, **kwargs): 19 | return (False, None) 20 | 21 | def partial_merge(self, *args, **kwargs): 22 | return (False, None) 23 | 24 | def name(self): 25 | return b'testmergeop' 26 | 27 | class TestOptions(unittest.TestCase): 28 | # def test_default_merge_operator(self): 29 | # opts = rocksdb.Options() 30 | # self.assertEqual(True, opts.paranoid_checks) 31 | # opts.paranoid_checks = False 32 | # self.assertEqual(False, opts.paranoid_checks) 33 | 34 | # self.assertIsNone(opts.merge_operator) 35 | # opts.merge_operator = "uint64add" 36 | # self.assertIsNotNone(opts.merge_operator) 37 | # self.assertEqual(opts.merge_operator, "uint64add") 38 | # with self.assertRaises(TypeError): 39 | # opts.merge_operator = "not an operator" 40 | 41 | # FIXME: travis test should include the latest version of rocksdb 42 | def test_compaction_pri(self): 43 | opts = rocksdb.Options() 44 | # default compaction_pri 45 | self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.min_overlapping_ratio) 46 | opts.compaction_pri = rocksdb.CompactionPri.by_compensated_size 47 | self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.by_compensated_size) 48 | opts.compaction_pri = rocksdb.CompactionPri.oldest_largest_seq_first 49 | self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.oldest_largest_seq_first) 50 | opts.compaction_pri = rocksdb.CompactionPri.min_overlapping_ratio 51 | self.assertEqual(opts.compaction_pri, rocksdb.CompactionPri.min_overlapping_ratio) 52 | 53 | def test_enable_write_thread_adaptive_yield(self): 54 | opts = rocksdb.Options() 55 | self.assertEqual(opts.enable_write_thread_adaptive_yield, True) 56 | opts.enable_write_thread_adaptive_yield = False 57 | self.assertEqual(opts.enable_write_thread_adaptive_yield, False) 58 | 59 | def test_allow_concurrent_memtable_write(self): 60 | opts = rocksdb.Options() 61 | self.assertEqual(opts.allow_concurrent_memtable_write, True) 62 | opts.allow_concurrent_memtable_write = False 63 | self.assertEqual(opts.allow_concurrent_memtable_write, False) 64 | 65 | def test_compression_opts(self): 66 | opts = rocksdb.Options() 67 | compression_opts = opts.compression_opts 68 | # default value 69 | self.assertEqual(isinstance(compression_opts, dict), True) 70 | self.assertEqual(compression_opts['window_bits'], -14) 71 | self.assertEqual(compression_opts['level'], 2**15 - 1) 72 | self.assertEqual(compression_opts['strategy'], 0) 73 | self.assertEqual(compression_opts['max_dict_bytes'], 0) 74 | self.assertEqual(compression_opts['zstd_max_train_bytes'], 0) 75 | self.assertEqual(compression_opts['parallel_threads'], 1) 76 | self.assertEqual(compression_opts['enabled'], False) 77 | 78 | with self.assertRaises(TypeError): 79 | opts.compression_opts = list(1, 2) 80 | 81 | new_opts = { 82 | 'window_bits': 1, 83 | 'level': 2, 84 | 'strategy': 3, 85 | 'max_dict_bytes': 4, 86 | 'zstd_max_train_bytes': 15, 87 | 'parallel_threads': 4, 88 | 'enabled': True} 89 | opts.compression_opts = new_opts 90 | self.assertIsNot(new_opts, opts.compression_opts) 91 | for key, value in new_opts.items(): 92 | self.assertEqual(opts.compression_opts[key], value) 93 | 94 | def test_bottommost_compression_opts(self): 95 | opts = rocksdb.Options() 96 | bottommost_compression_opts = opts.bottommost_compression_opts 97 | # default value 98 | self.assertEqual(isinstance(bottommost_compression_opts, dict), True) 99 | self.assertEqual(bottommost_compression_opts['window_bits'], -14) 100 | self.assertEqual(bottommost_compression_opts['level'], 2**15 - 1) 101 | self.assertEqual(bottommost_compression_opts['strategy'], 0) 102 | self.assertEqual(bottommost_compression_opts['max_dict_bytes'], 0) 103 | self.assertEqual(bottommost_compression_opts['zstd_max_train_bytes'], 0) 104 | self.assertEqual(bottommost_compression_opts['parallel_threads'], 1) 105 | self.assertEqual(bottommost_compression_opts['enabled'], False) 106 | 107 | with self.assertRaises(TypeError): 108 | opts.compression_opts = list(1, 2) 109 | 110 | new_opts = { 111 | 'window_bits': 1, 112 | 'level': 2, 113 | 'strategy': 3, 114 | 'max_dict_bytes': 4, 115 | 'zstd_max_train_bytes': 15, 116 | 'parallel_threads': 4, 117 | 'enabled': True, 118 | } 119 | opts.bottommost_compression_opts = new_opts 120 | self.assertIsNot(new_opts, opts.bottommost_compression_opts) 121 | for key, value in new_opts.items(): 122 | self.assertEqual(opts.bottommost_compression_opts[key], value) 123 | 124 | def test_simple(self): 125 | opts = rocksdb.Options() 126 | self.assertEqual(True, opts.paranoid_checks) 127 | opts.paranoid_checks = False 128 | self.assertEqual(False, opts.paranoid_checks) 129 | 130 | self.assertIsNone(opts.merge_operator) 131 | ob = TestMergeOperator() 132 | opts.merge_operator = ob 133 | self.assertEqual(opts.merge_operator, ob) 134 | 135 | self.assertIsInstance( 136 | opts.comparator, 137 | rocksdb.BytewiseComparator) 138 | 139 | self.assertIn(opts.compression, 140 | (rocksdb.CompressionType.no_compression, 141 | rocksdb.CompressionType.snappy_compression)) 142 | 143 | opts.compression = rocksdb.CompressionType.zstd_compression 144 | self.assertEqual(rocksdb.CompressionType.zstd_compression, opts.compression) 145 | 146 | def test_block_options(self): 147 | rocksdb.BlockBasedTableFactory( 148 | block_size=4096, 149 | filter_policy=TestFilterPolicy(), 150 | block_cache=rocksdb.LRUCache(100)) 151 | 152 | def test_unicode_path(self): 153 | name = b'/tmp/M\xc3\xbcnchen'.decode('utf8') 154 | opts = rocksdb.Options() 155 | opts.db_log_dir = name 156 | opts.wal_dir = name 157 | 158 | self.assertEqual(name, opts.db_log_dir) 159 | self.assertEqual(name, opts.wal_dir) 160 | 161 | def test_table_factory(self): 162 | opts = rocksdb.Options() 163 | self.assertIsNone(opts.table_factory) 164 | 165 | opts.table_factory = rocksdb.BlockBasedTableFactory() 166 | opts.table_factory = rocksdb.PlainTableFactory() 167 | 168 | def test_compaction_style(self): 169 | opts = rocksdb.Options() 170 | self.assertEqual('level', opts.compaction_style) 171 | 172 | opts.compaction_style = 'universal' 173 | self.assertEqual('universal', opts.compaction_style) 174 | 175 | opts.compaction_style = 'level' 176 | self.assertEqual('level', opts.compaction_style) 177 | 178 | if sys.version_info[0] == 3: 179 | assertRaisesRegex = self.assertRaisesRegex 180 | else: 181 | assertRaisesRegex = self.assertRaisesRegexp 182 | 183 | with assertRaisesRegex(Exception, 'Unknown compaction style'): 184 | opts.compaction_style = 'foo' 185 | 186 | def test_compaction_opts_universal(self): 187 | opts = rocksdb.Options() 188 | uopts = opts.compaction_options_universal 189 | self.assertEqual(-1, uopts['compression_size_percent']) 190 | self.assertEqual(200, uopts['max_size_amplification_percent']) 191 | self.assertEqual('total_size', uopts['stop_style']) 192 | self.assertEqual(1, uopts['size_ratio']) 193 | self.assertEqual(2, uopts['min_merge_width']) 194 | self.assertGreaterEqual(4294967295, uopts['max_merge_width']) 195 | 196 | new_opts = {'stop_style': 'similar_size', 'max_merge_width': 30} 197 | opts.compaction_options_universal = new_opts 198 | uopts = opts.compaction_options_universal 199 | 200 | self.assertEqual(-1, uopts['compression_size_percent']) 201 | self.assertEqual(200, uopts['max_size_amplification_percent']) 202 | self.assertEqual('similar_size', uopts['stop_style']) 203 | self.assertEqual(1, uopts['size_ratio']) 204 | self.assertEqual(2, uopts['min_merge_width']) 205 | self.assertEqual(30, uopts['max_merge_width']) 206 | 207 | def test_rocksdb_options(self): 208 | NOTNONE = object() 209 | UNSETTABLE = object() 210 | for option, def_value, new_value in ( 211 | ('max_open_files', NOTNONE, 10), 212 | ('row_cache', None, rocksdb.LRUCache(2*1024*1024)), 213 | ('max_file_opening_threads', NOTNONE, 10), 214 | ('max_total_wal_size', NOTNONE, 10), 215 | ('max_background_jobs', NOTNONE, 10), 216 | ('base_background_compactions', NOTNONE, 10), 217 | ('max_background_compactions', NOTNONE, 10), 218 | ('max_subcompactions', NOTNONE, 10), 219 | ('max_background_flushes', NOTNONE, 10), 220 | ('max_log_file_size', NOTNONE, 10), 221 | ('log_file_time_to_roll', NOTNONE, 10), 222 | ('keep_log_file_num', 1000, 10), 223 | ('recycle_log_file_num', NOTNONE, 10), 224 | ('max_manifest_file_size', NOTNONE, 10), 225 | ('table_cache_numshardbits', NOTNONE, 10), 226 | ('wal_ttl_seconds', NOTNONE, 10), 227 | ('wal_size_limit_mb', NOTNONE, 10), 228 | ('manifest_preallocation_size', NOTNONE, 10), 229 | ('allow_mmap_reads', False, True), 230 | ('allow_mmap_writes', False, True), 231 | ('use_direct_reads', False, True), 232 | ('use_direct_io_for_flush_and_compaction', False, True), 233 | ('allow_fallocate', True, False), 234 | ('is_fd_close_on_exec', True, False), 235 | ('stats_dump_period_sec', 600, 3600), 236 | ('stats_persist_period_sec', 600, 3600), 237 | ('persist_stats_to_disk', False, True), 238 | ('stats_history_buffer_size', 1024*1024, 1024), 239 | ('advise_random_on_open', True, False), 240 | ('db_write_buffer_size', 0, 100), 241 | ('new_table_reader_for_compaction_inputs', False, True), 242 | ('compaction_readahead_size', 0, 10), 243 | ('random_access_max_buffer_size', 1024*1024, 100), 244 | ('writable_file_max_buffer_size', 1024*1024, 100), 245 | ('use_adaptive_mutex', False, True), 246 | ('bytes_per_sync', 0, 10), 247 | ('wal_bytes_per_sync', 0, 10), 248 | ('strict_bytes_per_sync', False, True), 249 | ('enable_thread_tracking', False, True), 250 | ('delayed_write_rate', 0, 10), 251 | ('enable_pipelined_write', False, True), 252 | ('unordered_write', False, True), 253 | ('allow_concurrent_memtable_write', True, False), 254 | ('enable_write_thread_adaptive_yield', True, False), 255 | ('max_write_batch_group_size_bytes', 1 << 20, 10), 256 | ('write_thread_max_yield_usec', 100, 200), 257 | ('write_thread_slow_yield_usec', 3, 2000), 258 | ('skip_stats_update_on_db_open', False, True), 259 | ('skip_checking_sst_file_sizes_on_db_open', False, True), 260 | ('allow_2pc', False, True), 261 | ('fail_if_options_file_error', False, True), 262 | ('dump_malloc_stats', False, True), 263 | ('avoid_flush_during_recovery', False, True), 264 | ('avoid_flush_during_shutdown', False, True), 265 | ('allow_ingest_behind', False, True), 266 | ('preserve_deletes', False, True), 267 | ('two_write_queues', False, True), 268 | ('manual_wal_flush', False, True), 269 | ('atomic_flush', False, True), 270 | ('avoid_unnecessary_blocking_io', False, True), 271 | ('write_dbid_to_manifest', False, True), 272 | ('log_readahead_size', 0, 10), 273 | ('best_efforts_recovery', False, True), 274 | ): 275 | with self.subTest(option=option): 276 | opts = rocksdb.Options() 277 | if def_value is NOTNONE: 278 | self.assertIsNotNone(getattr(opts, option)) 279 | else: 280 | self.assertEqual(def_value, getattr(opts, option)) 281 | if new_value is UNSETTABLE: 282 | self.assertRaises( 283 | Exception, setattr, opts, option, new_value) 284 | else: 285 | setattr(opts, option, new_value) 286 | self.assertEqual(getattr(opts, option), new_value) 287 | -------------------------------------------------------------------------------- /docs/api/database.rst: -------------------------------------------------------------------------------- 1 | Database interactions 2 | ********************* 3 | 4 | Database object 5 | =============== 6 | 7 | .. py:class:: rocksdb.DB 8 | 9 | .. py:method:: __init__(db_name, Options opts, read_only=False) 10 | 11 | :param unicode db_name: Name of the database to open 12 | :param opts: Options for this specific database 13 | :type opts: :py:class:`rocksdb.Options` 14 | :param bool read_only: If ``True`` the database is opened read-only. 15 | All DB calls which modify data will raise an 16 | Exception. 17 | 18 | 19 | .. py:method:: put(key, value, sync=False, disable_wal=False, ignore_missing_column_families=False, no_slowdown=False, low_pri=False) 20 | 21 | Set the database entry for "key" to "value". 22 | 23 | :param bytes key: Name for this entry 24 | :param bytes value: Data for this entry 25 | :param bool sync: 26 | If ``True``, the write will be flushed from the operating system 27 | buffer cache (by calling WritableFile::Sync()) before the write 28 | is considered complete. If this flag is true, writes will be 29 | slower. 30 | 31 | If this flag is ``False``, and the machine crashes, some recent 32 | writes may be lost. Note that if it is just the process that 33 | crashes (i.e., the machine does not reboot), no writes will be 34 | lost even if ``sync == False``. 35 | 36 | In other words, a DB write with ``sync == False`` has similar 37 | crash semantics as the "write()" system call. A DB write 38 | with ``sync == True`` has similar crash semantics to a "write()" 39 | system call followed by "fdatasync()". 40 | 41 | :param bool disable_wal: 42 | If ``True``, writes will not first go to the write ahead log, 43 | and the write may got lost after a crash. 44 | 45 | :param bool ignore_missing_column_families: 46 | If ``True`` and if user is trying to write to column families that don't exist 47 | (they were dropped), ignore the write (don't return an error). If there 48 | are multiple writes in a WriteBatch, other writes will succeed. 49 | 50 | :param bool no_slowdown: 51 | If ``True`` and we need to wait or sleep for the write request, fails 52 | immediately with Status::Incomplete(). 53 | 54 | :param bool low_pri: 55 | If ``True``, this write request is of lower priority if compaction is 56 | behind. In this case, no_slowdown = true, the request will be cancelled 57 | immediately with Status::Incomplete() returned. Otherwise, it will be 58 | slowed down. The slowdown value is determined by RocksDB to guarantee 59 | it introduces minimum impacts to high priority writes. 60 | 61 | .. py:method:: delete(key, sync=False, disable_wal=False, ignore_missing_column_families=False, no_slowdown=False, low_pri=False) 62 | 63 | Remove the database entry for "key". 64 | 65 | :param bytes key: Name to delete 66 | :param sync: See :py:meth:`rocksdb.DB.put` 67 | :param disable_wal: See :py:meth:`rocksdb.DB.put` 68 | :param ignore_missing_column_families: See :py:meth:`rocksdb.DB.put` 69 | :param no_slowdown: See :py:meth:`rocksdb.DB.put` 70 | :param low_pri: See :py:meth:`rocksdb.DB.put` 71 | :raises rocksdb.errors.NotFound: If the key did not exists 72 | 73 | .. py:method:: merge(key, value, sync=False, disable_wal=False, ignore_missing_column_families=False, no_slowdown=False, low_pri=False) 74 | 75 | Merge the database entry for "key" with "value". 76 | The semantics of this operation is determined by the user provided 77 | merge_operator when opening DB. 78 | 79 | See :py:meth:`rocksdb.DB.put` for the parameters 80 | 81 | :raises: 82 | :py:exc:`rocksdb.errors.NotSupported` if this is called and 83 | no :py:attr:`rocksdb.Options.merge_operator` was set at creation 84 | 85 | 86 | .. py:method:: write(batch, sync=False, disable_wal=False, ignore_missing_column_families=False, no_slowdown=False, low_pri=False) 87 | 88 | Apply the specified updates to the database. 89 | 90 | :param rocksdb.WriteBatch batch: Batch to apply 91 | :param sync: See :py:meth:`rocksdb.DB.put` 92 | :param disable_wal: See :py:meth:`rocksdb.DB.put` 93 | :param ignore_missing_column_families: See :py:meth:`rocksdb.DB.put` 94 | :param no_slowdown: See :py:meth:`rocksdb.DB.put` 95 | :param low_pri: See :py:meth:`rocksdb.DB.put` 96 | 97 | .. py:method:: get(key, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 98 | 99 | :param bytes key: Name to get 100 | 101 | :param bool verify_checksums: 102 | If ``True``, all data read from underlying storage will be 103 | verified against corresponding checksums. 104 | 105 | :param bool fill_cache: 106 | Should the "data block", "index block" or "filter block" 107 | read for this iteration be cached in memory? 108 | Callers may wish to set this field to ``False`` for bulk scans. 109 | 110 | :param snapshot: 111 | If not ``None``, read as of the supplied snapshot 112 | (which must belong to the DB that is being read and which must 113 | not have been released). Is it ``None`` a implicit snapshot of the 114 | state at the beginning of this read operation is used 115 | :type snapshot: :py:class:`rocksdb.Snapshot` 116 | 117 | :param string read_tier: 118 | Specify if this read request should process data that ALREADY 119 | resides on a particular cache. If the required data is not 120 | found at the specified cache, 121 | then :py:exc:`rocksdb.errors.Incomplete` is raised. 122 | 123 | | Use ``all`` if a fetch from disk is allowed. 124 | | Use ``cache`` if only data from cache is allowed. 125 | 126 | :returns: ``None`` if not found, else the value for this key 127 | 128 | .. py:method:: multi_get(keys, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 129 | 130 | :param keys: Keys to fetch 131 | :type keys: list of bytes 132 | 133 | For the other params see :py:meth:`rocksdb.DB.get` 134 | 135 | :returns: 136 | A ``dict`` where the value is either ``bytes`` or ``None`` if not found 137 | 138 | :raises: If the fetch for a single key fails 139 | 140 | .. note:: 141 | keys will not be "de-duplicated". 142 | Duplicate keys will return duplicate values in order. 143 | 144 | .. py:method:: key_may_exist(key, fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 145 | 146 | If the key definitely does not exist in the database, then this method 147 | returns ``False``, else ``True``. If the caller wants to obtain value 148 | when the key is found in memory, fetch should be set to ``True``. 149 | This check is potentially lighter-weight than invoking DB::get(). 150 | One way to make this lighter weight is to avoid doing any IOs. 151 | 152 | :param bytes key: Key to check 153 | :param bool fetch: Obtain also the value if found 154 | 155 | For the other params see :py:meth:`rocksdb.DB.get` 156 | 157 | :returns: 158 | * ``(True, None)`` if key is found but value not in memory 159 | * ``(True, None)`` if key is found and ``fetch=False`` 160 | * ``(True, )`` if key is found and value in memory and ``fetch=True`` 161 | * ``(False, None)`` if key is not found 162 | 163 | .. py:method:: iterkeys(verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 164 | 165 | Iterate over the keys 166 | 167 | For other params see :py:meth:`rocksdb.DB.get` 168 | 169 | :returns: 170 | A iterator object which is not valid yet. 171 | Call first one of the seek methods of the iterator to position it 172 | 173 | :rtype: :py:class:`rocksdb.BaseIterator` 174 | 175 | .. py:method:: itervalues(verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 176 | 177 | Iterate over the values 178 | 179 | For other params see :py:meth:`rocksdb.DB.get` 180 | 181 | :returns: 182 | A iterator object which is not valid yet. 183 | Call first one of the seek methods of the iterator to position it 184 | 185 | :rtype: :py:class:`rocksdb.BaseIterator` 186 | 187 | .. py:method:: iteritems(verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all") 188 | 189 | Iterate over the items 190 | 191 | For other params see :py:meth:`rocksdb.DB.get` 192 | 193 | :returns: 194 | A iterator object which is not valid yet. 195 | Call first one of the seek methods of the iterator to position it 196 | 197 | :rtype: :py:class:`rocksdb.BaseIterator` 198 | 199 | .. py:method:: snapshot() 200 | 201 | Return a handle to the current DB state. 202 | Iterators created with this handle will all observe a stable snapshot 203 | of the current DB state. 204 | 205 | :rtype: :py:class:`rocksdb.Snapshot` 206 | 207 | 208 | .. py:method:: get_property(prop) 209 | 210 | DB implementations can export properties about their state 211 | via this method. If "property" is a valid property understood by this 212 | DB implementation, a byte string with its value is returned. 213 | Otherwise ``None`` 214 | 215 | Valid property names include: 216 | 217 | * ``b"rocksdb.num-files-at-level"``: return the number of files at level , 218 | where is an ASCII representation of a level number (e.g. "0"). 219 | 220 | * ``b"rocksdb.stats"``: returns a multi-line byte string that describes statistics 221 | about the internal operation of the DB. 222 | 223 | * ``b"rocksdb.sstables"``: returns a multi-line byte string that describes all 224 | of the sstables that make up the db contents. 225 | 226 | * ``b"rocksdb.num-immutable-mem-table"``: Number of immutable mem tables. 227 | 228 | * ``b"rocksdb.mem-table-flush-pending"``: Returns ``1`` if mem table flush is pending, otherwise ``0``. 229 | 230 | * ``b"rocksdb.compaction-pending"``: Returns ``1`` if a compaction is pending, otherweise ``0``. 231 | 232 | * ``b"rocksdb.background-errors"``: Returns accumulated background errors encountered. 233 | 234 | * ``b"rocksdb.cur-size-active-mem-table"``: Returns current size of the active memtable. 235 | 236 | .. py:method:: get_live_files_metadata() 237 | 238 | Returns a list of all table files. 239 | 240 | It returns a list of dict's were each dict has the following keys. 241 | 242 | ``name`` 243 | Name of the file 244 | 245 | ``level`` 246 | Level at which this file resides 247 | 248 | ``size`` 249 | File size in bytes 250 | 251 | ``smallestkey`` 252 | Smallest user defined key in the file 253 | 254 | ``largestkey`` 255 | Largest user defined key in the file 256 | 257 | ``smallest_seqno`` 258 | smallest seqno in file 259 | 260 | ``largest_seqno`` 261 | largest seqno in file 262 | 263 | .. py:method:: compact_range(begin=None, end=None, ** options) 264 | 265 | Compact the underlying storage for the key range [begin,end]. 266 | The actual compaction interval might be superset of [begin, end]. 267 | In particular, deleted and overwritten versions are discarded, 268 | and the data is rearranged to reduce the cost of operations 269 | needed to access the data. 270 | 271 | This operation should typically only be invoked by users who understand 272 | the underlying implementation. 273 | 274 | ``begin == None`` is treated as a key before all keys in the database. 275 | ``end == None`` is treated as a key after all keys in the database. 276 | Therefore the following call will compact the entire database: ``db.compact_range()``. 277 | 278 | Note that after the entire database is compacted, all data are pushed 279 | down to the last level containing any data. If the total data size 280 | after compaction is reduced, that level might not be appropriate for 281 | hosting all the files. In this case, client could set change_level 282 | to ``True``, to move the files back to the minimum level capable of holding 283 | the data set or a given level (specified by non-negative target_level). 284 | 285 | :param bytes begin: Key where to start compaction. 286 | If ``None`` start at the beginning of the database. 287 | :param bytes end: Key where to end compaction. 288 | If ``None`` end at the last key of the database. 289 | :param bool change_level: If ``True``, compacted files will be moved to 290 | the minimum level capable of holding the data 291 | or given level (specified by non-negative target_level). 292 | If ``False`` you may end with a bigger level 293 | than configured. Default is ``False``. 294 | :param int target_level: If change_level is true and target_level have non-negative 295 | value, compacted files will be moved to target_level. 296 | Default is ``-1``. 297 | :param string bottommost_level_compaction: 298 | For level based compaction, we can configure if we want to 299 | skip/force bottommost level compaction. By default level based 300 | compaction will only compact the bottommost level if there is a 301 | compaction filter. It can be set to the following values. 302 | 303 | ``skip`` 304 | Skip bottommost level compaction 305 | 306 | ``if_compaction_filter`` 307 | Only compact bottommost level if there is a compaction filter. 308 | This is the default. 309 | 310 | ``force`` 311 | Always compact bottommost level 312 | 313 | .. py:attribute:: options 314 | 315 | Returns the associated :py:class:`rocksdb.Options` instance. 316 | 317 | .. note:: 318 | 319 | Changes to this object have no effect anymore. 320 | Consider this as read-only 321 | 322 | Iterator 323 | ======== 324 | 325 | .. py:class:: rocksdb.BaseIterator 326 | 327 | Base class for all iterators in this module. After creation a iterator is 328 | invalid. Call one of the seek methods first before starting iteration 329 | 330 | .. py:method:: seek_to_first() 331 | 332 | Position at the first key in the source 333 | 334 | .. py:method:: seek_to_last() 335 | 336 | Position at the last key in the source 337 | 338 | .. py:method:: seek(key) 339 | 340 | :param bytes key: Position at the first key in the source that at or past 341 | 342 | Methods to support the python iterator protocol 343 | 344 | .. py:method:: __iter__() 345 | .. py:method:: __next__() 346 | .. py:method:: __reversed__() 347 | 348 | Snapshot 349 | ======== 350 | 351 | .. py:class:: rocksdb.Snapshot 352 | 353 | Opaque handler for a single Snapshot. 354 | Snapshot is released if nobody holds a reference on it. 355 | Retrieved via :py:meth:`rocksdb.DB.snapshot` 356 | 357 | WriteBatch 358 | ========== 359 | 360 | .. py:class:: rocksdb.WriteBatch 361 | 362 | WriteBatch holds a collection of updates to apply atomically to a DB. 363 | 364 | The updates are applied in the order in which they are added 365 | to the WriteBatch. For example, the value of "key" will be "v3" 366 | after the following batch is written:: 367 | 368 | batch = rocksdb.WriteBatch() 369 | batch.put(b"key", b"v1") 370 | batch.delete(b"key") 371 | batch.put(b"key", b"v2") 372 | batch.put(b"key", b"v3") 373 | 374 | .. py:method:: __init__(data=None) 375 | 376 | Creates a WriteBatch. 377 | 378 | :param bytes data: 379 | A serialized version of a previous WriteBatch. As retrieved 380 | from a previous .data() call. If ``None`` a empty WriteBatch is 381 | generated 382 | 383 | .. py:method:: put(key, value) 384 | 385 | Store the mapping "key->value" in the database. 386 | 387 | :param bytes key: Name of the entry to store 388 | :param bytes value: Data of this entry 389 | 390 | .. py:method:: merge(key, value) 391 | 392 | Merge "value" with the existing value of "key" in the database. 393 | 394 | :param bytes key: Name of the entry to merge 395 | :param bytes value: Data to merge 396 | 397 | .. py:method:: delete(key) 398 | 399 | If the database contains a mapping for "key", erase it. Else do nothing. 400 | 401 | :param bytes key: Key to erase 402 | 403 | .. py:method:: clear() 404 | 405 | Clear all updates buffered in this batch. 406 | 407 | .. note:: 408 | Don't call this method if there is an outstanding iterator. 409 | Calling :py:meth:`rocksdb.WriteBatch.clear()` with outstanding 410 | iterator, leads to SEGFAULT. 411 | 412 | .. py:method:: data() 413 | 414 | Retrieve the serialized version of this batch. 415 | 416 | :rtype: ``bytes`` 417 | 418 | .. py:method:: count() 419 | 420 | Returns the number of updates in the batch 421 | 422 | :rtype: int 423 | 424 | .. py:method:: __iter__() 425 | 426 | Returns an iterator over the current contents of the write batch. 427 | 428 | If you add new items to the batch, they are not visible for this 429 | iterator. Create a new one if you need to see them. 430 | 431 | .. note:: 432 | Calling :py:meth:`rocksdb.WriteBatch.clear()` on the write batch 433 | invalidates the iterator. Using a iterator where its corresponding 434 | write batch has been cleared, leads to SEGFAULT. 435 | 436 | :rtype: :py:class:`rocksdb.WriteBatchIterator` 437 | 438 | WriteBatchIterator 439 | ================== 440 | 441 | .. py:class:: rocksdb.WriteBatchIterator 442 | 443 | .. py:method:: __iter__() 444 | 445 | Returns self. 446 | 447 | .. py:method:: __next__() 448 | 449 | Returns the next item inside the corresponding write batch. 450 | The return value is a tuple of always size three. 451 | 452 | First item (Name of the operation): 453 | 454 | * ``"Put"`` 455 | * ``"Merge"`` 456 | * ``"Delete"`` 457 | 458 | Second item (key): 459 | Key for this operation. 460 | 461 | Third item (value): 462 | The value for this operation. Empty for ``"Delete"``. 463 | 464 | Repair DB 465 | ========= 466 | 467 | .. py:function:: repair_db(db_name, opts) 468 | 469 | :param unicode db_name: Name of the database to open 470 | :param opts: Options for this specific database 471 | :type opts: :py:class:`rocksdb.Options` 472 | 473 | If a DB cannot be opened, you may attempt to call this method to 474 | resurrect as much of the contents of the database as possible. 475 | Some data may be lost, so be careful when calling this function 476 | on a database that contains important information. 477 | 478 | 479 | Errors 480 | ====== 481 | 482 | .. py:exception:: rocksdb.errors.NotFound 483 | .. py:exception:: rocksdb.errors.Corruption 484 | .. py:exception:: rocksdb.errors.NotSupported 485 | .. py:exception:: rocksdb.errors.InvalidArgument 486 | .. py:exception:: rocksdb.errors.RocksIOError 487 | .. py:exception:: rocksdb.errors.MergeInProgress 488 | .. py:exception:: rocksdb.errors.Incomplete 489 | 490 | 491 | -------------------------------------------------------------------------------- /rocksdb/tests/test_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import shutil 4 | import gc 5 | import unittest 6 | import rocksdb 7 | from itertools import takewhile 8 | import struct 9 | import tempfile 10 | from rocksdb.merge_operators import UintAddOperator, StringAppendOperator 11 | 12 | def int_to_bytes(ob): 13 | return str(ob).encode('ascii') 14 | 15 | class TestHelper(unittest.TestCase): 16 | 17 | def setUp(self): 18 | self.db_loc = tempfile.mkdtemp() 19 | self.addCleanup(self._close_db) 20 | 21 | def _close_db(self): 22 | if hasattr(self, 'db'): 23 | del self.db 24 | gc.collect() 25 | if os.path.exists(self.db_loc): 26 | shutil.rmtree(self.db_loc) 27 | 28 | 29 | class TestDB(TestHelper): 30 | def setUp(self): 31 | TestHelper.setUp(self) 32 | opts = rocksdb.Options(create_if_missing=True) 33 | self.db = rocksdb.DB(os.path.join(self.db_loc, "test"), opts) 34 | 35 | def test_options_used_twice(self): 36 | if sys.version_info[0] == 3: 37 | assertRaisesRegex = self.assertRaisesRegex 38 | else: 39 | assertRaisesRegex = self.assertRaisesRegexp 40 | expected = "Options object is already used by another DB" 41 | with assertRaisesRegex(Exception, expected): 42 | rocksdb.DB(os.path.join(self.db_loc, "test2"), self.db.options) 43 | 44 | def test_unicode_path(self): 45 | name = os.path.join(self.db_loc, b'M\xc3\xbcnchen'.decode('utf8')) 46 | rocksdb.DB(name, rocksdb.Options(create_if_missing=True)) 47 | self.addCleanup(shutil.rmtree, name) 48 | self.assertTrue(os.path.isdir(name)) 49 | 50 | def test_get_none(self): 51 | self.assertIsNone(self.db.get(b'xxx')) 52 | 53 | def test_put_get(self): 54 | self.db.put(b"a", b"b") 55 | self.assertEqual(b"b", self.db.get(b"a")) 56 | 57 | def test_multi_get(self): 58 | self.db.put(b"a", b"1") 59 | self.db.put(b"b", b"2") 60 | self.db.put(b"c", b"3") 61 | 62 | ret = self.db.multi_get([b'a', b'b', b'c']) 63 | ref = {b'a': b'1', b'c': b'3', b'b': b'2'} 64 | self.assertEqual(ref, ret) 65 | 66 | def test_delete(self): 67 | self.db.put(b"a", b"b") 68 | self.assertEqual(b"b", self.db.get(b"a")) 69 | self.db.delete(b"a") 70 | self.assertIsNone(self.db.get(b"a")) 71 | 72 | def test_write_batch(self): 73 | batch = rocksdb.WriteBatch() 74 | batch.put(b"key", b"v1") 75 | batch.delete(b"key") 76 | batch.put(b"key", b"v2") 77 | batch.put(b"key", b"v3") 78 | batch.put(b"a", b"b") 79 | 80 | self.db.write(batch) 81 | ref = {b'a': b'b', b'key': b'v3'} 82 | ret = self.db.multi_get([b'key', b'a']) 83 | self.assertEqual(ref, ret) 84 | 85 | def test_write_batch_iter(self): 86 | batch = rocksdb.WriteBatch() 87 | self.assertEqual([], list(batch)) 88 | 89 | batch.put(b"key1", b"v1") 90 | batch.put(b"key2", b"v2") 91 | batch.put(b"key3", b"v3") 92 | batch.delete(b'a') 93 | batch.delete(b'key1') 94 | batch.merge(b'xxx', b'value') 95 | 96 | it = iter(batch) 97 | del batch 98 | ref = [ 99 | ('Put', b'key1', b'v1'), 100 | ('Put', b'key2', b'v2'), 101 | ('Put', b'key3', b'v3'), 102 | ('Delete', b'a', b''), 103 | ('Delete', b'key1', b''), 104 | ('Merge', b'xxx', b'value') 105 | ] 106 | self.assertEqual(ref, list(it)) 107 | 108 | 109 | def test_key_may_exists(self): 110 | self.db.put(b"a", b'1') 111 | 112 | self.assertEqual((False, None), self.db.key_may_exist(b"x")) 113 | self.assertEqual((False, None), self.db.key_may_exist(b'x', True)) 114 | self.assertEqual((True, None), self.db.key_may_exist(b'a')) 115 | self.assertEqual((True, b'1'), self.db.key_may_exist(b'a', True)) 116 | 117 | def test_seek_for_prev(self): 118 | self.db.put(b'a1', b'a1_value') 119 | self.db.put(b'a3', b'a3_value') 120 | self.db.put(b'b1', b'b1_value') 121 | self.db.put(b'b2', b'b2_value') 122 | self.db.put(b'c2', b'c2_value') 123 | self.db.put(b'c4', b'c4_value') 124 | 125 | self.assertEqual(self.db.get(b'a1'), b'a1_value') 126 | 127 | it = self.db.iterkeys() 128 | 129 | it.seek(b'a1') 130 | self.assertEqual(it.get(), b'a1') 131 | it.seek(b'a3') 132 | self.assertEqual(it.get(), b'a3') 133 | it.seek_for_prev(b'c4') 134 | self.assertEqual(it.get(), b'c4') 135 | it.seek_for_prev(b'c3') 136 | self.assertEqual(it.get(), b'c2') 137 | 138 | it = self.db.itervalues() 139 | it.seek(b'a1') 140 | self.assertEqual(it.get(), b'a1_value') 141 | it.seek(b'a3') 142 | self.assertEqual(it.get(), b'a3_value') 143 | it.seek_for_prev(b'c4') 144 | self.assertEqual(it.get(), b'c4_value') 145 | it.seek_for_prev(b'c3') 146 | self.assertEqual(it.get(), b'c2_value') 147 | 148 | it = self.db.iteritems() 149 | it.seek(b'a1') 150 | self.assertEqual(it.get(), (b'a1', b'a1_value')) 151 | it.seek(b'a3') 152 | self.assertEqual(it.get(), (b'a3', b'a3_value')) 153 | it.seek_for_prev(b'c4') 154 | self.assertEqual(it.get(), (b'c4', b'c4_value')) 155 | it.seek_for_prev(b'c3') 156 | self.assertEqual(it.get(), (b'c2', b'c2_value')) 157 | 158 | reverse_it = reversed(it) 159 | it.seek_for_prev(b'c3') 160 | self.assertEqual(it.get(), (b'c2', b'c2_value')) 161 | 162 | 163 | def test_iter_keys(self): 164 | for x in range(300): 165 | self.db.put(int_to_bytes(x), int_to_bytes(x)) 166 | 167 | it = self.db.iterkeys() 168 | 169 | self.assertEqual([], list(it)) 170 | 171 | it.seek_to_last() 172 | self.assertEqual([b'99'], list(it)) 173 | 174 | ref = sorted([int_to_bytes(x) for x in range(300)]) 175 | it.seek_to_first() 176 | self.assertEqual(ref, list(it)) 177 | 178 | it.seek(b'90') 179 | ref = [ 180 | b'90', 181 | b'91', 182 | b'92', 183 | b'93', 184 | b'94', 185 | b'95', 186 | b'96', 187 | b'97', 188 | b'98', 189 | b'99' 190 | ] 191 | self.assertEqual(ref, list(it)) 192 | 193 | def test_iter_values(self): 194 | for x in range(300): 195 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 196 | 197 | it = self.db.itervalues() 198 | 199 | self.assertEqual([], list(it)) 200 | 201 | it.seek_to_last() 202 | self.assertEqual([b'99000'], list(it)) 203 | 204 | ref = sorted([int_to_bytes(x) for x in range(300)]) 205 | ref = [int_to_bytes(int(x) * 1000) for x in ref] 206 | it.seek_to_first() 207 | self.assertEqual(ref, list(it)) 208 | 209 | it.seek(b'90') 210 | ref = [int_to_bytes(x * 1000) for x in range(90, 100)] 211 | self.assertEqual(ref, list(it)) 212 | 213 | def test_iter_items(self): 214 | for x in range(300): 215 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 216 | 217 | it = self.db.iteritems() 218 | 219 | self.assertEqual([], list(it)) 220 | 221 | it.seek_to_last() 222 | self.assertEqual([(b'99', b'99000')], list(it)) 223 | 224 | ref = sorted([int_to_bytes(x) for x in range(300)]) 225 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 226 | it.seek_to_first() 227 | self.assertEqual(ref, list(it)) 228 | 229 | it.seek(b'90') 230 | ref = [(int_to_bytes(x), int_to_bytes(x * 1000)) for x in range(90, 100)] 231 | self.assertEqual(ref, list(it)) 232 | 233 | def test_reverse_iter(self): 234 | for x in range(100): 235 | self.db.put(int_to_bytes(x), int_to_bytes(x * 1000)) 236 | 237 | it = self.db.iteritems() 238 | it.seek_to_last() 239 | 240 | ref = reversed(sorted([int_to_bytes(x) for x in range(100)])) 241 | ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref] 242 | 243 | self.assertEqual(ref, list(reversed(it))) 244 | 245 | def test_snapshot(self): 246 | self.db.put(b"a", b"1") 247 | self.db.put(b"b", b"2") 248 | 249 | snapshot = self.db.snapshot() 250 | self.db.put(b"a", b"2") 251 | self.db.delete(b"b") 252 | 253 | it = self.db.iteritems() 254 | it.seek_to_first() 255 | self.assertEqual({b'a': b'2'}, dict(it)) 256 | 257 | it = self.db.iteritems(snapshot=snapshot) 258 | it.seek_to_first() 259 | self.assertEqual({b'a': b'1', b'b': b'2'}, dict(it)) 260 | 261 | def test_get_property(self): 262 | for x in range(300): 263 | x = int_to_bytes(x) 264 | self.db.put(x, x) 265 | 266 | self.assertIsNotNone(self.db.get_property(b'rocksdb.stats')) 267 | self.assertIsNotNone(self.db.get_property(b'rocksdb.sstables')) 268 | self.assertIsNotNone(self.db.get_property(b'rocksdb.num-files-at-level0')) 269 | self.assertIsNone(self.db.get_property(b'does not exsits')) 270 | 271 | def test_compact_range(self): 272 | for x in range(10000): 273 | x = int_to_bytes(x) 274 | self.db.put(x, x) 275 | 276 | self.db.compact_range() 277 | 278 | def test_write_ignore_missing_column_families(self): 279 | self.db.put(b"a", b"1", ignore_missing_column_families=True) 280 | 281 | def test_write_no_slowdown(self): 282 | self.db.put(b"a", b"1", no_slowdown=True) 283 | 284 | def test_write_low_pri(self): 285 | self.db.put(b"a", b"1", low_pri=True) 286 | 287 | class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator): 288 | def merge(self, key, existing_value, value): 289 | if existing_value: 290 | return (True, int_to_bytes(int(existing_value) + int(value))) 291 | return (True, value) 292 | 293 | def name(self): 294 | return b'AssocCounter' 295 | 296 | 297 | class TestUint64Merge(TestHelper): 298 | def setUp(self): 299 | TestHelper.setUp(self) 300 | opts = rocksdb.Options() 301 | opts.create_if_missing = True 302 | opts.merge_operator = UintAddOperator() 303 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 304 | 305 | def test_merge(self): 306 | self.db.put(b'a', struct.pack('Q', 5566)) 307 | for x in range(1000): 308 | self.db.merge(b"a", struct.pack('Q', x)) 309 | self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) 310 | 311 | 312 | # class TestPutMerge(TestHelper): 313 | # def setUp(self): 314 | # TestHelper.setUp(self) 315 | # opts = rocksdb.Options() 316 | # opts.create_if_missing = True 317 | # opts.merge_operator = "put" 318 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 319 | 320 | # def test_merge(self): 321 | # self.db.put(b'a', b'ccc') 322 | # self.db.merge(b'a', b'ddd') 323 | # self.assertEqual(self.db.get(b'a'), 'ddd') 324 | 325 | # class TestPutV1Merge(TestHelper): 326 | # def setUp(self): 327 | # TestHelper.setUp(self) 328 | # opts = rocksdb.Options() 329 | # opts.create_if_missing = True 330 | # opts.merge_operator = "put_v1" 331 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 332 | 333 | # def test_merge(self): 334 | # self.db.put(b'a', b'ccc') 335 | # self.db.merge(b'a', b'ddd') 336 | # self.assertEqual(self.db.get(b'a'), 'ddd') 337 | 338 | class TestStringAppendOperatorMerge(TestHelper): 339 | def setUp(self): 340 | TestHelper.setUp(self) 341 | opts = rocksdb.Options() 342 | opts.create_if_missing = True 343 | opts.merge_operator = StringAppendOperator() 344 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 345 | 346 | # NOTE(sileht): Raise "Corruption: Error: Could not perform merge." on PY3 347 | #@unittest.skipIf(sys.version_info[0] == 3, 348 | # "Unexpected behavior on PY3") 349 | def test_merge(self): 350 | self.db.put(b'a', b'ccc') 351 | self.db.merge(b'a', b'ddd') 352 | self.assertEqual(self.db.get(b'a'), b'ccc,ddd') 353 | 354 | # class TestStringMaxOperatorMerge(TestHelper): 355 | # def setUp(self): 356 | # TestHelper.setUp(self) 357 | # opts = rocksdb.Options() 358 | # opts.create_if_missing = True 359 | # opts.merge_operator = "max" 360 | # self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 361 | 362 | # def test_merge(self): 363 | # self.db.put(b'a', int_to_bytes(55)) 364 | # self.db.merge(b'a', int_to_bytes(56)) 365 | # self.assertEqual(int(self.db.get(b'a')), 56) 366 | 367 | 368 | class TestAssocMerge(TestHelper): 369 | def setUp(self): 370 | TestHelper.setUp(self) 371 | opts = rocksdb.Options() 372 | opts.create_if_missing = True 373 | opts.merge_operator = AssocCounter() 374 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 375 | 376 | def test_merge(self): 377 | for x in range(1000): 378 | self.db.merge(b"a", int_to_bytes(x)) 379 | self.assertEqual(sum(range(1000)), int(self.db.get(b'a'))) 380 | 381 | 382 | class FullCounter(rocksdb.interfaces.MergeOperator): 383 | def name(self): 384 | return b'fullcounter' 385 | 386 | def full_merge(self, key, existing_value, operand_list): 387 | ret = sum([int(x) for x in operand_list]) 388 | if existing_value: 389 | ret += int(existing_value) 390 | 391 | return (True, int_to_bytes(ret)) 392 | 393 | def partial_merge(self, key, left, right): 394 | return (True, int_to_bytes(int(left) + int(right))) 395 | 396 | 397 | class TestFullMerge(TestHelper): 398 | def setUp(self): 399 | TestHelper.setUp(self) 400 | opts = rocksdb.Options() 401 | opts.create_if_missing = True 402 | opts.merge_operator = FullCounter() 403 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 404 | 405 | def test_merge(self): 406 | for x in range(1000): 407 | self.db.merge(b"a", int_to_bytes(x)) 408 | self.assertEqual(sum(range(1000)), int(self.db.get(b'a'))) 409 | 410 | 411 | class SimpleComparator(rocksdb.interfaces.Comparator): 412 | def name(self): 413 | return b'mycompare' 414 | 415 | def compare(self, a, b): 416 | a = int(a) 417 | b = int(b) 418 | if a < b: 419 | return -1 420 | if a == b: 421 | return 0 422 | if a > b: 423 | return 1 424 | 425 | 426 | class TestComparator(TestHelper): 427 | def setUp(self): 428 | TestHelper.setUp(self) 429 | opts = rocksdb.Options() 430 | opts.create_if_missing = True 431 | opts.comparator = SimpleComparator() 432 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 433 | 434 | def test_compare(self): 435 | for x in range(1000): 436 | self.db.put(int_to_bytes(x), int_to_bytes(x)) 437 | 438 | self.assertEqual(b'300', self.db.get(b'300')) 439 | 440 | class StaticPrefix(rocksdb.interfaces.SliceTransform): 441 | def name(self): 442 | return b'static' 443 | 444 | def transform(self, src): 445 | return (0, 5) 446 | 447 | def in_domain(self, src): 448 | return len(src) >= 5 449 | 450 | def in_range(self, dst): 451 | return len(dst) == 5 452 | 453 | class TestPrefixExtractor(TestHelper): 454 | def setUp(self): 455 | TestHelper.setUp(self) 456 | opts = rocksdb.Options(create_if_missing=True) 457 | opts.prefix_extractor = StaticPrefix() 458 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 459 | 460 | def _fill_db(self): 461 | for x in range(3000): 462 | keyx = hex(x)[2:].zfill(5).encode('utf8') + b'.x' 463 | keyy = hex(x)[2:].zfill(5).encode('utf8') + b'.y' 464 | keyz = hex(x)[2:].zfill(5).encode('utf8') + b'.z' 465 | self.db.put(keyx, b'x') 466 | self.db.put(keyy, b'y') 467 | self.db.put(keyz, b'z') 468 | 469 | 470 | def test_prefix_iterkeys(self): 471 | self._fill_db() 472 | self.assertEqual(b'x', self.db.get(b'00001.x')) 473 | self.assertEqual(b'y', self.db.get(b'00001.y')) 474 | self.assertEqual(b'z', self.db.get(b'00001.z')) 475 | 476 | it = self.db.iterkeys() 477 | it.seek(b'00002') 478 | 479 | ref = [b'00002.x', b'00002.y', b'00002.z'] 480 | ret = takewhile(lambda key: key.startswith(b'00002'), it) 481 | self.assertEqual(ref, list(ret)) 482 | 483 | def test_prefix_iteritems(self): 484 | self._fill_db() 485 | 486 | it = self.db.iteritems() 487 | it.seek(b'00002') 488 | 489 | ref = {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'} 490 | ret = takewhile(lambda item: item[0].startswith(b'00002'), it) 491 | self.assertEqual(ref, dict(ret)) 492 | 493 | class TestFixedPrefixExtractor(TestHelper): 494 | def setUp(self): 495 | TestHelper.setUp(self) 496 | opts = rocksdb.Options(create_if_missing=True, prefix_extractor=4) 497 | self.db = rocksdb.DB(os.path.join(self.db_loc, 'test'), opts) 498 | 499 | def _fill_db(self): 500 | for x in range(3000): 501 | keyx = b'%05x.%b' % (x, b'x') 502 | keyy = b'%05x.%b' % (x, b'y') 503 | keyz = b'%05x.%b' % (x, b'z') 504 | self.db.put(keyx, b'x') 505 | self.db.put(keyy, b'y') 506 | self.db.put(keyz, b'z') 507 | 508 | def test_prefix_iterkeys(self): 509 | self._fill_db() 510 | self.assertEqual(b'x', self.db.get(b'00001.x')) 511 | self.assertEqual(b'y', self.db.get(b'00001.y')) 512 | self.assertEqual(b'z', self.db.get(b'00001.z')) 513 | 514 | it = self.db.iterkeys() 515 | it.seek(b'00002') 516 | 517 | ref = [b'00002.x', b'00002.y', b'00002.z'] 518 | ret = takewhile(lambda key: key.startswith(b'00002'), it) 519 | self.assertEqual(ref, list(ret)) 520 | 521 | def test_prefix_iteritems(self): 522 | self._fill_db() 523 | 524 | it = self.db.iteritems() 525 | it.seek(b'00002') 526 | 527 | ref = {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'} 528 | ret = takewhile(lambda item: item[0].startswith(b'00002'), it) 529 | self.assertEqual(ref, dict(ret)) 530 | 531 | class TestDBColumnFamilies(TestHelper): 532 | def setUp(self): 533 | TestHelper.setUp(self) 534 | opts = rocksdb.Options(create_if_missing=True) 535 | self.db = rocksdb.DB( 536 | os.path.join(self.db_loc, 'test'), 537 | opts, 538 | ) 539 | 540 | self.cf_a = self.db.create_column_family(b'A', rocksdb.ColumnFamilyOptions()) 541 | self.cf_b = self.db.create_column_family(b'B', rocksdb.ColumnFamilyOptions()) 542 | 543 | def test_column_families(self): 544 | families = self.db.column_families 545 | names = [handle.name for handle in families] 546 | self.assertEqual([b'default', b'A', b'B'], names) 547 | for name in names: 548 | self.assertIn(self.db.get_column_family(name), families) 549 | 550 | self.assertEqual( 551 | names, 552 | rocksdb.list_column_families( 553 | os.path.join(self.db_loc, 'test'), 554 | rocksdb.Options(), 555 | ) 556 | ) 557 | 558 | def test_get_none(self): 559 | self.assertIsNone(self.db.get(b'k')) 560 | self.assertIsNone(self.db.get((self.cf_a, b'k'))) 561 | self.assertIsNone(self.db.get((self.cf_b, b'k'))) 562 | 563 | def test_put_get(self): 564 | key = (self.cf_a, b'k') 565 | self.db.put(key, b"v") 566 | self.assertEqual(b"v", self.db.get(key)) 567 | self.assertIsNone(self.db.get(b"k")) 568 | self.assertIsNone(self.db.get((self.cf_b, b"k"))) 569 | 570 | def test_multi_get(self): 571 | data = [ 572 | (b'a', b'1default'), 573 | (b'b', b'2default'), 574 | (b'c', b'3default'), 575 | ((self.cf_a, b'a'), b'1a'), 576 | ((self.cf_a, b'b'), b'2a'), 577 | ((self.cf_a, b'c'), b'3a'), 578 | ((self.cf_b, b'a'), b'1b'), 579 | ((self.cf_b, b'b'), b'2b'), 580 | ((self.cf_b, b'c'), b'3b'), 581 | ] 582 | for value in data: 583 | self.db.put(*value) 584 | 585 | multi_get_lookup = [value[0] for value in data] 586 | 587 | ret = self.db.multi_get(multi_get_lookup) 588 | ref = {value[0]: value[1] for value in data} 589 | self.assertEqual(ref, ret) 590 | 591 | def test_delete(self): 592 | self.db.put((self.cf_a, b"a"), b"b") 593 | self.assertEqual(b"b", self.db.get((self.cf_a, b"a"))) 594 | self.db.delete((self.cf_a, b"a")) 595 | self.assertIsNone(self.db.get((self.cf_a, b"a"))) 596 | 597 | def test_write_batch(self): 598 | cfa = self.db.get_column_family(b"A") 599 | batch = rocksdb.WriteBatch() 600 | batch.put((cfa, b"key"), b"v1") 601 | batch.delete((self.cf_a, b"key")) 602 | batch.put((cfa, b"key"), b"v2") 603 | batch.put((cfa, b"key"), b"v3") 604 | batch.put((cfa, b"a"), b"1") 605 | batch.put((cfa, b"b"), b"2") 606 | 607 | self.db.write(batch) 608 | query = [(cfa, b"key"), (cfa, b"a"), (cfa, b"b")] 609 | ret = self.db.multi_get(query) 610 | 611 | self.assertEqual(b"v3", ret[query[0]]) 612 | self.assertEqual(b"1", ret[query[1]]) 613 | self.assertEqual(b"2", ret[query[2]]) 614 | 615 | def test_key_may_exists(self): 616 | self.db.put((self.cf_a, b"a"), b'1') 617 | 618 | self.assertEqual( 619 | (False, None), 620 | self.db.key_may_exist((self.cf_a, b"x")) 621 | ) 622 | self.assertEqual( 623 | (False, None), 624 | self.db.key_may_exist((self.cf_a, b'x'), fetch=True) 625 | ) 626 | self.assertEqual( 627 | (True, None), 628 | self.db.key_may_exist((self.cf_a, b'a')) 629 | ) 630 | self.assertEqual( 631 | (True, b'1'), 632 | self.db.key_may_exist((self.cf_a, b'a'), fetch=True) 633 | ) 634 | 635 | def test_iter_keys(self): 636 | for x in range(300): 637 | self.db.put((self.cf_a, int_to_bytes(x)), int_to_bytes(x)) 638 | 639 | it = self.db.iterkeys(self.cf_a) 640 | self.assertEqual([], list(it)) 641 | 642 | it.seek_to_last() 643 | self.assertEqual([(self.cf_a, b'99')], list(it)) 644 | 645 | ref = sorted([(self.cf_a, int_to_bytes(x)) for x in range(300)]) 646 | it.seek_to_first() 647 | self.assertEqual(ref, list(it)) 648 | 649 | it.seek(b'90') 650 | ref = sorted([(self.cf_a, int_to_bytes(x)) for x in range(90, 100)]) 651 | self.assertEqual(ref, list(it)) 652 | 653 | def test_iter_values(self): 654 | for x in range(300): 655 | self.db.put((self.cf_b, int_to_bytes(x)), int_to_bytes(x * 1000)) 656 | 657 | it = self.db.itervalues(self.cf_b) 658 | self.assertEqual([], list(it)) 659 | 660 | it.seek_to_last() 661 | self.assertEqual([b'99000'], list(it)) 662 | 663 | ref = sorted([int_to_bytes(x) for x in range(300)]) 664 | ref = [int_to_bytes(int(x) * 1000) for x in ref] 665 | it.seek_to_first() 666 | self.assertEqual(ref, list(it)) 667 | 668 | it.seek(b'90') 669 | ref = [int_to_bytes(x * 1000) for x in range(90, 100)] 670 | self.assertEqual(ref, list(it)) 671 | 672 | def test_iter_items(self): 673 | for x in range(300): 674 | self.db.put((self.cf_b, int_to_bytes(x)), int_to_bytes(x * 1000)) 675 | 676 | it = self.db.iteritems(self.cf_b) 677 | self.assertEqual([], list(it)) 678 | 679 | it.seek_to_last() 680 | self.assertEqual([((self.cf_b, b'99'), b'99000')], list(it)) 681 | 682 | ref = sorted([int_to_bytes(x) for x in range(300)]) 683 | ref = [((self.cf_b, x), int_to_bytes(int(x) * 1000)) for x in ref] 684 | it.seek_to_first() 685 | self.assertEqual(ref, list(it)) 686 | 687 | it.seek(b'90') 688 | ref = [((self.cf_b, int_to_bytes(x)), int_to_bytes(x * 1000)) for x in range(90, 100)] 689 | self.assertEqual(ref, list(it)) 690 | 691 | def test_reverse_iter(self): 692 | for x in range(100): 693 | self.db.put((self.cf_a, int_to_bytes(x)), int_to_bytes(x * 1000)) 694 | 695 | it = self.db.iteritems(self.cf_a) 696 | it.seek_to_last() 697 | 698 | ref = reversed(sorted([(self.cf_a, int_to_bytes(x)) for x in range(100)])) 699 | ref = [(x, int_to_bytes(int(x[1]) * 1000)) for x in ref] 700 | 701 | self.assertEqual(ref, list(reversed(it))) 702 | 703 | def test_snapshot(self): 704 | cfa = self.db.get_column_family(b'A') 705 | self.db.put((cfa, b"a"), b"1") 706 | self.db.put((cfa, b"b"), b"2") 707 | 708 | snapshot = self.db.snapshot() 709 | self.db.put((cfa, b"a"), b"2") 710 | self.db.delete((cfa, b"b")) 711 | 712 | it = self.db.iteritems(cfa) 713 | it.seek_to_first() 714 | self.assertEqual({(cfa, b'a'): b'2'}, dict(it)) 715 | 716 | it = self.db.iteritems(cfa, snapshot=snapshot) 717 | it.seek_to_first() 718 | self.assertEqual({(cfa, b'a'): b'1', (cfa, b'b'): b'2'}, dict(it)) 719 | 720 | def test_get_property(self): 721 | for x in range(300): 722 | x = int_to_bytes(x) 723 | self.db.put((self.cf_a, x), x) 724 | 725 | self.assertEqual(b"300", 726 | self.db.get_property(b'rocksdb.estimate-num-keys', 727 | self.cf_a)) 728 | self.assertIsNone(self.db.get_property(b'does not exsits', 729 | self.cf_a)) 730 | 731 | def test_compact_range(self): 732 | for x in range(10000): 733 | x = int_to_bytes(x) 734 | self.db.put((self.cf_b, x), x) 735 | 736 | self.db.compact_range(column_family=self.cf_b) 737 | 738 | --------------------------------------------------------------------------------