├── .gitignore ├── Makefile ├── README.md ├── build_detect_platform ├── examples ├── Makefile ├── file_reader_writer_example.cc ├── log_example.cc ├── lru_cache_example.cc ├── mutexlock_example.cc ├── thread_local_example.cc ├── thread_pool_example.cc └── wal_example.cc ├── include └── rocksutil │ ├── auto_roll_logger.h │ ├── cache.h │ ├── coding.h │ ├── crc32c.h │ ├── env.h │ ├── file_reader_writer.h │ ├── hash.h │ ├── log_reader.h │ ├── log_writer.h │ ├── mutexlock.h │ ├── slice.h │ ├── status.h │ ├── thread_local.h │ └── version.h ├── rport ├── likely.h ├── port.h ├── port_posix.cc ├── port_posix.h ├── sys_time.h └── util_logger.h ├── rutil ├── aligned_buffer.h ├── auto_roll_logger.cc ├── autovector.h ├── build_version.cc.in ├── build_version.h ├── coding.cc ├── crc32c.cc ├── env.cc ├── env_posix.cc ├── file_reader_writer.cc ├── hash.cc ├── io_posix.cc ├── io_posix.h ├── log_format.h ├── log_reader.cc ├── log_writer.cc ├── lru_cache.cc ├── lru_cache.h ├── posix_logger.h ├── random.cc ├── random.h ├── sharded_cache.cc ├── sharded_cache.h ├── status.cc ├── status_message.cc ├── string_util.cc ├── string_util.h ├── thread_local.cc ├── threadpool.h ├── threadpool_imp.cc └── threadpool_imp.h └── src.mk /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | 31 | tags 32 | make_config.mk 33 | rutil/build_version.cc 34 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CLEAN_FILES = # deliberately empty, so we can append below. 2 | CFLAGS += ${EXTRA_CFLAGS} 3 | CXXFLAGS += ${EXTRA_CXXFLAGS} 4 | LDFLAGS += $(EXTRA_LDFLAGS) 5 | ARFLAGS = rs 6 | OPT= 7 | 8 | # Set the default DEBUG_LEVEL to 1 9 | DEBUG_LEVEL?=0 10 | 11 | ifeq ($(MAKECMDGOALS),dbg) 12 | DEBUG_LEVEL=2 13 | endif 14 | 15 | ifeq ($(MAKECMDGOALS),all) 16 | DEBUG_LEVEL=0 17 | endif 18 | 19 | ifeq ($(MAKECMDGOALS),clean) 20 | DEBUG_LEVEL=0 21 | endif 22 | 23 | ifeq ($(MAKECMDGOALS),shared_lib) 24 | DEBUG_LEVEL=0 25 | endif 26 | 27 | ifeq ($(MAKECMDGOALS),static_lib) 28 | DEBUG_LEVEL=0 29 | endif 30 | 31 | # compile with -O2 if debug level is not 2 32 | ifneq ($(DEBUG_LEVEL), 2) 33 | OPT += -O2 -fno-omit-frame-pointer 34 | # Skip for archs that don't support -momit-leaf-frame-pointer 35 | ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1)) 36 | OPT += -momit-leaf-frame-pointer 37 | endif 38 | endif 39 | 40 | # if we're compiling for release, compile without debug code (-DNDEBUG) and 41 | # don't treat warnings as errors 42 | ifeq ($(DEBUG_LEVEL),0) 43 | OPT += -DNDEBUG 44 | DISABLE_WARNING_AS_ERROR=1 45 | else 46 | $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) 47 | endif 48 | 49 | #----------------------------------------------- 50 | 51 | include ./src.mk 52 | 53 | AM_DEFAULT_VERBOSITY = 0 54 | 55 | AM_V_GEN = $(am__v_GEN_$(V)) 56 | am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) 57 | am__v_GEN_0 = @echo " GEN " $@; 58 | am__v_GEN_1 = 59 | AM_V_at = $(am__v_at_$(V)) 60 | am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) 61 | am__v_at_0 = @ 62 | am__v_at_1 = 63 | 64 | AM_V_CC = $(am__v_CC_$(V)) 65 | am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) 66 | am__v_CC_0 = @echo " CC " $@; 67 | am__v_CC_1 = 68 | CCLD = $(CC) 69 | LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ 70 | AM_V_CCLD = $(am__v_CCLD_$(V)) 71 | am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY)) 72 | am__v_CCLD_0 = @echo " CCLD " $@; 73 | am__v_CCLD_1 = 74 | AM_V_AR = $(am__v_AR_$(V)) 75 | am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) 76 | am__v_AR_0 = @echo " AR " $@; 77 | am__v_AR_1 = 78 | 79 | AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) 80 | # detect what platform we're building on 81 | dummy := $(shell (export ROCKSUTIL_ROOT="$(CURDIR)"; "$(CURDIR)/build_detect_platform" "$(CURDIR)/make_config.mk")) 82 | # this file is generated by the previous line to set build flags and sources 83 | include make_config.mk 84 | CLEAN_FILES += make_config.mk 85 | 86 | missing_make_config_paths := $(shell \ 87 | grep "\/\S*" -o $(CURDIR)/make_config.mk | \ 88 | while read path; \ 89 | do [ -e $$path ] || echo $$path; \ 90 | done | sort | uniq) 91 | 92 | $(foreach path, $(missing_make_config_paths), \ 93 | $(warning Warning: $(path) dont exist)) 94 | 95 | ifneq ($(PLATFORM), IOS) 96 | CFLAGS += -g 97 | CXXFLAGS += -g 98 | else 99 | # no debug info for IOS, that will make our library big 100 | OPT += -DNDEBUG 101 | endif 102 | 103 | ifeq ($(PLATFORM), OS_SOLARIS) 104 | PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99 105 | endif 106 | 107 | # This (the first rule) must depend on "all". 108 | default: all 109 | 110 | WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \ 111 | -Wno-unused-parameter 112 | 113 | ifndef DISABLE_WARNING_AS_ERROR 114 | WARNING_FLAGS += -Werror 115 | endif 116 | 117 | CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) 118 | CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers 119 | 120 | LDFLAGS += $(PLATFORM_LDFLAGS) 121 | 122 | date := $(shell date +%F) 123 | git_sha := $(shell git rev-parse HEAD 2>/dev/null) 124 | gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ rutil/build_version.cc.in 125 | # Record the version of the source that we are compiling. 126 | # We keep a record of the git revision in this file. It is then built 127 | # as a regular source file as part of the compilation process. 128 | # One can run "strings executable_filename | grep _build_" to find 129 | # the version of the source that we used to build the executable file. 130 | CLEAN_FILES += rutil/build_version.cc 131 | 132 | rutil/build_version.cc: FORCE 133 | $(AM_V_GEN)rm -f $@-t 134 | $(AM_V_at)$(gen_build_version) > $@-t 135 | $(AM_V_at)if test -f $@; then \ 136 | cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \ 137 | else mv -f $@-t $@; fi 138 | FORCE: 139 | 140 | LIBOBJECTS = $(LIB_SOURCES:.cc=.o) 141 | 142 | # if user didn't config LIBNAME, set the default 143 | ifeq ($(LIBNAME),) 144 | # we should only run rocksutil in production with DEBUG_LEVEL 0 145 | ifeq ($(DEBUG_LEVEL),0) 146 | LIBNAME=librocksutil 147 | else 148 | LIBNAME=librocksutil_debug 149 | endif 150 | endif 151 | LIBRARY = ${LIBNAME}.a 152 | 153 | ROCKSUTIL_MAJOR = $(shell egrep "ROCKSUTIL_MAJOR.[0-9]" include/rocksutil/version.h | cut -d ' ' -f 3) 154 | ROCKSUTIL_MINOR = $(shell egrep "ROCKSUTIL_MINOR.[0-9]" include/rocksutil/version.h | cut -d ' ' -f 3) 155 | ROCKSUTIL_PATCH = $(shell egrep "ROCKSUTIL_PATCH.[0-9]" include/rocksutil/version.h | cut -d ' ' -f 3) 156 | 157 | #----------------------------------------------- 158 | # Create platform independent shared libraries. 159 | #----------------------------------------------- 160 | ifneq ($(PLATFORM_SHARED_EXT),) 161 | 162 | ifneq ($(PLATFORM_SHARED_VERSIONED),true) 163 | SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT) 164 | SHARED2 = $(SHARED1) 165 | SHARED3 = $(SHARED1) 166 | SHARED4 = $(SHARED1) 167 | SHARED = $(SHARED1) 168 | else 169 | SHARED_MAJOR = $(ROCKSUTIL_MAJOR) 170 | SHARED_MINOR = $(ROCKSUTIL_MINOR) 171 | SHARED_PATCH = $(ROCKSUTIL_PATCH) 172 | SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT) 173 | ifeq ($(PLATFORM), OS_MACOSX) 174 | SHARED_OSX = $(LIBNAME).$(SHARED_MAJOR) 175 | SHARED2 = $(SHARED_OSX).$(PLATFORM_SHARED_EXT) 176 | SHARED3 = $(SHARED_OSX).$(SHARED_MINOR).$(PLATFORM_SHARED_EXT) 177 | SHARED4 = $(SHARED_OSX).$(SHARED_MINOR).$(SHARED_PATCH).$(PLATFORM_SHARED_EXT) 178 | else 179 | SHARED2 = $(SHARED1).$(SHARED_MAJOR) 180 | SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) 181 | SHARED4 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR).$(SHARED_PATCH) 182 | endif 183 | SHARED = $(SHARED1) $(SHARED2) $(SHARED3) $(SHARED4) 184 | $(SHARED1): $(SHARED4) 185 | ln -fs $(SHARED4) $(SHARED1) 186 | $(SHARED2): $(SHARED4) 187 | ln -fs $(SHARED4) $(SHARED2) 188 | $(SHARED3): $(SHARED4) 189 | ln -fs $(SHARED4) $(SHARED3) 190 | endif 191 | 192 | $(SHARED4): $(LIBOBJECTS) 193 | $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) \ 194 | $(LDFLAGS) -o $@ 195 | 196 | endif # PLATFORM_SHARED_EXT 197 | 198 | .PHONY: clean tags dbg static_lib shared_lib all 199 | 200 | EXAMPLES = log_example thread_local_example mutexlock_example thread_pool_example lru_cache_example \ 201 | file_reader_writer_example wal_example 202 | 203 | .cc.o: 204 | $(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS) 205 | 206 | .c.o: 207 | $(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@ 208 | 209 | all: $(LIBRARY) 210 | 211 | static_lib: $(LIBRARY) 212 | 213 | shared_lib: $(SHARED) 214 | 215 | example: $(EXAMPLES) 216 | 217 | dbg: $(LIBRARY) $(EXAMPLES) 218 | 219 | $(LIBRARY): $(LIBOBJECTS) 220 | $(AM_V_AR)rm -f $@ 221 | $(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS) 222 | 223 | log_example: examples/log_example.o $(LIBOBJECTS) 224 | $(AM_LINK) 225 | 226 | thread_local_example: examples/thread_local_example.o $(LIBOBJECTS) 227 | $(AM_LINK) 228 | 229 | mutexlock_example: examples/mutexlock_example.o $(LIBOBJECTS) 230 | $(AM_LINK) 231 | 232 | thread_pool_example: examples/thread_pool_example.o $(LIBOBJECTS) 233 | $(AM_LINK) 234 | 235 | lru_cache_example: examples/lru_cache_example.o $(LIBOBJECTS) 236 | $(AM_LINK) 237 | 238 | file_reader_writer_example: examples/file_reader_writer_example.o $(LIBOBJECTS) 239 | $(AM_LINK) 240 | 241 | wal_example: examples/wal_example.o $(LIBOBJECTS) 242 | $(AM_LINK) 243 | 244 | clean: 245 | make -C ./examples clean 246 | rm -f $(EXAMPLES) $(LIBRARY) $(SHARED) 247 | rm -rf $(CLEAN_FILES) ios-x86 ios-arm 248 | find . -name "*.[oda]" -exec rm -f {} \; 249 | find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rocksutil 2 | rocksutil is a ToolKit for C++ developer，It has a lot of useful tools to make coding more faster and easier. It supports LINUX and MACOS platforms 3 | 4 | ## tools 5 | 6 | ### 1. Env 7 | 8 | Easy way to operate files, directories, threads, times, etc 9 | 10 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/thread_pool_example.cc) 11 | 12 | Find more usages [Here](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/env.h) 13 | 14 | |Files & Directories    | 15 | | --- | 16 | |Env::NewDirectory         | 17 | |Env::FileExists | 18 | |Env::GetChildren | 19 | |Env::GetChildrenFileAttributes| 20 | |Env::DeleteFile | 21 | |Env::CreateDir | 22 | |Env::CreateDirIfMissing | 23 | |Env::GetFileSize | 24 | |Env::GetFileModificationTime | 25 | |Env::RenameFile | 26 | |Env::LinkFile | 27 | |Env::LockFile | 28 | |Env::UnlockFile               | 29 | |Env::GetAbsolutePath          | 30 | 31 | --- 32 | |Thread & ThreadPool             | 33 | | --- | 34 | |Env::Schedule                   | 35 | |Env::UnSchedule                 | 36 | |Env::StartThread               | 37 | |Env::WaitForJoin               | 38 | |Env::GetThreadPoolQueueLen     | 39 | |Env::SetBackgroundThreads       | 40 | |Env::IncBackgroundThreadsIfNeeded| 41 | |Env::LowerThreadPoolIOPriority | 42 | |Env::GetThreadID               | 43 | 44 | --- 45 | |Time & System         | 46 | | --- | 47 | |Env::NowMicros         | 48 | |Env::NowNanos         | 49 | |Env::SleepForMicroseconds| 50 | |Env::GetHostName       | 51 | |Env::GetCurrentTime     | 52 | 53 | ### 2. Log 54 | 55 | Easy way to use Log 56 | 57 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/log_example.cc) 58 | 59 | ### 3. ThreadLocal 60 | 61 | Easy way to use thread-specific data 62 | 63 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/thread_pool_example.cc) 64 | 65 | ### 4. MutexLock 66 | 67 | Easy way to use Lock, RWLock, SpinLock 68 | 69 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/mutexlock_example.cc) 70 | 71 | ### 5. LRUCache 72 | Easy way to use lru cache 73 | 74 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/lru_cache_example.cc) 75 | 76 | ### 6. File Writer & Reader 77 | 78 | Easy way to read & write file in random or sequential mode, and use EnvOptions to manipulate 79 | the writer & reader to use [mmap, direct_io, buffer_read_write...] 80 | 81 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/file_reader_writer_example.cc) 82 | 83 | ### 7. Write Ahead Log 84 | 85 | Easy way to write or recover from a robust [format](https://github.com/KernelMaker/rocksutil/blob/master/util/log_format.h) wal, based on FileWriter & FileReader 86 | 87 | Here is an [EXAMPLE](https://github.com/KernelMaker/rocksutil/blob/master/examples/wal_example.cc) 88 | 89 | ### 8. Others 90 | Find more info in 91 | 92 | [Slice](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/slice.h) 93 | 94 | [Coding](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/coding.h) 95 | 96 | [Status](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/status.h) 97 | 98 | [Hash](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/hash.h ) 99 | 100 | [CRC32](https://github.com/KernelMaker/rocksutil/blob/master/include/rocksutil/crc32c.h ) 101 | 102 | ## Where is it from? 103 | This ToolKit is mainly extracted from [rocksdb](https://github.com/facebook/rocksdb), I remove some specific features, change some and make it more 104 | **UNIVERSAL** 105 | 106 | -------------------------------------------------------------------------------- /build_detect_platform: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Detects OS we're compiling on and outputs a file specified by the first 4 | # argument, which in turn gets read while processing Makefile. 5 | # 6 | # The output will set the following variables: 7 | # CC C Compiler path 8 | # CXX C++ Compiler path 9 | # PLATFORM_LDFLAGS Linker flags 10 | # PLATFORM_SHARED_EXT Extension for shared libraries 11 | # PLATFORM_SHARED_LDFLAGS Flags for building shared library 12 | # PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library 13 | # PLATFORM_CCFLAGS C compiler flags 14 | # PLATFORM_CXXFLAGS C++ compiler flags. Will contain: 15 | # PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned 16 | # shared libraries, empty otherwise. 17 | 18 | OUTPUT=$1 19 | if test -z "$OUTPUT"; then 20 | echo "usage: $0 " >&2 21 | exit 1 22 | fi 23 | 24 | # we depend on C++11 25 | PLATFORM_CXXFLAGS="-std=c++11" 26 | # we currently depend on POSIX platform 27 | COMMON_FLAGS= 28 | 29 | # Delete existing output, if it exists 30 | rm -f "$OUTPUT" 31 | touch "$OUTPUT" 32 | 33 | if test -z "$CC"; then 34 | CC=cc 35 | fi 36 | 37 | if test -z "$CXX"; then 38 | CXX=g++ 39 | fi 40 | 41 | # Detect OS 42 | if test -z "$TARGET_OS"; then 43 | TARGET_OS=`uname -s` 44 | fi 45 | 46 | if test -z "$TARGET_ARCHITECTURE"; then 47 | TARGET_ARCHITECTURE=`uname -m` 48 | fi 49 | 50 | COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}" 51 | CROSS_COMPILE= 52 | PLATFORM_CCFLAGS= 53 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS" 54 | PLATFORM_SHARED_EXT="so" 55 | PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl," 56 | PLATFORM_SHARED_CFLAGS="-fPIC" 57 | PLATFORM_SHARED_VERSIONED=true 58 | 59 | # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp 60 | case "$TARGET_OS" in 61 | Darwin) 62 | PLATFORM=OS_MACOSX 63 | COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX" 64 | PLATFORM_SHARED_EXT=dylib 65 | PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " 66 | # PORT_FILES=port/darwin/darwin_specific.cc 67 | ;; 68 | IOS) 69 | PLATFORM=IOS 70 | COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE " 71 | PLATFORM_SHARED_EXT=dylib 72 | PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " 73 | CROSS_COMPILE=true 74 | PLATFORM_SHARED_VERSIONED= 75 | ;; 76 | Linux) 77 | PLATFORM=OS_LINUX 78 | COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX" 79 | if [ -z "$USE_CLANG" ]; then 80 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" 81 | fi 82 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" 83 | # PORT_FILES=port/linux/linux_specific.cc 84 | ;; 85 | SunOS) 86 | PLATFORM=OS_SOLARIS 87 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS" 88 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" 89 | # PORT_FILES=port/sunos/sunos_specific.cc 90 | ;; 91 | FreeBSD) 92 | PLATFORM=OS_FREEBSD 93 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" 94 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" 95 | # PORT_FILES=port/freebsd/freebsd_specific.cc 96 | ;; 97 | NetBSD) 98 | PLATFORM=OS_NETBSD 99 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD" 100 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s" 101 | # PORT_FILES=port/netbsd/netbsd_specific.cc 102 | ;; 103 | OpenBSD) 104 | PLATFORM=OS_OPENBSD 105 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD" 106 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread" 107 | # PORT_FILES=port/openbsd/openbsd_specific.cc 108 | ;; 109 | DragonFly) 110 | PLATFORM=OS_DRAGONFLYBSD 111 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD" 112 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread" 113 | # PORT_FILES=port/dragonfly/dragonfly_specific.cc 114 | ;; 115 | Cygwin) 116 | PLATFORM=CYGWIN 117 | PLATFORM_SHARED_CFLAGS="" 118 | PLATFORM_CXXFLAGS="-std=gnu++11" 119 | COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN" 120 | if [ -z "$USE_CLANG" ]; then 121 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" 122 | fi 123 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" 124 | # PORT_FILES=port/linux/linux_specific.cc 125 | ;; 126 | OS_ANDROID_CROSSCOMPILE) 127 | PLATFORM=OS_ANDROID 128 | COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID " 129 | PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library 130 | # PORT_FILES=port/android/android.cc 131 | CROSS_COMPILE=true 132 | ;; 133 | *) 134 | echo "Unknown platform!" >&2 135 | exit 1 136 | esac 137 | 138 | PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}" 139 | 140 | if [ "$CROSS_COMPILE" = "true" ]; then 141 | # Cross-compiling; do not try any compilation tests. 142 | # Also don't need any compilation tests if compiling on fbcode 143 | true 144 | else 145 | if ! test $ROCKSUTIL_DISABLE_FALLOCATE; then 146 | # Test whether fallocate is available 147 | $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < 149 | #include 150 | int main() { 151 | int fd = open("/dev/null", 0); 152 | fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024); 153 | } 154 | EOF 155 | if [ "$?" = 0 ]; then 156 | COMMON_FLAGS="$COMMON_FLAGS -DROCKSUTIL_FALLOCATE_PRESENT" 157 | fi 158 | fi 159 | 160 | # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available 161 | $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < 163 | int main() { 164 | int x = PTHREAD_MUTEX_ADAPTIVE_NP; 165 | return 0; 166 | } 167 | EOF 168 | if [ "$?" = 0 ]; then 169 | COMMON_FLAGS="$COMMON_FLAGS -DROCKSUTIL_PTHREAD_ADAPTIVE_MUTEX" 170 | fi 171 | 172 | # Test if -pg is supported 173 | $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null </dev/null <> "$OUTPUT" 220 | echo "CXX=$CXX" >> "$OUTPUT" 221 | echo "PLATFORM=$PLATFORM" >> "$OUTPUT" 222 | echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT" 223 | echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT" 224 | echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT" 225 | echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> "$OUTPUT" 226 | echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> "$OUTPUT" 227 | echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> "$OUTPUT" 228 | echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT" 229 | echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT" 230 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | include ../make_config.mk 2 | 3 | .PHONY: clean librocksutil 4 | 5 | all: log_example mutexlock_example thread_local_example thread_pool_example lru_cache_example \ 6 | file_reader_writer_example wal_example 7 | 8 | log_example: librocksutil log_example.cc 9 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 10 | 11 | mutexlock_example: librocksutil mutexlock_example.cc 12 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 13 | 14 | thread_local_example: librocksutil thread_local_example.cc 15 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 16 | 17 | thread_pool_example: librocksutil thread_pool_example.cc 18 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 19 | 20 | lru_cache_example: librocksutil lru_cache_example.cc 21 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 22 | 23 | file_reader_writer_example: librocksutil file_reader_writer_example.cc 24 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 25 | 26 | wal_example: librocksutil wal_example.cc 27 | $(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksutil.a -I../ -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) 28 | 29 | clean: 30 | find . -name "*.[oda]" -exec rm -f {} \; 31 | rm -rf ./log_example ./mutexlock_example ./thread_local_example ./thread_pool_example \ 32 | ./lru_cache_example ./file_reader_writer_example ./wal_example 33 | 34 | librocksutil: 35 | cd .. && $(MAKE) static_lib 36 | -------------------------------------------------------------------------------- /examples/file_reader_writer_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/file_reader_writer.h" 2 | 3 | #include 4 | /* 5 | * Usage: 6 | * 1. WritableFileWriter 7 | * 2. RandomAccessFileReader 8 | * 3. SequenialFileReader 9 | * 4. ReadaheadRandomAccessFile 10 | * 11 | * use EnvOptions to tune the writer & reader: 12 | * 13 | * If true, then allow caching of data in environment buffers 14 | * bool use_os_buffer = true; 15 | * 16 | * If true, then use mmap to read data 17 | * bool use_mmap_reads = false; 18 | * 19 | * If true, then use mmap to write data 20 | * bool use_mmap_writes = true; 21 | * 22 | * If true, then use O_DIRECT for reading data 23 | * bool use_direct_reads = false; 24 | * 25 | * If true, then use O_DIRECT for writing data 26 | * bool use_direct_writes = false; 27 | * 28 | * If false, fallocate() calls are bypassed 29 | * bool allow_fallocate = true; 30 | * 31 | * If true, set the FD_CLOEXEC on open fd. 32 | * bool set_fd_cloexec = true; 33 | * 34 | * Allows OS to incrementally sync files to disk while they are being 35 | * written, in the background. Issue one request for every bytes_per_sync 36 | * written. 0 turns it off. 37 | * Default: 0 38 | * uint64_t bytes_per_sync = 0; 39 | * 40 | * If true, we will preallocate the file with FALLOC_FL_KEEP_SIZE flag, which 41 | * means that file size won't change as part of preallocation. 42 | * If false, preallocation will also change the file size. This option will 43 | * improve the performance in workloads where you sync the data on every 44 | * write. By default, we set it to true for MANIFEST writes and false for 45 | * WAL writes 46 | * bool fallocate_with_keep_size = true; 47 | * 48 | * size_t writable_file_max_buffer_size = 1024 * 1024; 49 | * 50 | */ 51 | using namespace rocksutil; 52 | 53 | int main() { 54 | Env* env = Env::Default(); 55 | EnvOptions env_options; 56 | /* 57 | * 1. Write [0 - 10000] (int32_t) to tmp_file sequentially 58 | */ 59 | // 1-1. Create a WritableFile and use it to Create a WritableFileWriter 60 | unique_ptr writable_file; 61 | Status s = NewWritableFile(env, "./tmp_file", &writable_file, env_options); 62 | if (!s.ok()) { 63 | std::cout << "NewWritableFile Error: " << s.ToString() << std::endl; 64 | } 65 | unique_ptr writer( 66 | new WritableFileWriter(std::move(writable_file), env_options)); 67 | 68 | // 1-2. Append sequentially 69 | for (int32_t i = 0; i < 10000; i++) { 70 | s = writer->Append(Slice((char*)&i, sizeof(i))); 71 | if (!s.ok()) { 72 | std::cout << "Append Error: " << s.ToString() << std::endl; 73 | } 74 | } 75 | uint64_t file_size = writer->GetFileSize(); 76 | std::cout << "[WritableFileWriter]" << std::endl; 77 | std::cout << "file_size: " << file_size << std::endl; 78 | 79 | // 1-3. Flush buffered data to OS buffer 80 | s = writer->Flush(); 81 | if (!s.ok()) { 82 | std::cout << "Flush Error: " << s.ToString() << std::endl; 83 | } 84 | 85 | // 1-4. Sync to disk 86 | s = writer->Sync(false); 87 | if (!s.ok()) { 88 | std::cout << "Sync Error: " << s.ToString() << std::endl; 89 | } 90 | 91 | // 1-5. Close the WritableFile 92 | s = writer->Close(); 93 | if (!s.ok()) { 94 | std::cout << "Close Error: " << s.ToString() << std::endl; 95 | } 96 | 97 | /* 98 | * 2. [Random] Read the 68th num in the tmp_file file, it tends to be 68; 99 | */ 100 | // 2-1. Create RandomAccessFile and use it to create a RandomAccessFileReader 101 | unique_ptr random_access_file; 102 | s = NewRandomAccessFile(env, "./tmp_file", &random_access_file, env_options); 103 | if (!s.ok()) { 104 | std::cout << "NewRandomAccessFile Error: " << s.ToString() << std::endl; 105 | } 106 | unique_ptr random_access_reader( 107 | new RandomAccessFileReader(std::move(random_access_file))); 108 | // 2-2. Read the 68th num 109 | char backing_store[1024]; 110 | Slice result; 111 | s = random_access_reader->Read(68 * sizeof(int32_t), sizeof(int32_t), &result, backing_store); 112 | if (!s.ok()) { 113 | std::cout << "[Random]Read Error: " << s.ToString() << std::endl; 114 | } 115 | int32_t num; 116 | memcpy(&num, result.data(), sizeof(int32_t)); 117 | std::cout << std::endl << "[RandomAccessFileReader]" << std::endl; 118 | std::cout << num << std::endl; 119 | 120 | /* 121 | * 3. [Sequential] Read the [69-168]th num in the tmp_file file, they tend to be [68-167] 122 | */ 123 | // 3-1. Create SequentialFile and use it to create a SequenialFileReader 124 | unique_ptr sequential_file; 125 | s = NewSequentialFile(env, "./tmp_file", &sequential_file, env_options); 126 | if (!s.ok()) { 127 | std::cout << "NewSequentialFile Error: " << s.ToString() << std::endl; 128 | } 129 | unique_ptr sequential_reader( 130 | new SequentialFileReader(std::move(sequential_file))); 131 | 132 | // 3-2. Skip to the 69th num 133 | s = sequential_reader->Skip(68 * sizeof(int32_t)); 134 | if (!s.ok()) { 135 | std::cout << "Skip Error: " << s.ToString() << std::endl; 136 | } 137 | // 3-3. Sequential Read 138 | std::cout << std::endl << "[SequentialFileReader]" << std::endl; 139 | for (int i = 0; i < 100; i++) { 140 | s = sequential_reader->Read(sizeof(int32_t), &result, backing_store); 141 | if (!s.ok()) { 142 | std::cout << "[Sequential]Read Error: " << s.ToString() << std::endl; 143 | break; 144 | } 145 | memcpy(&num, result.data(), sizeof(int32_t)); 146 | std::cout << num << " "; 147 | } 148 | std::cout << std::endl; 149 | 150 | /* 151 | * 4. [Read Ahead Random] Read the 668th & 688th num in the tmp_file file, it tends to be 668 & 688; 152 | */ 153 | // 4-1. Create a RandomAccessFile and use it to create a [ReadAhead]RandomAccessFile 154 | s = NewRandomAccessFile(env, "./tmp_file", &random_access_file, env_options); 155 | if (!s.ok()) { 156 | std::cout << "NewRandomAccessFile Error: " << s.ToString() << std::endl; 157 | } 158 | unique_ptr readahead_random_access_file( 159 | NewReadaheadRandomAccessFile(std::move(random_access_file), 1024)); 160 | s = readahead_random_access_file->Read(668 * sizeof(int32_t), sizeof(int32_t), &result, backing_store); 161 | if (!s.ok()) { 162 | std::cout << "[RandAhead]Read Error: " << s.ToString() << std::endl; 163 | } 164 | memcpy(&num, result.data(), sizeof(int32_t)); 165 | std::cout << std::endl << "[ReadaheadRandomAccessFile]" << std::endl; 166 | std::cout << num << " "; 167 | s = readahead_random_access_file->Read(688 * sizeof(int32_t), sizeof(int32_t), &result, backing_store); 168 | if (!s.ok()) { 169 | std::cout << "[RandAhead]Read Error: " << s.ToString() << std::endl; 170 | } 171 | memcpy(&num, result.data(), sizeof(int32_t)); 172 | std::cout << num << std::endl; 173 | 174 | return 0; 175 | } 176 | -------------------------------------------------------------------------------- /examples/log_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/auto_roll_logger.h" 2 | 3 | #include 4 | 5 | using namespace rocksutil; 6 | 7 | /* 8 | * Log Level: 9 | * InfoLogLevel::DEBUG_LEVEL; 10 | * InfoLogLevel::INFO_LEVEL; 11 | * InfoLogLevel::WARN_LEVEL; 12 | * InfoLogLevel::ERROR_LEVEL; 13 | * InfoLogLevel::FATAL_LEVEL; 14 | * InfoLogLevel::HEADER_LEVEL; 15 | * 16 | * Log Function: 17 | * Debug(log, ...); or LOG(InfoLogLevel::DEBUG_LEVEL, log, ...); 18 | * Info(log, ...); or LOG(InfoLogLevel::DEBUG_INFO, log, ...); 19 | * Warn(log, ...); or LOG(InfoLogLevel::DEBUG_WARN, log, ...); 20 | * Error(log, ...); or LOG(InfoLogLevel::DEBUG_ERROR, log, ...); 21 | * Fatal(log, ...); or LOG(InfoLogLevel::DEBUG_FATAL, log, ...); 22 | * Header(log, ...); or LOG(InfoLogLevel::DEBUG_HEADER, log, ...); 23 | * 24 | * Usage: 25 | * Status CreateLogger(const std::string log_path, std::shared_ptr* log, 26 | * size_t log_max_size = 0 [never roll], 27 | * size_t log_file_time_to_roll = 0 [never roll], 28 | * const InfoLogLevel = InfoLogLevel::INFO_LEVEL, 29 | * uint64_t flush_every_seconds = 5); 30 | */ 31 | 32 | 33 | std::atomic exit_flag(false); 34 | 35 | void WriteLog(std::shared_ptrlog, int thread_num) { 36 | while(exit_flag.load(std::memory_order_acquire) == false) { 37 | Info(log, "Thread %d write a log\n", thread_num); 38 | } 39 | } 40 | 41 | int main() { 42 | 43 | Env* env = Env::Default(); 44 | std::shared_ptr log; 45 | // 1. Default logger: never roll log file, log_level = InfoLogLevel::INFO_LEVEL; 46 | // flush_every_seconds = 5 47 | Status s = CreateLogger("./log_path", &log, env); 48 | 49 | // 2. logger: roll to next file per 5s, log_level = InfoLogLevel::WARN_LEVEL; 50 | // always flush 51 | // Status s = CreateLogger("tmp", &log, 0, 5, InfoLogLevel::WARN_LEVEL, 0); 52 | 53 | // 3. logger roll to next file per 100M, log_level = InfoLogLevel::ERROR_LEVEL; 54 | // flush every 3s 55 | // Status s = CreateLogger("tmp", &log, 1024*1024*100, 0, InfoLogLevel::WARN_LEVEL, 3); 56 | std::cout << s.ToString() << std::endl; 57 | 58 | Header(log, "-------------------------------------------------------------------"); 59 | Header(log, "| |"); 60 | Header(log, "| This is the header of log, it will be printed in the header of |"); 61 | Header(log, "| every log file. |"); 62 | Header(log, "| |"); 63 | Header(log, "-------------------------------------------------------------------"); 64 | 65 | std::thread t1 = std::thread(WriteLog, log, 1); 66 | std::thread t2 = std::thread(WriteLog, log, 2); 67 | 68 | getchar(); 69 | exit_flag.store(true, std::memory_order_release); 70 | 71 | t1.join(); 72 | t2.join(); 73 | 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /examples/lru_cache_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/cache.h" 2 | 3 | #include 4 | 5 | using namespace rocksutil; 6 | 7 | /* Usage: 8 | * 9 | * 1. 10 | * Create a new cache with a fixed size capacity. The cache is sharded 11 | * to 2^num_shard_bits shards, by hash of the key. The total capacity 12 | * is divided and evenly assigned to each shard. If strict_capacity_limit 13 | * is set, insert to the cache will fail when cache is full. User can also 14 | * set percentage of the cache reserves for high priority entries via 15 | * high_pri_pool_pct. 16 | * 17 | * std::shared_ptr NewLRUCache(size_t capacity, 18 | * int num_shard_bits = 6, 19 | * bool strict_capacity_limit = false, 20 | * double high_pri_pool_ratio = 0.0); 21 | * 22 | * 2. 23 | * Insert a mapping from key->value into the cache and assign it 24 | * the specified charge against the total cache capacity. 25 | * If strict_capacity_limit is true and cache reaches its full capacity, 26 | * return Status::Incomplete. 27 | * 28 | * If handle is not nullptr, returns a handle that corresponds to the 29 | * mapping. The caller must call this->Release(handle) when the returned 30 | * mapping is no longer needed. In case of error caller is responsible to 31 | * cleanup the value (i.e. calling "deleter"). 32 | * 33 | * If handle is nullptr, it is as if Release is called immediately after 34 | * insert. In case of error value will be cleanup. 35 | * 36 | * When the inserted entry is no longer needed, the key and 37 | * value will be passed to "deleter". 38 | * 39 | * virtual Status Insert(const Slice& key, void* value, size_t charge, 40 | * void (*deleter)(const Slice& key, void* value), 41 | * Handle** handle = nullptr, 42 | * Priority priority = Priority::LOW); 43 | * 44 | * 3. 45 | * If the cache has no mapping for "key", returns nullptr. 46 | * 47 | * Else return a handle that corresponds to the mapping. The caller 48 | * must call this->Release(handle) when the returned mapping is no 49 | * longer needed. 50 | * 51 | * virtual Handle* Lookup(const Slice& key); 52 | * 53 | * 4. 54 | * Release a mapping returned by a previous Lookup(). 55 | * REQUIRES: handle must not have been released yet. 56 | * REQUIRES: handle must have been returned by a method on *this. 57 | * 58 | * virtual void Release(Handle* handle); 59 | * 60 | * 5. 61 | * Return the value encapsulated in a handle returned by a 62 | * successful Lookup(). 63 | * REQUIRES: handle must not have been released yet. 64 | * REQUIRES: handle must have been returned by a method on *this. 65 | * 66 | * virtual void* Value(Handle* handle); 67 | * 68 | * More Usage in include/rocksutil/cache.h 69 | */ 70 | 71 | class Entity { 72 | public: 73 | Entity(int n) : num_(n) { 74 | } 75 | ~Entity() { 76 | std::cout << "Destruct Entity " << num_ << std::endl; 77 | } 78 | int num() { 79 | return num_; 80 | } 81 | private: 82 | int num_; 83 | }; 84 | 85 | void Deleter(const Slice& key, void* value) { 86 | delete static_cast(value); 87 | } 88 | 89 | int main() { 90 | 91 | 92 | std::shared_ptr lru_cache = NewLRUCache(64); 93 | Entity* ptr = nullptr; 94 | Status s; 95 | Cache::Handle* handles[43]; 96 | for (int i = 0; i < 86; i++) { 97 | ptr = new Entity(i); 98 | if (i % 2 == 0) { 99 | s = lru_cache->Insert(std::to_string(i), ptr, 1, 100 | &Deleter, &(handles[i/2])); 101 | } else { 102 | s = lru_cache->Insert(std::to_string(i), ptr, 1, 103 | &Deleter); 104 | } 105 | std::cout << "Insert key: " << std::to_string(i) << ", return: " 106 | << s.ToString() << " " << lru_cache->GetCapacity() << " " 107 | << lru_cache->GetUsage() << " " << lru_cache->GetPinnedUsage() << std::endl; 108 | } 109 | getchar(); 110 | 111 | for (int i = 0; i < 43; i++) { 112 | lru_cache->Release(handles[i]); 113 | } 114 | getchar(); 115 | 116 | Cache::Handle* handle = nullptr; 117 | for (int i = 0; i < 86; i++) { 118 | handle = lru_cache->Lookup(std::to_string(i)); 119 | if (handle) { 120 | std::cout << "Hit, key: " << std::to_string(i) << ", value: " 121 | << static_cast(lru_cache->Value(handle))->num() << std::endl; 122 | lru_cache->Release(handle); 123 | } else { 124 | std::cout << "Miss key: " << std::to_string(i) << std::endl; 125 | } 126 | } 127 | getchar(); 128 | 129 | return 0; 130 | } 131 | -------------------------------------------------------------------------------- /examples/mutexlock_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/mutexlock.h" 2 | 3 | #include 4 | 5 | using namespace rocksutil; 6 | 7 | int main() { 8 | /* 9 | * 1. MutexLock 10 | */ 11 | port::Mutex mutex; 12 | { 13 | MutexLock l(&mutex); 14 | std::cout << "In MutexLock Scope" << std::endl; 15 | } 16 | /* 17 | * 2. ReadLock & ReadUnlock 18 | */ 19 | port::RWMutex rw_mutex; 20 | { 21 | ReadLock l(&rw_mutex); 22 | std::cout << "In RWMutex ReadLock Scope" << std::endl; 23 | } 24 | { 25 | rw_mutex.ReadLock(); 26 | ReadUnlock l(&rw_mutex); 27 | std::cout << "In RWMutex ReadLock Scope" << std::endl; 28 | } 29 | /* 30 | * 3. WriteLock 31 | */ 32 | { 33 | WriteLock l(&rw_mutex); 34 | std::cout << "In RWMutex WriteLock Scope" << std::endl; 35 | } 36 | /* 37 | * 4. SpinMutex 38 | */ 39 | SpinMutex spin_mutex; 40 | spin_mutex.lock(); 41 | std::cout << "SpinMutex, lock" << std::endl; 42 | bool ret = spin_mutex.try_lock(); 43 | std::cout << "SpinMutex, try_lock return " << ret << std::endl; 44 | spin_mutex.unlock(); 45 | std::cout << "SpinMutex, unlock" << std::endl; 46 | 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /examples/thread_local_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/thread_local.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace rocksutil; 8 | 9 | /* 10 | * Usage: 11 | * 12 | * 1) UnrefHandle: 13 | * Cleanup function that will be called for a stored thread local 14 | * pointer (if not NULL) when one of the following happens: 15 | * (1) a thread terminates 16 | * (2) a ThreadLocalPtr is destroyed 17 | * 18 | * 2) void* Get(): 19 | * Return the current pointer stored in thread local 20 | * 21 | * 3) void Reset(void* ptr): 22 | * Set a new pointer value to the thread local storage. 23 | * 24 | * 4) void* Swap(void* ptr): 25 | * Atomically swap the supplied ptr and return the previous value 26 | * 27 | * 5) bool CompareAndSwap(void* ptr, void*& expected): 28 | * Atomically compare the stored value with expected. Set the new 29 | * pointer value to thread local only if the comparison is true. 30 | * Otherwise, expected returns the stored value. 31 | * Return true on success, false on failure 32 | * 33 | * 6) void Scrape(autovector* ptrs, void* const replacement): 34 | * Reset all thread local data to replacement, and return non-nullptr 35 | * data for all existing threads 36 | */ 37 | 38 | class LocalData { 39 | public: 40 | LocalData(int first) { 41 | array = new int[2]; 42 | array[0] = first; 43 | } 44 | ~LocalData() { 45 | std::cout << "Destruct LocalData with First = " << array[0] << std::endl; 46 | delete[] array; 47 | } 48 | int GetFirst() { 49 | return array[0]; 50 | } 51 | void SetFirst(int num) { 52 | array[0] = num; 53 | } 54 | private: 55 | int* array; 56 | }; 57 | void UnrefHandle(void* ptr) { 58 | delete static_cast(ptr); 59 | } 60 | 61 | void VisitLocalData(ThreadLocalPtr* thread_local_ptr, int first) { 62 | LocalData *p = new LocalData(first); 63 | 64 | void* old_ptr = thread_local_ptr->Swap(p); 65 | std::cout << "thread " << std::this_thread::get_id() << 66 | " Swap, old_ptr = " << old_ptr << std::endl; 67 | 68 | void* cur_ptr = thread_local_ptr->Get(); 69 | std::cout << "thread " << std::this_thread::get_id() << 70 | " Get, GetFirst = " << static_cast(cur_ptr)->GetFirst() << 71 | std::endl; 72 | } 73 | 74 | int main() { 75 | ThreadLocalPtr* thread_local_ptr = new ThreadLocalPtr(&UnrefHandle); 76 | std::thread t1 = std::thread(VisitLocalData, thread_local_ptr, 6); 77 | std::this_thread::sleep_for(std::chrono::milliseconds(100)); 78 | std::thread t2 = std::thread(VisitLocalData, thread_local_ptr, 8); 79 | 80 | t1.join(); 81 | t2.join(); 82 | std::this_thread::sleep_for(std::chrono::milliseconds(500)); 83 | std::cout << "Bye" << std::endl; 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /examples/thread_pool_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/env.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace rocksutil; 8 | /* 9 | * Usage: 10 | * 11 | * Arrange to run "(*function)(arg)" once in a background thread, in 12 | * the thread pool specified by pri. By default, jobs go to the 'LOW' 13 | * priority thread pool. 14 | * 15 | * 1. 16 | * "function" may run in an unspecified thread. Multiple functions 17 | * added to the same Env may run concurrently in different threads. 18 | * I.e., the caller may not assume that background work items are 19 | * serialized. 20 | * When the UnSchedule function is called, the unschedFunction 21 | * registered at the time of Schedule is invoked with arg as a parameter. 22 | * void Schedule(void (*function)(void* arg), void* arg, 23 | * Priority pri = LOW, void* tag = nullptr, 24 | * void (*unschedFunction)(void* arg) = 0); 25 | * 26 | * 2. 27 | * Arrange to remove jobs for given arg from the queue_ if they are not 28 | * already scheduled. Caller is expected to have exclusive lock on arg. 29 | * int UnSchedule(void* arg, Priority pri); 30 | * 31 | * 3. 32 | * Start a new thread, invoking "function(arg)" within the new thread. 33 | * When "function(arg)" returns, the thread will be destroyed. 34 | * void StartThread(void (*function)(void* arg), void* arg); 35 | * 36 | * 4. 37 | * Wait for all threads started by StartThread to terminate. 38 | * void WaitForJoin(); 39 | * 40 | * 5. 41 | * Get thread pool queue length for specific thrad pool. 42 | * virtual unsigned int GetThreadPoolQueueLen(Priority pri = LOW); 43 | * 44 | * 6. 45 | * The number of background worker threads of a specific thread pool 46 | * for this environment. 'LOW' is the default pool. 47 | * default number: 1 48 | * void SetBackgroundThreads(int number, Priority pri = LOW); 49 | * 50 | * 7. 51 | * Enlarge number of background worker threads of a specific thread pool 52 | * for this environment if it is smaller than specified. 'LOW' is the default 53 | * pool. 54 | * void IncBackgroundThreadsIfNeeded(int number, Priority pri); 55 | * 56 | * 8. 57 | * Lower IO priority for threads from the specified pool. 58 | * void LowerThreadPoolIOPriority(Priority pool = LOW); 59 | */ 60 | 61 | struct Msg { 62 | int num; 63 | std::string str; 64 | }; 65 | 66 | void HighPrioFunc(void* arg) { 67 | std::cout << "High Priority[Fast] thread " << std::this_thread::get_id() << 68 | " proccess:" << (static_cast(arg))->str << std::endl; 69 | } 70 | 71 | void HighPrioSlowFunc(void* arg) { 72 | std::cout << "High Priority[Slow] thread " << std::this_thread::get_id() << 73 | " proccess:" << (static_cast(arg))->num << std::endl; 74 | std::this_thread::sleep_for(std::chrono::milliseconds(1)); 75 | } 76 | 77 | void LowPrioFunc(void* arg) { 78 | std::cout << "Low Priority thread " << std::this_thread::get_id() << 79 | " proccess:" << (static_cast(arg))->num << std::endl; 80 | } 81 | 82 | int main() { 83 | Env* env = Env::Default(); 84 | Msg msg = {6, "hello"}; 85 | /* 86 | * 1. Lower the LOW thread priority 87 | */ 88 | env->LowerThreadPoolIOPriority(Env::Priority::LOW); 89 | /* 90 | * 2. Schedule 91 | */ 92 | env->Schedule(&HighPrioFunc, &msg, Env::Priority::HIGH); 93 | std::this_thread::sleep_for(std::chrono::milliseconds(10)); 94 | env->Schedule(&LowPrioFunc, &msg, Env::Priority::LOW); 95 | std::this_thread::sleep_for(std::chrono::milliseconds(10)); 96 | /* 97 | * 3. Increase the number of HIGH priority background threads 98 | * to 2 TEMPORIRARILY, If you want to Change it PERMANENTLY, 99 | * use SetBackgroundThreads(int number, Priority pri); 100 | */ 101 | env->IncBackgroundThreadsIfNeeded(2, Env::Priority::HIGH); 102 | /* 103 | * 4. continue Scheduling 104 | */ 105 | for (int i = 0; i < 50; i++) { 106 | env->Schedule(&HighPrioSlowFunc, &msg, Env::Priority::HIGH); 107 | env->Schedule(&HighPrioFunc, &msg, Env::Priority::HIGH); 108 | } 109 | 110 | std::this_thread::sleep_for(std::chrono::milliseconds(10)); 111 | // GetThreadPoolQueueLen 112 | std::cout << "Before Unschedule Queue size: " << env->GetThreadPoolQueueLen( 113 | Env::Priority::HIGH) << std::endl; 114 | /* 115 | * 5. should Unschedule 116 | */ 117 | env->UnSchedule(nullptr, Env::Priority::HIGH); 118 | std::cout << "After Unschedule Queue size: " << env->GetThreadPoolQueueLen( 119 | Env::Priority::HIGH) << std::endl; 120 | /* 121 | * 6. should WaitForJoin 122 | */ 123 | env->WaitForJoin(); 124 | /* 125 | * sleep for a while to wait for all background threads exiting 126 | */ 127 | std::this_thread::sleep_for(std::chrono::milliseconds(500)); 128 | 129 | return 0; 130 | } 131 | -------------------------------------------------------------------------------- /examples/wal_example.cc: -------------------------------------------------------------------------------- 1 | #include "rocksutil/log_writer.h" 2 | #include "rocksutil/log_reader.h" 3 | #include "rocksutil/file_reader_writer.h" 4 | #include "rocksutil/mutexlock.h" 5 | #include "rocksutil/slice.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | /* 13 | * Usage: 14 | * 1. Writer(unique_ptr&& dest, 15 | * uint64_t log_number = 0, 16 | * bool recycle_log_files = false); 17 | * 18 | * Status AddRecord(const Slice& slice); 19 | * 20 | * 2. Reader(unique_ptr&& file, 21 | * Reporter* reporter, bool checksum, uint64_t initial_offset, 22 | * uint64_t log_num = 0); 23 | * 24 | * bool ReadRecord(Slice* record, std::string* scratch, 25 | * WALRecoveryMode wal_recovery_mode = 26 | * WALRecoveryMode::kTolerateCorruptedTailRecords); 27 | * 28 | */ 29 | using namespace rocksutil; 30 | 31 | Env* env = Env::Default(); 32 | EnvOptions env_options; 33 | port::Mutex mutex; 34 | port::CondVar cv(&mutex); 35 | std::atomic should_exit(false); 36 | uint64_t writer_offset = 0; 37 | uint64_t reader_offset = 0; 38 | 39 | void WriterFunc() { 40 | /* 41 | * 1. Create a log::Writer: WritableFile -> WritableFileWriter -> 42 | * log::Writer 43 | * NOTICE: log::Writer would truncate the walfile to 0; 44 | */ 45 | unique_ptr writable_file; 46 | Status s = NewWritableFile(env, "./wal_file", &writable_file, env_options); 47 | if (!s.ok()) { 48 | std::cout << "NewWritableFile Error: " << s.ToString() << std::endl; 49 | return; 50 | } 51 | unique_ptr writer( 52 | new WritableFileWriter(std::move(writable_file), env_options)); 53 | log::Writer log_writer(std::move(writer)); 54 | 55 | uint64_t i = 0; 56 | while (!should_exit) { 57 | { 58 | MutexLock l(&mutex); 59 | s = log_writer.AddRecord("WAL_" + std::to_string(i)); 60 | if (!s.ok()) { 61 | std::cout << "AddRecord Error: " << s.ToString() << std::endl; 62 | } else { 63 | i++; 64 | writer_offset = log_writer.file()->GetFileSize(); 65 | cv.SignalAll(); 66 | } 67 | } 68 | std::this_thread::sleep_for(std::chrono::milliseconds(500)); 69 | } 70 | } 71 | 72 | void ReaderFunc() { 73 | /* 74 | * 2. Create a log::Reader: SequentialFile -> SequentialFileReader -> 75 | * log::Reader 76 | */ 77 | unique_ptr sequential_file; 78 | Status s; 79 | s = NewSequentialFile(env, "./wal_file", &sequential_file, env_options); 80 | if (!s.ok()) { 81 | std::cout << "NewSequentialFile Error: " << s.ToString() << std::endl; 82 | return; 83 | } 84 | unique_ptr sequential_reader( 85 | new SequentialFileReader(std::move(sequential_file))); 86 | 87 | // if you want to get the inner error in ReadRecord, 88 | // construct a log::Reader::LogReporter and pass its 89 | // address to the log::Reader Construction, otherwise, 90 | // just pass nullptr; 91 | log::Reader::LogReporter reporter; 92 | reporter.status = &s; 93 | 94 | log::Reader log_reader(std::move(sequential_reader), &reporter, 95 | true, 0); 96 | std::string scratch; 97 | Slice record; 98 | // std::cout << log_reader.IsEOF() << std::endl; 99 | bool ret = false; 100 | while (!should_exit) { 101 | ret = log_reader.ReadRecord(&record, &scratch); 102 | if (ret) { 103 | std::cout << std::string(record.data(), record.size()) << std::endl; 104 | } else { 105 | if (s.ok()) { 106 | mutex.Lock(); 107 | reader_offset = log_reader.EndOfBufferOffset(); 108 | while (writer_offset == reader_offset) { 109 | cv.Wait(); 110 | if (should_exit) { 111 | mutex.Unlock(); 112 | return; 113 | } 114 | } 115 | mutex.Unlock(); 116 | log_reader.UnmarkEOF(); 117 | } else { 118 | std::cout << "log_reader error: " << s.ToString() << std::endl; 119 | return; 120 | } 121 | } 122 | } 123 | } 124 | 125 | int main() { 126 | std::thread writer(WriterFunc); 127 | std::thread reader(ReaderFunc); 128 | getchar(); 129 | should_exit = true; 130 | cv.SignalAll(); 131 | writer.join(); 132 | reader.join(); 133 | return 0; 134 | } 135 | -------------------------------------------------------------------------------- /include/rocksutil/auto_roll_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Logger implementation that can be shared by all environments 7 | // where enough posix functionality is available. 8 | 9 | #pragma once 10 | #include 11 | #include 12 | 13 | #include "rport/port.h" 14 | #include "rport/util_logger.h" 15 | #include "rocksutil/mutexlock.h" 16 | 17 | namespace rocksutil { 18 | 19 | std::string InfoLogFileName(const std::string& log_path); 20 | 21 | // Rolls the log file by size and/or time 22 | class AutoRollLogger : public Logger { 23 | public: 24 | AutoRollLogger(Env* env, const std::string& log_path, 25 | size_t log_max_size, size_t log_file_time_to_roll, 26 | const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL, 27 | uint64_t flush_every_seconds = 5) 28 | : Logger(flush_every_seconds, log_level), 29 | log_path_(log_path), 30 | env_(env), 31 | status_(Status::OK()), 32 | kMaxLogFileSize(log_max_size), 33 | kLogFileTimeToRoll(log_file_time_to_roll), 34 | cached_now(static_cast(env_->NowMicros() * 1e-6)), 35 | ctime_(cached_now), 36 | cached_now_access_count(0), 37 | call_NowMicros_every_N_records_(100), 38 | mutex_() { 39 | log_fname_ = InfoLogFileName(log_path_); 40 | RollLogFile(); 41 | ResetLogger(); 42 | } 43 | 44 | using Logger::Logv; 45 | void Logv(const char* format, va_list ap) override; 46 | 47 | // Write a header entry to the log. All header information will be written 48 | // again every time the log rolls over. 49 | virtual void LogHeader(const char* format, va_list ap) override; 50 | 51 | // check if the logger has encountered any problem. 52 | Status GetStatus() { 53 | return status_; 54 | } 55 | 56 | size_t GetLogFileSize() const override { 57 | std::shared_ptr logger; 58 | { 59 | MutexLock l(&mutex_); 60 | // pin down the current logger_ instance before releasing the mutex. 61 | logger = logger_; 62 | } 63 | return logger->GetLogFileSize(); 64 | } 65 | 66 | void Flush() override { 67 | std::shared_ptr logger; 68 | { 69 | MutexLock l(&mutex_); 70 | // pin down the current logger_ instance before releasing the mutex. 71 | logger = logger_; 72 | } 73 | if (logger) { 74 | logger->Flush(); 75 | } 76 | } 77 | 78 | virtual ~AutoRollLogger() { 79 | } 80 | 81 | void SetCallNowMicrosEveryNRecords(uint64_t call_NowMicros_every_N_records) { 82 | call_NowMicros_every_N_records_ = call_NowMicros_every_N_records; 83 | } 84 | 85 | // Expose the log file path for testing purpose 86 | std::string TEST_log_fname() const { 87 | return log_fname_; 88 | } 89 | 90 | private: 91 | bool LogExpired(); 92 | Status ResetLogger(); 93 | void RollLogFile(); 94 | // Log message to logger without rolling 95 | void LogInternal(const char* format, ...); 96 | // Serialize the va_list to a string 97 | std::string ValistToString(const char* format, va_list args) const; 98 | // Write the logs marked as headers to the new log file 99 | void WriteHeaderInfo(); 100 | 101 | std::string log_fname_; // Current active info log's file name. 102 | std::string log_path_; 103 | Env* env_; 104 | std::shared_ptr logger_; 105 | // current status of the logger 106 | Status status_; 107 | const size_t kMaxLogFileSize; 108 | const size_t kLogFileTimeToRoll; 109 | // header information 110 | std::list headers_; 111 | // to avoid frequent env->NowMicros() calls, we cached the current time 112 | uint64_t cached_now; 113 | uint64_t ctime_; 114 | uint64_t cached_now_access_count; 115 | uint64_t call_NowMicros_every_N_records_; 116 | mutable port::Mutex mutex_; 117 | }; 118 | 119 | Status CreateLogger(const std::string& log_path, 120 | std::shared_ptr* logger, Env* env, 121 | size_t log_max_size = 0, size_t log_file_time_to_roll = 0, 122 | const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL, 123 | uint64_t flush_every_seconds = 5); 124 | 125 | } // namespace rocksutil 126 | -------------------------------------------------------------------------------- /include/rocksutil/cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // A Cache is an interface that maps keys to values. It has internal 11 | // synchronization and may be safely accessed concurrently from 12 | // multiple threads. It may automatically evict entries to make room 13 | // for new entries. Values have a specified charge against the cache 14 | // capacity. For example, a cache where the values are variable 15 | // length strings, may use the length of the string as the charge for 16 | // the string. 17 | // 18 | // A builtin cache implementation with a least-recently-used eviction 19 | // policy is provided. Clients may use their own implementations if 20 | // they want something more sophisticated (like scan-resistance, a 21 | // custom eviction policy, variable cache sizing, etc.) 22 | 23 | #pragma once 24 | 25 | #include 26 | #include 27 | #include "rocksutil/slice.h" 28 | #include "rocksutil/status.h" 29 | 30 | namespace rocksutil { 31 | 32 | class Cache; 33 | 34 | // Create a new cache with a fixed size capacity. The cache is sharded 35 | // to 2^num_shard_bits shards, by hash of the key. The total capacity 36 | // is divided and evenly assigned to each shard. If strict_capacity_limit 37 | // is set, insert to the cache will fail when cache is full. User can also 38 | // set percentage of the cache reserves for high priority entries via 39 | // high_pri_pool_pct. 40 | extern std::shared_ptr NewLRUCache(size_t capacity, 41 | int num_shard_bits = 6, 42 | bool strict_capacity_limit = false, 43 | double high_pri_pool_ratio = 0.0); 44 | 45 | // Similar to NewLRUCache, but create a cache based on CLOCK algorithm with 46 | // better concurrent performance in some cases. See util/clock_cache.cc for 47 | // more detail. 48 | // 49 | // Return nullptr if it is not supported. 50 | //extern std::shared_ptr NewClockCache(size_t capacity, 51 | // int num_shard_bits = 6, 52 | // bool strict_capacity_limit = false); 53 | 54 | class Cache { 55 | public: 56 | // Depending on implementation, cache entries with high priority could be less 57 | // likely to get evicted than low priority entries. 58 | enum class Priority { HIGH, LOW }; 59 | 60 | Cache() {} 61 | 62 | // Destroys all existing entries by calling the "deleter" 63 | // function that was passed via the Insert() function. 64 | // 65 | // @See Insert 66 | virtual ~Cache() {} 67 | 68 | // Opaque handle to an entry stored in the cache. 69 | struct Handle {}; 70 | 71 | // The type of the Cache 72 | virtual const char* Name() const = 0; 73 | 74 | // Insert a mapping from key->value into the cache and assign it 75 | // the specified charge against the total cache capacity. 76 | // If strict_capacity_limit is true and cache reaches its full capacity, 77 | // return Status::Incomplete. 78 | // 79 | // If handle is not nullptr, returns a handle that corresponds to the 80 | // mapping. The caller must call this->Release(handle) when the returned 81 | // mapping is no longer needed. In case of error caller is responsible to 82 | // cleanup the value (i.e. calling "deleter"). 83 | // 84 | // If handle is nullptr, it is as if Release is called immediately after 85 | // insert. In case of error value will be cleanup. 86 | // 87 | // When the inserted entry is no longer needed, the key and 88 | // value will be passed to "deleter". 89 | virtual Status Insert(const Slice& key, void* value, size_t charge, 90 | void (*deleter)(const Slice& key, void* value), 91 | Handle** handle = nullptr, 92 | Priority priority = Priority::LOW) = 0; 93 | 94 | // If the cache has no mapping for "key", returns nullptr. 95 | // 96 | // Else return a handle that corresponds to the mapping. The caller 97 | // must call this->Release(handle) when the returned mapping is no 98 | // longer needed. 99 | virtual Handle* Lookup(const Slice& key) = 0; 100 | 101 | // Release a mapping returned by a previous Lookup(). 102 | // REQUIRES: handle must not have been released yet. 103 | // REQUIRES: handle must have been returned by a method on *this. 104 | virtual void Release(Handle* handle) = 0; 105 | 106 | // Return the value encapsulated in a handle returned by a 107 | // successful Lookup(). 108 | // REQUIRES: handle must not have been released yet. 109 | // REQUIRES: handle must have been returned by a method on *this. 110 | virtual void* Value(Handle* handle) = 0; 111 | 112 | // If the cache contains entry for key, erase it. Note that the 113 | // underlying entry will be kept around until all existing handles 114 | // to it have been released. 115 | virtual void Erase(const Slice& key) = 0; 116 | // Return a new numeric id. May be used by multiple clients who are 117 | // sharding the same cache to partition the key space. Typically the 118 | // client will allocate a new id at startup and prepend the id to 119 | // its cache keys. 120 | virtual uint64_t NewId() = 0; 121 | 122 | // sets the maximum configured capacity of the cache. When the new 123 | // capacity is less than the old capacity and the existing usage is 124 | // greater than new capacity, the implementation will do its best job to 125 | // purge the released entries from the cache in order to lower the usage 126 | virtual void SetCapacity(size_t capacity) = 0; 127 | 128 | // Set whether to return error on insertion when cache reaches its full 129 | // capacity. 130 | virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0; 131 | 132 | // Get the flag whether to return error on insertion when cache reaches its 133 | // full capacity. 134 | virtual bool HasStrictCapacityLimit() const = 0; 135 | 136 | // returns the maximum configured capacity of the cache 137 | virtual size_t GetCapacity() const = 0; 138 | 139 | // returns the memory size for the entries residing in the cache. 140 | virtual size_t GetUsage() const = 0; 141 | 142 | // returns the memory size for a specific entry in the cache. 143 | virtual size_t GetUsage(Handle* handle) const = 0; 144 | 145 | // returns the memory size for the entries in use by the system 146 | virtual size_t GetPinnedUsage() const = 0; 147 | 148 | // Call this on shutdown if you want to speed it up. Cache will disown 149 | // any underlying data and will not free it on delete. This call will leak 150 | // memory - call this only if you're shutting down the process. 151 | // Any attempts of using cache after this call will fail terribly. 152 | // Always delete the DB object before calling this method! 153 | virtual void DisownData(){ 154 | // default implementation is noop 155 | }; 156 | 157 | // Apply callback to all entries in the cache 158 | // If thread_safe is true, it will also lock the accesses. Otherwise, it will 159 | // access the cache without the lock held 160 | virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t), 161 | bool thread_safe) = 0; 162 | 163 | // Remove all entries. 164 | // Prerequisit: no entry is referenced. 165 | virtual void EraseUnRefEntries() = 0; 166 | 167 | private: 168 | // No copying allowed 169 | Cache(const Cache&); 170 | Cache& operator=(const Cache&); 171 | }; 172 | 173 | } // namespace rocksutil 174 | -------------------------------------------------------------------------------- /include/rocksutil/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | #include 12 | #include 13 | 14 | namespace rocksutil { 15 | namespace crc32c { 16 | 17 | extern bool IsFastCrc32Supported(); 18 | 19 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 20 | // crc32c of some string A. Extend() is often used to maintain the 21 | // crc32c of a stream of data. 22 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 23 | 24 | // Return the crc32c of data[0,n-1] 25 | inline uint32_t Value(const char* data, size_t n) { 26 | return Extend(0, data, n); 27 | } 28 | 29 | static const uint32_t kMaskDelta = 0xa282ead8ul; 30 | 31 | // Return a masked representation of crc. 32 | // 33 | // Motivation: it is problematic to compute the CRC of a string that 34 | // contains embedded CRCs. Therefore we recommend that CRCs stored 35 | // somewhere (e.g., in files) should be masked before being stored. 36 | inline uint32_t Mask(uint32_t crc) { 37 | // Rotate right by 15 bits and add a constant. 38 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 39 | } 40 | 41 | // Return the crc whose masked representation is masked_crc. 42 | inline uint32_t Unmask(uint32_t masked_crc) { 43 | uint32_t rot = masked_crc - kMaskDelta; 44 | return ((rot >> 17) | (rot << 15)); 45 | } 46 | 47 | } // namespace crc32c 48 | } // namespace rocksutil 49 | -------------------------------------------------------------------------------- /include/rocksutil/file_reader_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | #include 11 | #include "rocksutil/env.h" 12 | #include "rutil/aligned_buffer.h" 13 | #include "rport/port.h" 14 | 15 | namespace rocksutil { 16 | 17 | std::unique_ptr NewReadaheadRandomAccessFile( 18 | std::unique_ptr&& file, size_t readahead_size); 19 | 20 | class SequentialFileReader { 21 | private: 22 | std::unique_ptr file_; 23 | 24 | public: 25 | explicit SequentialFileReader(std::unique_ptr&& _file) 26 | : file_(std::move(_file)) {} 27 | 28 | SequentialFileReader(SequentialFileReader&& o) ROCKSUTIL_NOEXCEPT { 29 | *this = std::move(o); 30 | } 31 | 32 | SequentialFileReader& operator=(SequentialFileReader&& o) ROCKSUTIL_NOEXCEPT { 33 | file_ = std::move(o.file_); 34 | return *this; 35 | } 36 | 37 | SequentialFileReader(const SequentialFileReader&) = delete; 38 | SequentialFileReader& operator=(const SequentialFileReader&) = delete; 39 | 40 | Status Read(size_t n, Slice* result, char* scratch); 41 | 42 | Status Skip(uint64_t n); 43 | 44 | SequentialFile* file() { return file_.get(); } 45 | }; 46 | 47 | class RandomAccessFileReader { 48 | private: 49 | std::unique_ptr file_; 50 | 51 | public: 52 | explicit RandomAccessFileReader(std::unique_ptr&& raf) 53 | : file_(std::move(raf)) {} 54 | 55 | RandomAccessFileReader(RandomAccessFileReader&& o) ROCKSUTIL_NOEXCEPT { 56 | *this = std::move(o); 57 | } 58 | 59 | RandomAccessFileReader& operator=(RandomAccessFileReader&& o) ROCKSUTIL_NOEXCEPT{ 60 | file_ = std::move(o.file_); 61 | return *this; 62 | } 63 | 64 | RandomAccessFileReader(const RandomAccessFileReader&) = delete; 65 | RandomAccessFileReader& operator=(const RandomAccessFileReader&) = delete; 66 | 67 | Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const; 68 | 69 | RandomAccessFile* file() { return file_.get(); } 70 | }; 71 | 72 | // Use posix write to write data to a file. 73 | class WritableFileWriter { 74 | private: 75 | std::unique_ptr writable_file_; 76 | AlignedBuffer buf_; 77 | size_t max_buffer_size_; 78 | // Actually written data size can be used for truncate 79 | // not counting padding data 80 | uint64_t filesize_; 81 | // This is necessary when we use unbuffered access 82 | // and writes must happen on aligned offsets 83 | // so we need to go back and write that page again 84 | uint64_t next_write_offset_; 85 | bool pending_sync_; 86 | const bool direct_io_; 87 | const bool use_os_buffer_; 88 | uint64_t last_sync_size_; 89 | uint64_t bytes_per_sync_; 90 | 91 | public: 92 | WritableFileWriter(std::unique_ptr&& file, 93 | const EnvOptions& options) 94 | : writable_file_(std::move(file)), 95 | buf_(), 96 | max_buffer_size_(options.writable_file_max_buffer_size), 97 | filesize_(0), 98 | next_write_offset_(0), 99 | pending_sync_(false), 100 | direct_io_(writable_file_->UseDirectIO()), 101 | use_os_buffer_(writable_file_->UseOSBuffer()), 102 | last_sync_size_(0), 103 | bytes_per_sync_(options.bytes_per_sync) { 104 | buf_.Alignment(writable_file_->GetRequiredBufferAlignment()); 105 | buf_.AllocateNewBuffer(65536); 106 | } 107 | 108 | WritableFileWriter(const WritableFileWriter&) = delete; 109 | 110 | WritableFileWriter& operator=(const WritableFileWriter&) = delete; 111 | 112 | ~WritableFileWriter() { Close(); } 113 | 114 | Status Append(const Slice& data); 115 | 116 | Status Flush(); 117 | 118 | Status Close(); 119 | 120 | Status Sync(bool use_fsync); 121 | 122 | // Sync only the data that was already Flush()ed. Safe to call concurrently 123 | // with Append() and Flush(). If !writable_file_->IsSyncThreadSafe(), 124 | // returns NotSupported status. 125 | Status SyncWithoutFlush(bool use_fsync); 126 | 127 | uint64_t GetFileSize() { return filesize_; } 128 | 129 | Status InvalidateCache(size_t offset, size_t length) { 130 | return writable_file_->InvalidateCache(offset, length); 131 | } 132 | 133 | WritableFile* writable_file() const { return writable_file_.get(); } 134 | 135 | private: 136 | // Used when os buffering is OFF and we are writing 137 | // DMA such as in Windows unbuffered mode 138 | Status WriteUnbuffered(); 139 | // Normal write 140 | Status WriteBuffered(const char* data, size_t size); 141 | Status RangeSync(uint64_t offset, uint64_t nbytes); 142 | Status SyncInternal(bool use_fsync); 143 | }; 144 | 145 | extern Status NewWritableFile(Env* env, const std::string& fname, 146 | unique_ptr* result, 147 | const EnvOptions& options); 148 | extern Status NewRandomAccessFile(Env* env, const std::string& fname, 149 | unique_ptr* result, 150 | const EnvOptions& options); 151 | extern Status NewSequentialFile(Env* env, const std::string& fname, 152 | unique_ptr* result, 153 | const EnvOptions& options); 154 | } // namespace rocksutil 155 | -------------------------------------------------------------------------------- /include/rocksutil/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // Simple hash function used for internal data structures 11 | 12 | #pragma once 13 | #include 14 | #include 15 | 16 | #include "rocksutil/slice.h" 17 | 18 | namespace rocksutil { 19 | 20 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 21 | 22 | inline uint32_t BloomHash(const Slice& key) { 23 | return Hash(key.data(), key.size(), 0xbc9f1d34); 24 | } 25 | 26 | inline uint32_t GetSliceHash(const Slice& s) { 27 | return Hash(s.data(), s.size(), 397); 28 | } 29 | 30 | // std::hash compatible interface. 31 | struct SliceHasher { 32 | uint32_t operator()(const Slice& s) const { return GetSliceHash(s); } 33 | }; 34 | 35 | } // namespace rocksutil 36 | -------------------------------------------------------------------------------- /include/rocksutil/log_reader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | #include 12 | #include 13 | 14 | #include "rutil/log_format.h" 15 | #include "rocksutil/slice.h" 16 | #include "rocksutil/status.h" 17 | 18 | namespace rocksutil { 19 | 20 | class SequentialFileReader; 21 | class Logger; 22 | using std::unique_ptr; 23 | 24 | namespace log { 25 | 26 | enum class WALRecoveryMode : char { 27 | // Original levelDB recovery 28 | // We tolerate incomplete record in trailing data on all logs 29 | // Use case : This is legacy behavior (default) 30 | kTolerateCorruptedTailRecords = 0x00, 31 | // Recover from clean shutdown 32 | // We don't expect to find any corruption in the WAL 33 | // Use case : This is ideal for unit tests and rare applications that 34 | // can require high consistency guarantee 35 | kAbsoluteConsistency = 0x01, 36 | // Recover to point-in-time consistency 37 | // We stop the WAL playback on discovering WAL inconsistency 38 | // Use case : Ideal for systems that have disk controller cache like 39 | // hard disk, SSD without super capacitor that store related data 40 | kPointInTimeRecovery = 0x02, 41 | // Recovery after a disaster 42 | // We ignore any corruption in the WAL and try to salvage as much data as 43 | // possible 44 | // Use case : Ideal for last ditch effort to recover data or systems that 45 | // operate with low grade unrelated data 46 | kSkipAnyCorruptedRecords = 0x03, 47 | }; 48 | 49 | /** 50 | * Reader is a general purpose log stream reader implementation. The actual job 51 | * of reading from the device is implemented by the SequentialFile interface. 52 | * 53 | * Please see Writer for details on the file and record layout. 54 | */ 55 | class Reader { 56 | public: 57 | // Interface for reporting errors. 58 | class Reporter { 59 | public: 60 | virtual ~Reporter(); 61 | 62 | // Some corruption was detected. "size" is the approximate number 63 | // of bytes dropped due to the corruption. 64 | virtual void Corruption(size_t bytes, const Status& status) = 0; 65 | }; 66 | 67 | struct LogReporter : Reporter { 68 | Status* status; 69 | virtual void Corruption(size_t bytes, const Status& s) override { 70 | if (this->status->ok()) *this->status = s; 71 | } 72 | }; 73 | 74 | // Create a reader that will return log records from "*file". 75 | // "*file" must remain live while this Reader is in use. 76 | // 77 | // If "reporter" is non-nullptr, it is notified whenever some data is 78 | // dropped due to a detected corruption. "*reporter" must remain 79 | // live while this Reader is in use. 80 | // 81 | // If "checksum" is true, verify checksums if available. 82 | // 83 | // The Reader will start reading at the first record located at physical 84 | // position >= initial_offset within the file. 85 | Reader(unique_ptr&& file, 86 | Reporter* reporter, bool checksum, uint64_t initial_offset, 87 | uint64_t log_num = 0); 88 | 89 | ~Reader(); 90 | 91 | // Read the next record into *record. Returns true if read 92 | // successfully, false if we hit end of the input. May use 93 | // "*scratch" as temporary storage. The contents filled in *record 94 | // will only be valid until the next mutating operation on this 95 | // reader or the next mutation to *scratch. 96 | bool ReadRecord(Slice* record, std::string* scratch, 97 | WALRecoveryMode wal_recovery_mode = 98 | WALRecoveryMode::kTolerateCorruptedTailRecords); 99 | 100 | // Returns the physical offset of the last record returned by ReadRecord. 101 | // 102 | // Undefined before the first call to ReadRecord. 103 | uint64_t LastRecordOffset(); 104 | 105 | uint64_t EndOfBufferOffset(); 106 | 107 | // returns true if the reader has encountered an eof condition. 108 | bool IsEOF() { 109 | return eof_; 110 | } 111 | 112 | // when we know more data has been written to the file. we can use this 113 | // function to force the reader to look again in the file. 114 | // Also aligns the file position indicator to the start of the next block 115 | // by reading the rest of the data from the EOF position to the end of the 116 | // block that was partially read. 117 | void UnmarkEOF(); 118 | 119 | SequentialFileReader* file() { return file_.get(); } 120 | 121 | private: 122 | const unique_ptr file_; 123 | Reporter* const reporter_; 124 | bool const checksum_; 125 | char* const backing_store_; 126 | Slice buffer_; 127 | bool eof_; // Last Read() indicated EOF by returning < kBlockSize 128 | bool read_error_; // Error occurred while reading from file 129 | 130 | // Offset of the file position indicator within the last block when an 131 | // EOF was detected. 132 | size_t eof_offset_; 133 | 134 | // Offset of the last record returned by ReadRecord. 135 | uint64_t last_record_offset_; 136 | // Offset of the first location past the end of buffer_. 137 | uint64_t end_of_buffer_offset_; 138 | 139 | // Offset at which to start looking for the first record to return 140 | uint64_t const initial_offset_; 141 | 142 | // which log number this is 143 | uint64_t const log_number_; 144 | 145 | // Whether this is a recycled log file 146 | bool recycled_; 147 | 148 | // Extend record types with the following special values 149 | enum { 150 | kEof = kMaxRecordType + 1, 151 | // Returned whenever we find an invalid physical record. 152 | // Currently there are three situations in which this happens: 153 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 154 | // * The record is a 0-length record (No drop is reported) 155 | // * The record is below constructor's initial_offset (No drop is reported) 156 | kBadRecord = kMaxRecordType + 2, 157 | // Returned when we fail to read a valid header. 158 | kBadHeader = kMaxRecordType + 3, 159 | // Returned when we read an old record from a previous user of the log. 160 | kOldRecord = kMaxRecordType + 4, 161 | // Returned when we get a bad record length 162 | kBadRecordLen = kMaxRecordType + 5, 163 | // Returned when we get a bad record checksum 164 | kBadRecordChecksum = kMaxRecordType + 6, 165 | }; 166 | 167 | // Skips all blocks that are completely before "initial_offset_". 168 | // 169 | // Returns true on success. Handles reporting. 170 | bool SkipToInitialBlock(); 171 | 172 | // Return type, or one of the preceding special values 173 | unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size); 174 | 175 | // Read some more 176 | bool ReadMore(size_t* drop_size, int *error); 177 | 178 | // Reports dropped bytes to the reporter. 179 | // buffer_ must be updated to remove the dropped bytes prior to invocation. 180 | void ReportCorruption(size_t bytes, const char* reason); 181 | void ReportDrop(size_t bytes, const Status& reason); 182 | 183 | // No copying allowed 184 | Reader(const Reader&); 185 | void operator=(const Reader&); 186 | }; 187 | 188 | } // namespace log 189 | } // namespace rocksutil 190 | -------------------------------------------------------------------------------- /include/rocksutil/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | 11 | #include 12 | 13 | #include 14 | 15 | #include "rutil/log_format.h" 16 | #include "rocksutil/slice.h" 17 | #include "rocksutil/status.h" 18 | 19 | namespace rocksutil { 20 | 21 | class WritableFileWriter; 22 | 23 | using std::unique_ptr; 24 | 25 | namespace log { 26 | 27 | /** 28 | * Writer is a general purpose log stream writer. It provides an append-only 29 | * abstraction for writing data. The details of the how the data is written is 30 | * handled by the WriteableFile sub-class implementation. 31 | * 32 | * File format: 33 | * 34 | * File is broken down into variable sized records. The format of each record 35 | * is described below. 36 | * +-----+-------------+--+----+----------+------+-- ... ----+ 37 | * File | r0 | r1 |P | r2 | r3 | r4 | | 38 | * +-----+-------------+--+----+----------+------+-- ... ----+ 39 | * <--- kBlockSize ------>|<-- kBlockSize ------>| 40 | * rn = variable size records 41 | * P = Padding 42 | * 43 | * Data is written out in kBlockSize chunks. If next record does not fit 44 | * into the space left, the leftover space will be padded with \0. 45 | * 46 | * Legacy record format: 47 | * 48 | * +---------+-----------+-----------+--- ... ---+ 49 | * |CRC (4B) | Size (2B) | Type (1B) | Payload | 50 | * +---------+-----------+-----------+--- ... ---+ 51 | * 52 | * CRC = 32bit hash computed over the payload using CRC 53 | * Size = Length of the payload data 54 | * Type = Type of record 55 | * (kZeroType, kFullType, kFirstType, kLastType, kMiddleType ) 56 | * The type is used to group a bunch of records together to represent 57 | * blocks that are larger than kBlockSize 58 | * Payload = Byte stream as long as specified by the payload size 59 | * 60 | * Recyclable record format: 61 | * 62 | * +---------+-----------+-----------+----------------+--- ... ---+ 63 | * |CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload | 64 | * +---------+-----------+-----------+----------------+--- ... ---+ 65 | * 66 | * Same as above, with the addition of 67 | * Log number = 32bit log file number, so that we can distinguish between 68 | * records written by the most recent log writer vs a previous one. 69 | */ 70 | class Writer { 71 | public: 72 | // Create a writer that will append data to "*dest". 73 | // "*dest" must be initially empty. 74 | // "*dest" must remain live while this Writer is in use. 75 | explicit Writer(unique_ptr&& dest, 76 | uint64_t log_number = 0, 77 | bool recycle_log_files = false); 78 | ~Writer(); 79 | 80 | Status AddRecord(const Slice& slice); 81 | 82 | WritableFileWriter* file() { return dest_.get(); } 83 | const WritableFileWriter* file() const { return dest_.get(); } 84 | 85 | uint64_t get_log_number() const { return log_number_; } 86 | 87 | private: 88 | unique_ptr dest_; 89 | size_t block_offset_; // Current offset in block 90 | uint64_t log_number_; 91 | bool recycle_log_files_; 92 | 93 | // crc32c values for all supported record types. These are 94 | // pre-computed to reduce the overhead of computing the crc of the 95 | // record type stored in the header. 96 | uint32_t type_crc_[kMaxRecordType + 1]; 97 | 98 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 99 | 100 | // No copying allowed 101 | Writer(const Writer&); 102 | void operator=(const Writer&); 103 | }; 104 | 105 | } // namespace log 106 | } // namespace rocksutil 107 | -------------------------------------------------------------------------------- /include/rocksutil/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "rport/port.h" 16 | 17 | namespace rocksutil { 18 | 19 | // Helper class that locks a mutex on construction and unlocks the mutex when 20 | // the destructor of the MutexLock object is invoked. 21 | // 22 | // Typical usage: 23 | // 24 | // void MyClass::MyMethod() { 25 | // MutexLock l(&mu_); // mu_ is an instance variable 26 | // ... some complex code, possibly with multiple return paths ... 27 | // } 28 | 29 | class MutexLock { 30 | public: 31 | explicit MutexLock(port::Mutex *mu) : mu_(mu) { 32 | this->mu_->Lock(); 33 | } 34 | ~MutexLock() { this->mu_->Unlock(); } 35 | 36 | private: 37 | port::Mutex *const mu_; 38 | // No copying allowed 39 | MutexLock(const MutexLock&); 40 | void operator=(const MutexLock&); 41 | }; 42 | 43 | // 44 | // Acquire a ReadLock on the specified RWMutex. 45 | // The Lock will be automatically released then the 46 | // object goes out of scope. 47 | // 48 | class ReadLock { 49 | public: 50 | explicit ReadLock(port::RWMutex *mu) : mu_(mu) { 51 | this->mu_->ReadLock(); 52 | } 53 | ~ReadLock() { this->mu_->ReadUnlock(); } 54 | 55 | private: 56 | port::RWMutex *const mu_; 57 | // No copying allowed 58 | ReadLock(const ReadLock&); 59 | void operator=(const ReadLock&); 60 | }; 61 | 62 | // 63 | // Automatically unlock a locked mutex when the object is destroyed 64 | // 65 | class ReadUnlock { 66 | public: 67 | explicit ReadUnlock(port::RWMutex *mu) : mu_(mu) { mu->AssertHeld(); } 68 | ~ReadUnlock() { mu_->ReadUnlock(); } 69 | 70 | private: 71 | port::RWMutex *const mu_; 72 | // No copying allowed 73 | ReadUnlock(const ReadUnlock &) = delete; 74 | ReadUnlock &operator=(const ReadUnlock &) = delete; 75 | }; 76 | 77 | // 78 | // Acquire a WriteLock on the specified RWMutex. 79 | // The Lock will be automatically released then the 80 | // object goes out of scope. 81 | // 82 | class WriteLock { 83 | public: 84 | explicit WriteLock(port::RWMutex *mu) : mu_(mu) { 85 | this->mu_->WriteLock(); 86 | } 87 | ~WriteLock() { this->mu_->WriteUnlock(); } 88 | 89 | private: 90 | port::RWMutex *const mu_; 91 | // No copying allowed 92 | WriteLock(const WriteLock&); 93 | void operator=(const WriteLock&); 94 | }; 95 | 96 | // 97 | // SpinMutex has very low overhead for low-contention cases. Method names 98 | // are chosen so you can use std::unique_lock or std::lock_guard with it. 99 | // 100 | class SpinMutex { 101 | public: 102 | SpinMutex() : locked_(false) {} 103 | 104 | bool try_lock() { 105 | auto currently_locked = locked_.load(std::memory_order_relaxed); 106 | return !currently_locked && 107 | locked_.compare_exchange_weak(currently_locked, true, 108 | std::memory_order_acquire, 109 | std::memory_order_relaxed); 110 | } 111 | 112 | void lock() { 113 | for (size_t tries = 0;; ++tries) { 114 | if (try_lock()) { 115 | // success 116 | break; 117 | } 118 | port::AsmVolatilePause(); 119 | if (tries > 100) { 120 | std::this_thread::yield(); 121 | } 122 | } 123 | } 124 | 125 | void unlock() { locked_.store(false, std::memory_order_release); } 126 | 127 | private: 128 | std::atomic locked_; 129 | }; 130 | 131 | } // namespace rocksutil 132 | -------------------------------------------------------------------------------- /include/rocksutil/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 6 | // Use of this source code is governed by a BSD-style license that can be 7 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 8 | // 9 | // Slice is a simple structure containing a pointer into some external 10 | // storage and a size. The user of a Slice must ensure that the slice 11 | // is not used after the corresponding external storage has been 12 | // deallocated. 13 | // 14 | // Multiple threads can invoke const methods on a Slice without 15 | // external synchronization, but if any of the threads may call a 16 | // non-const method, all threads accessing the same Slice must use 17 | // external synchronization. 18 | 19 | #ifndef STORAGE_ROCKSUTIL_INCLUDE_SLICE_H_ 20 | #define STORAGE_ROCKSUTIL_INCLUDE_SLICE_H_ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | namespace rocksutil { 29 | 30 | class Slice { 31 | public: 32 | // Create an empty slice. 33 | Slice() : data_(""), size_(0) { } 34 | 35 | // Create a slice that refers to d[0,n-1]. 36 | Slice(const char* d, size_t n) : data_(d), size_(n) { } 37 | 38 | // Create a slice that refers to the contents of "s" 39 | /* implicit */ 40 | Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } 41 | 42 | // Create a slice that refers to s[0,strlen(s)-1] 43 | /* implicit */ 44 | Slice(const char* s) : data_(s), size_(strlen(s)) { } 45 | 46 | // Create a single slice from SliceParts using buf as storage. 47 | // buf must exist as long as the returned Slice exists. 48 | Slice(const struct SliceParts& parts, std::string* buf); 49 | 50 | // Return a pointer to the beginning of the referenced data 51 | const char* data() const { return data_; } 52 | 53 | // Return the length (in bytes) of the referenced data 54 | size_t size() const { return size_; } 55 | 56 | // Return true iff the length of the referenced data is zero 57 | bool empty() const { return size_ == 0; } 58 | 59 | // Return the ith byte in the referenced data. 60 | // REQUIRES: n < size() 61 | char operator[](size_t n) const { 62 | assert(n < size()); 63 | return data_[n]; 64 | } 65 | 66 | // Change this slice to refer to an empty array 67 | void clear() { data_ = ""; size_ = 0; } 68 | 69 | // Drop the first "n" bytes from this slice. 70 | void remove_prefix(size_t n) { 71 | assert(n <= size()); 72 | data_ += n; 73 | size_ -= n; 74 | } 75 | 76 | void remove_suffix(size_t n) { 77 | assert(n <= size()); 78 | size_ -= n; 79 | } 80 | 81 | // Return a string that contains the copy of the referenced data. 82 | // when hex is true, returns a string of twice the length hex encoded (0-9A-F) 83 | std::string ToString(bool hex = false) const; 84 | 85 | // Decodes the current slice interpreted as an hexadecimal string into result, 86 | // if successful returns true, if this isn't a valid hex string 87 | // (e.g not coming from Slice::ToString(true)) DecodeHex returns false. 88 | // This slice is expected to have an even number of 0-9A-F characters 89 | // also accepts lowercase (a-f) 90 | bool DecodeHex(std::string* result) const; 91 | 92 | // Three-way comparison. Returns value: 93 | // < 0 iff "*this" < "b", 94 | // == 0 iff "*this" == "b", 95 | // > 0 iff "*this" > "b" 96 | int compare(const Slice& b) const; 97 | 98 | // Return true iff "x" is a prefix of "*this" 99 | bool starts_with(const Slice& x) const { 100 | return ((size_ >= x.size_) && 101 | (memcmp(data_, x.data_, x.size_) == 0)); 102 | } 103 | 104 | bool ends_with(const Slice& x) const { 105 | return ((size_ >= x.size_) && 106 | (memcmp(data_ + size_ - x.size_, x.data_, x.size_) == 0)); 107 | } 108 | 109 | // Compare two slices and returns the first byte where they differ 110 | size_t difference_offset(const Slice& b) const; 111 | 112 | // private: make these public for rocksutiljni access 113 | const char* data_; 114 | size_t size_; 115 | 116 | // Intentionally copyable 117 | }; 118 | 119 | // A set of Slices that are virtually concatenated together. 'parts' points 120 | // to an array of Slices. The number of elements in the array is 'num_parts'. 121 | struct SliceParts { 122 | SliceParts(const Slice* _parts, int _num_parts) : 123 | parts(_parts), num_parts(_num_parts) { } 124 | SliceParts() : parts(nullptr), num_parts(0) {} 125 | 126 | const Slice* parts; 127 | int num_parts; 128 | }; 129 | 130 | inline bool operator==(const Slice& x, const Slice& y) { 131 | return ((x.size() == y.size()) && 132 | (memcmp(x.data(), y.data(), x.size()) == 0)); 133 | } 134 | 135 | inline bool operator!=(const Slice& x, const Slice& y) { 136 | return !(x == y); 137 | } 138 | 139 | inline int Slice::compare(const Slice& b) const { 140 | const size_t min_len = (size_ < b.size_) ? size_ : b.size_; 141 | int r = memcmp(data_, b.data_, min_len); 142 | if (r == 0) { 143 | if (size_ < b.size_) r = -1; 144 | else if (size_ > b.size_) r = +1; 145 | } 146 | return r; 147 | } 148 | 149 | inline size_t Slice::difference_offset(const Slice& b) const { 150 | size_t off = 0; 151 | const size_t len = (size_ < b.size_) ? size_ : b.size_; 152 | for (; off < len; off++) { 153 | if (data_[off] != b.data_[off]) break; 154 | } 155 | return off; 156 | } 157 | 158 | } // namespace rocksutil 159 | 160 | #endif // STORAGE_ROCKSUTIL_INCLUDE_SLICE_H_ 161 | -------------------------------------------------------------------------------- /include/rocksutil/thread_local.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "rutil/autovector.h" 18 | #include "rport/port.h" 19 | 20 | #ifndef ROCKSUTIL_SUPPORT_THREAD_LOCAL 21 | #define ROCKSUTIL_SUPPORT_THREAD_LOCAL \ 22 | !defined(OS_WIN) && !defined(OS_MACOSX) && !defined(IOS_CROSS_COMPILE) 23 | #endif 24 | 25 | namespace rocksutil { 26 | 27 | // Cleanup function that will be called for a stored thread local 28 | // pointer (if not NULL) when one of the following happens: 29 | // (1) a thread terminates 30 | // (2) a ThreadLocalPtr is destroyed 31 | typedef void (*UnrefHandler)(void* ptr); 32 | 33 | // ThreadLocalPtr stores only values of pointer type. Different from 34 | // the usual thread-local-storage, ThreadLocalPtr has the ability to 35 | // distinguish data coming from different threads and different 36 | // ThreadLocalPtr instances. For example, if a regular thread_local 37 | // variable A is declared in DBImpl, two DBImpl objects would share 38 | // the same A. However, a ThreadLocalPtr that is defined under the 39 | // scope of DBImpl can avoid such confliction. As a result, its memory 40 | // usage would be O(# of threads * # of ThreadLocalPtr instances). 41 | class ThreadLocalPtr { 42 | public: 43 | explicit ThreadLocalPtr(UnrefHandler handler = nullptr); 44 | 45 | ~ThreadLocalPtr(); 46 | 47 | // Return the current pointer stored in thread local 48 | void* Get() const; 49 | 50 | // Set a new pointer value to the thread local storage. 51 | void Reset(void* ptr); 52 | 53 | // Atomically swap the supplied ptr and return the previous value 54 | void* Swap(void* ptr); 55 | 56 | // Atomically compare the stored value with expected. Set the new 57 | // pointer value to thread local only if the comparison is true. 58 | // Otherwise, expected returns the stored value. 59 | // Return true on success, false on failure 60 | bool CompareAndSwap(void* ptr, void*& expected); 61 | 62 | // Reset all thread local data to replacement, and return non-nullptr 63 | // data for all existing threads 64 | void Scrape(autovector* ptrs, void* const replacement); 65 | 66 | typedef std::function FoldFunc; 67 | // Update res by applying func on each thread-local value. Holds a lock that 68 | // prevents unref handler from running during this call, but clients must 69 | // still provide external synchronization since the owning thread can 70 | // access the values without internal locking, e.g., via Get() and Reset(). 71 | void Fold(FoldFunc func, void* res); 72 | 73 | // Initialize the static singletons of the ThreadLocalPtr. 74 | // 75 | // If this function is not called, then the singletons will be 76 | // automatically initialized when they are used. 77 | // 78 | // Calling this function twice or after the singletons have been 79 | // initialized will be no-op. 80 | static void InitSingletons(); 81 | 82 | protected: 83 | struct Entry { 84 | Entry() : ptr(nullptr) {} 85 | Entry(const Entry& e) : ptr(e.ptr.load(std::memory_order_relaxed)) {} 86 | std::atomic ptr; 87 | }; 88 | 89 | class StaticMeta; 90 | 91 | // This is the structure that is declared as "thread_local" storage. 92 | // The vector keep list of atomic pointer for all instances for "current" 93 | // thread. The vector is indexed by an Id that is unique in process and 94 | // associated with one ThreadLocalPtr instance. The Id is assigned by a 95 | // global StaticMeta singleton. So if we instantiated 3 ThreadLocalPtr 96 | // instances, each thread will have a ThreadData with a vector of size 3: 97 | // --------------------------------------------------- 98 | // | | instance 1 | instance 2 | instnace 3 | 99 | // --------------------------------------------------- 100 | // | thread 1 | void* | void* | void* | <- ThreadData 101 | // --------------------------------------------------- 102 | // | thread 2 | void* | void* | void* | <- ThreadData 103 | // --------------------------------------------------- 104 | // | thread 3 | void* | void* | void* | <- ThreadData 105 | // --------------------------------------------------- 106 | struct ThreadData { 107 | explicit ThreadData(StaticMeta* _inst) : entries(), inst(_inst) {} 108 | std::vector entries; 109 | ThreadData* next; 110 | ThreadData* prev; 111 | StaticMeta* inst; 112 | }; 113 | 114 | class StaticMeta { 115 | public: 116 | StaticMeta(); 117 | 118 | // Return the next available Id 119 | uint32_t GetId(); 120 | // Return the next available Id without claiming it 121 | uint32_t PeekId() const; 122 | // Return the given Id back to the free pool. This also triggers 123 | // UnrefHandler for associated pointer value (if not NULL) for all threads. 124 | void ReclaimId(uint32_t id); 125 | 126 | // Return the pointer value for the given id for the current thread. 127 | void* Get(uint32_t id) const; 128 | // Reset the pointer value for the given id for the current thread. 129 | void Reset(uint32_t id, void* ptr); 130 | // Atomically swap the supplied ptr and return the previous value 131 | void* Swap(uint32_t id, void* ptr); 132 | // Atomically compare and swap the provided value only if it equals 133 | // to expected value. 134 | bool CompareAndSwap(uint32_t id, void* ptr, void*& expected); 135 | // Reset all thread local data to replacement, and return non-nullptr 136 | // data for all existing threads 137 | void Scrape(uint32_t id, autovector* ptrs, void* const replacement); 138 | // Update res by applying func on each thread-local value. Holds a lock that 139 | // prevents unref handler from running during this call, but clients must 140 | // still provide external synchronization since the owning thread can 141 | // access the values without internal locking, e.g., via Get() and Reset(). 142 | void Fold(uint32_t id, FoldFunc func, void* res); 143 | 144 | // Register the UnrefHandler for id 145 | void SetHandler(uint32_t id, UnrefHandler handler); 146 | 147 | // protect inst, next_instance_id_, free_instance_ids_, head_, 148 | // ThreadData.entries 149 | // 150 | // Note that here we prefer function static variable instead of the usual 151 | // global static variable. The reason is that c++ destruction order of 152 | // static variables in the reverse order of their construction order. 153 | // However, C++ does not guarantee any construction order when global 154 | // static variables are defined in different files, while the function 155 | // static variables are initialized when their function are first called. 156 | // As a result, the construction order of the function static variables 157 | // can be controlled by properly invoke their first function calls in 158 | // the right order. 159 | // 160 | // For instance, the following function contains a function static 161 | // variable. We place a dummy function call of this inside 162 | // Env::Default() to ensure the construction order of the construction 163 | // order. 164 | static port::Mutex* Mutex(); 165 | 166 | // Returns the member mutex of the current StaticMeta. In general, 167 | // Mutex() should be used instead of this one. However, in case where 168 | // the static variable inside Instance() goes out of scope, MemberMutex() 169 | // should be used. One example is OnThreadExit() function. 170 | port::Mutex* MemberMutex() { return &mutex_; } 171 | 172 | private: 173 | // Get UnrefHandler for id with acquiring mutex 174 | // REQUIRES: mutex locked 175 | UnrefHandler GetHandler(uint32_t id); 176 | 177 | // Triggered before a thread terminates 178 | static void OnThreadExit(void* ptr); 179 | 180 | // Add current thread's ThreadData to the global chain 181 | // REQUIRES: mutex locked 182 | void AddThreadData(ThreadData* d); 183 | 184 | // Remove current thread's ThreadData from the global chain 185 | // REQUIRES: mutex locked 186 | void RemoveThreadData(ThreadData* d); 187 | 188 | static ThreadData* GetThreadLocal(); 189 | 190 | uint32_t next_instance_id_; 191 | // Used to recycle Ids in case ThreadLocalPtr is instantiated and destroyed 192 | // frequently. This also prevents it from blowing up the vector space. 193 | autovector free_instance_ids_; 194 | // Chain all thread local structure together. This is necessary since 195 | // when one ThreadLocalPtr gets destroyed, we need to loop over each 196 | // thread's version of pointer corresponding to that instance and 197 | // call UnrefHandler for it. 198 | ThreadData head_; 199 | 200 | std::unordered_map handler_map_; 201 | 202 | // The private mutex. Developers should always use Mutex() instead of 203 | // using this variable directly. 204 | port::Mutex mutex_; 205 | #if ROCKSUTIL_SUPPORT_THREAD_LOCAL 206 | // Thread local storage 207 | static __thread ThreadData* tls_; 208 | #endif 209 | 210 | // Used to make thread exit trigger possible if !defined(OS_MACOSX). 211 | // Otherwise, used to retrieve thread data. 212 | pthread_key_t pthread_key_; 213 | }; 214 | 215 | static StaticMeta* Instance(); 216 | 217 | const uint32_t id_; 218 | }; 219 | 220 | } // namespace rocksutil 221 | -------------------------------------------------------------------------------- /include/rocksutil/version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | #pragma once 6 | 7 | #define ROCKSUTIL_MAJOR 5 8 | #define ROCKSUTIL_MINOR 0 9 | #define ROCKSUTIL_PATCH 1 10 | -------------------------------------------------------------------------------- /rport/likely.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #ifndef PORT_LIKELY_H_ 11 | #define PORT_LIKELY_H_ 12 | 13 | #if defined(__GNUC__) && __GNUC__ >= 4 14 | #define LIKELY(x) (__builtin_expect((x), 1)) 15 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 16 | #else 17 | #define LIKELY(x) (x) 18 | #define UNLIKELY(x) (x) 19 | #endif 20 | 21 | #endif // PORT_LIKELY_H_ 22 | -------------------------------------------------------------------------------- /rport/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | 12 | #include 13 | #include "rport/port_posix.h" 14 | -------------------------------------------------------------------------------- /rport/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rport/port_posix.h" 11 | 12 | #include 13 | #if defined(__i386__) || defined(__x86_64__) 14 | #include 15 | #endif 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace rocksutil { 26 | namespace port { 27 | 28 | static int PthreadCall(const char* label, int result) { 29 | if (result != 0 && result != ETIMEDOUT) { 30 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 31 | abort(); 32 | } 33 | return result; 34 | } 35 | 36 | Mutex::Mutex(bool adaptive) { 37 | #ifdef ROCKSUTIL_PTHREAD_ADAPTIVE_MUTEX 38 | if (!adaptive) { 39 | PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); 40 | } else { 41 | pthread_mutexattr_t mutex_attr; 42 | PthreadCall("init mutex attr", pthread_mutexattr_init(&mutex_attr)); 43 | PthreadCall("set mutex attr", 44 | pthread_mutexattr_settype(&mutex_attr, 45 | PTHREAD_MUTEX_ADAPTIVE_NP)); 46 | PthreadCall("init mutex", pthread_mutex_init(&mu_, &mutex_attr)); 47 | PthreadCall("destroy mutex attr", 48 | pthread_mutexattr_destroy(&mutex_attr)); 49 | } 50 | #else 51 | PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); 52 | #endif // ROCKSUTIL_PTHREAD_ADAPTIVE_MUTEX 53 | } 54 | 55 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 56 | 57 | void Mutex::Lock() { 58 | PthreadCall("lock", pthread_mutex_lock(&mu_)); 59 | #ifndef NDEBUG 60 | locked_ = true; 61 | #endif 62 | } 63 | 64 | void Mutex::Unlock() { 65 | #ifndef NDEBUG 66 | locked_ = false; 67 | #endif 68 | PthreadCall("unlock", pthread_mutex_unlock(&mu_)); 69 | } 70 | 71 | void Mutex::AssertHeld() { 72 | #ifndef NDEBUG 73 | assert(locked_); 74 | #endif 75 | } 76 | 77 | CondVar::CondVar(Mutex* mu) 78 | : mu_(mu) { 79 | PthreadCall("init cv", pthread_cond_init(&cv_, nullptr)); 80 | } 81 | 82 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 83 | 84 | void CondVar::Wait() { 85 | #ifndef NDEBUG 86 | mu_->locked_ = false; 87 | #endif 88 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 89 | #ifndef NDEBUG 90 | mu_->locked_ = true; 91 | #endif 92 | } 93 | 94 | bool CondVar::TimedWait(uint64_t abs_time_us) { 95 | struct timespec ts; 96 | ts.tv_sec = static_cast(abs_time_us / 1000000); 97 | ts.tv_nsec = static_cast((abs_time_us % 1000000) * 1000); 98 | 99 | #ifndef NDEBUG 100 | mu_->locked_ = false; 101 | #endif 102 | int err = pthread_cond_timedwait(&cv_, &mu_->mu_, &ts); 103 | #ifndef NDEBUG 104 | mu_->locked_ = true; 105 | #endif 106 | if (err == ETIMEDOUT) { 107 | return true; 108 | } 109 | if (err != 0) { 110 | PthreadCall("timedwait", err); 111 | } 112 | return false; 113 | } 114 | 115 | void CondVar::Signal() { 116 | PthreadCall("signal", pthread_cond_signal(&cv_)); 117 | } 118 | 119 | void CondVar::SignalAll() { 120 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 121 | } 122 | 123 | RWMutex::RWMutex() { 124 | PthreadCall("init mutex", pthread_rwlock_init(&mu_, nullptr)); 125 | } 126 | 127 | RWMutex::~RWMutex() { PthreadCall("destroy mutex", pthread_rwlock_destroy(&mu_)); } 128 | 129 | void RWMutex::ReadLock() { PthreadCall("read lock", pthread_rwlock_rdlock(&mu_)); } 130 | 131 | void RWMutex::WriteLock() { PthreadCall("write lock", pthread_rwlock_wrlock(&mu_)); } 132 | 133 | void RWMutex::ReadUnlock() { PthreadCall("read unlock", pthread_rwlock_unlock(&mu_)); } 134 | 135 | void RWMutex::WriteUnlock() { PthreadCall("write unlock", pthread_rwlock_unlock(&mu_)); } 136 | 137 | int PhysicalCoreID() { 138 | #if defined(__i386__) || defined(__x86_64__) 139 | // if you ever find that this function is hot on Linux, you can go from 140 | // ~200 nanos to ~20 nanos by adding the machinery to use __vdso_getcpu 141 | unsigned eax, ebx = 0, ecx, edx; 142 | __get_cpuid(1, &eax, &ebx, &ecx, &edx); 143 | return ebx >> 24; 144 | #else 145 | // getcpu or sched_getcpu could work here 146 | return -1; 147 | #endif 148 | } 149 | 150 | void InitOnce(OnceType* once, void (*initializer)()) { 151 | PthreadCall("once", pthread_once(once, initializer)); 152 | } 153 | 154 | void Crash(const std::string& srcfile, int srcline) { 155 | fprintf(stdout, "Crashing at %s:%d\n", srcfile.c_str(), srcline); 156 | fflush(stdout); 157 | kill(getpid(), SIGTERM); 158 | } 159 | 160 | int GetMaxOpenFiles() { 161 | #if defined(RLIMIT_NOFILE) 162 | struct rlimit no_files_limit; 163 | if (getrlimit(RLIMIT_NOFILE, &no_files_limit) != 0) { 164 | return -1; 165 | } 166 | // protect against overflow 167 | if (no_files_limit.rlim_cur >= std::numeric_limits::max()) { 168 | return std::numeric_limits::max(); 169 | } 170 | return static_cast(no_files_limit.rlim_cur); 171 | #endif 172 | return -1; 173 | } 174 | 175 | } // namespace port 176 | } // namespace rocksutil 177 | -------------------------------------------------------------------------------- /rport/port_posix.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // See port_example.h for documentation for the following types/functions. 11 | 12 | #pragma once 13 | 14 | // size_t printf formatting named in the manner of C99 standard formatting 15 | // strings such as PRIu64 16 | // in fact, we could use that one 17 | #define ROCKSUTIL_PRIszt "zu" 18 | 19 | #define __declspec(S) 20 | 21 | #define ROCKSUTIL_NOEXCEPT noexcept 22 | 23 | #undef PLATFORM_IS_LITTLE_ENDIAN 24 | #if defined(OS_MACOSX) 25 | #include 26 | #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) 27 | #define PLATFORM_IS_LITTLE_ENDIAN \ 28 | (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) 29 | #endif 30 | #elif defined(OS_SOLARIS) 31 | #include 32 | #ifdef _LITTLE_ENDIAN 33 | #define PLATFORM_IS_LITTLE_ENDIAN true 34 | #else 35 | #define PLATFORM_IS_LITTLE_ENDIAN false 36 | #endif 37 | #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \ 38 | defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) 39 | #include 40 | #include 41 | #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) 42 | #else 43 | #include 44 | #endif 45 | #include 46 | 47 | #include 48 | #include 49 | #include 50 | #include 51 | 52 | #ifndef PLATFORM_IS_LITTLE_ENDIAN 53 | #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) 54 | #endif 55 | 56 | #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ 57 | defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ 58 | defined(OS_ANDROID) || defined(CYGWIN) 59 | // Use fread/fwrite/fflush on platforms without _unlocked variants 60 | #define fread_unlocked fread 61 | #define fwrite_unlocked fwrite 62 | #define fflush_unlocked fflush 63 | #endif 64 | 65 | #if defined(OS_MACOSX) || defined(OS_FREEBSD) ||\ 66 | defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) 67 | // Use fsync() on platforms without fdatasync() 68 | #define fdatasync fsync 69 | #endif 70 | 71 | #if defined(OS_ANDROID) && __ANDROID_API__ < 9 72 | // fdatasync() was only introduced in API level 9 on Android. Use fsync() 73 | // when targeting older platforms. 74 | #define fdatasync fsync 75 | #endif 76 | 77 | namespace rocksutil { 78 | namespace port { 79 | 80 | // For use at db/file_indexer.h kLevelMaxIndex 81 | const int kMaxInt32 = std::numeric_limits::max(); 82 | const uint64_t kMaxUint64 = std::numeric_limits::max(); 83 | const int64_t kMaxInt64 = std::numeric_limits::max(); 84 | const size_t kMaxSizet = std::numeric_limits::max(); 85 | 86 | static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; 87 | #undef PLATFORM_IS_LITTLE_ENDIAN 88 | 89 | class CondVar; 90 | 91 | class Mutex { 92 | public: 93 | /* implicit */ Mutex(bool adaptive = false); 94 | ~Mutex(); 95 | 96 | void Lock(); 97 | void Unlock(); 98 | // this will assert if the mutex is not locked 99 | // it does NOT verify that mutex is held by a calling thread 100 | void AssertHeld(); 101 | 102 | private: 103 | friend class CondVar; 104 | pthread_mutex_t mu_; 105 | #ifndef NDEBUG 106 | bool locked_; 107 | #endif 108 | 109 | // No copying 110 | Mutex(const Mutex&); 111 | void operator=(const Mutex&); 112 | }; 113 | 114 | class RWMutex { 115 | public: 116 | RWMutex(); 117 | ~RWMutex(); 118 | 119 | void ReadLock(); 120 | void WriteLock(); 121 | void ReadUnlock(); 122 | void WriteUnlock(); 123 | void AssertHeld() { } 124 | 125 | private: 126 | pthread_rwlock_t mu_; // the underlying platform mutex 127 | 128 | // No copying allowed 129 | RWMutex(const RWMutex&); 130 | void operator=(const RWMutex&); 131 | }; 132 | 133 | class CondVar { 134 | public: 135 | explicit CondVar(Mutex* mu); 136 | ~CondVar(); 137 | void Wait(); 138 | // Timed condition wait. Returns true if timeout occurred. 139 | bool TimedWait(uint64_t abs_time_us); 140 | void Signal(); 141 | void SignalAll(); 142 | private: 143 | pthread_cond_t cv_; 144 | Mutex* mu_; 145 | }; 146 | 147 | static inline void AsmVolatilePause() { 148 | #if defined(__i386__) || defined(__x86_64__) 149 | asm volatile("pause"); 150 | #elif defined(__aarch64__) 151 | asm volatile("wfe"); 152 | #elif defined(__powerpc64__) 153 | asm volatile("or 27,27,27"); 154 | #endif 155 | // it's okay for other platforms to be no-ops 156 | } 157 | 158 | // Returns -1 if not available on this platform 159 | extern int PhysicalCoreID(); 160 | 161 | typedef pthread_once_t OnceType; 162 | #define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT 163 | extern void InitOnce(OnceType* once, void (*initializer)()); 164 | 165 | #define CACHE_LINE_SIZE 64U 166 | 167 | #define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality) 168 | 169 | extern void Crash(const std::string& srcfile, int srcline); 170 | 171 | extern int GetMaxOpenFiles(); 172 | 173 | } // namespace port 174 | } // namespace rocksutil 175 | -------------------------------------------------------------------------------- /rport/sys_time.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | // This file is a portable substitute for sys/time.h which does not exist on 11 | // Windows 12 | 13 | #ifndef STORAGE_LEVELDB_PORT_SYS_TIME_H_ 14 | #define STORAGE_LEVELDB_PORT_SYS_TIME_H_ 15 | 16 | #include 17 | #include 18 | 19 | #endif // STORAGE_LEVELDB_PORT_SYS_TIME_H_ 20 | -------------------------------------------------------------------------------- /rport/util_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #ifndef STORAGE_LEVELDB_PORT_UTIL_LOGGER_H_ 11 | #define STORAGE_LEVELDB_PORT_UTIL_LOGGER_H_ 12 | 13 | // Include the appropriate platform specific file below. If you are 14 | // porting to a new platform, see "port_example.h" for documentation 15 | // of what the new port_.h file must provide. 16 | 17 | #include "rutil/posix_logger.h" 18 | 19 | #endif // STORAGE_LEVELDB_PORT_UTIL_LOGGER_H_ 20 | -------------------------------------------------------------------------------- /rutil/aligned_buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | 11 | #include 12 | #include "rport/port.h" 13 | 14 | namespace rocksutil { 15 | 16 | inline size_t TruncateToPageBoundary(size_t page_size, size_t s) { 17 | s -= (s & (page_size - 1)); 18 | assert((s % page_size) == 0); 19 | return s; 20 | } 21 | 22 | inline size_t Roundup(size_t x, size_t y) { 23 | return ((x + y - 1) / y) * y; 24 | } 25 | 26 | // This class is to manage an aligned user 27 | // allocated buffer for unbuffered I/O purposes 28 | // though can be used for any purpose. 29 | class AlignedBuffer { 30 | size_t alignment_; 31 | std::unique_ptr buf_; 32 | size_t capacity_; 33 | size_t cursize_; 34 | char* bufstart_; 35 | 36 | public: 37 | AlignedBuffer() 38 | : alignment_(), 39 | capacity_(0), 40 | cursize_(0), 41 | bufstart_(nullptr) { 42 | } 43 | 44 | AlignedBuffer(AlignedBuffer&& o) ROCKSUTIL_NOEXCEPT { 45 | *this = std::move(o); 46 | } 47 | 48 | AlignedBuffer& operator=(AlignedBuffer&& o) ROCKSUTIL_NOEXCEPT { 49 | alignment_ = std::move(o.alignment_); 50 | buf_ = std::move(o.buf_); 51 | capacity_ = std::move(o.capacity_); 52 | cursize_ = std::move(o.cursize_); 53 | bufstart_ = std::move(o.bufstart_); 54 | return *this; 55 | } 56 | 57 | AlignedBuffer(const AlignedBuffer&) = delete; 58 | 59 | AlignedBuffer& operator=(const AlignedBuffer&) = delete; 60 | 61 | size_t Alignment() const { 62 | return alignment_; 63 | } 64 | 65 | size_t Capacity() const { 66 | return capacity_; 67 | } 68 | 69 | size_t CurrentSize() const { 70 | return cursize_; 71 | } 72 | 73 | const char* BufferStart() const { 74 | return bufstart_; 75 | } 76 | 77 | void Clear() { 78 | cursize_ = 0; 79 | } 80 | 81 | void Alignment(size_t alignment) { 82 | assert(alignment > 0); 83 | assert((alignment & (alignment - 1)) == 0); 84 | alignment_ = alignment; 85 | } 86 | 87 | // Allocates a new buffer and sets bufstart_ to the aligned first byte 88 | void AllocateNewBuffer(size_t requestedCapacity) { 89 | 90 | assert(alignment_ > 0); 91 | assert((alignment_ & (alignment_ - 1)) == 0); 92 | 93 | size_t size = Roundup(requestedCapacity, alignment_); 94 | buf_.reset(new char[size + alignment_]); 95 | 96 | char* p = buf_.get(); 97 | bufstart_ = reinterpret_cast( 98 | (reinterpret_cast(p)+(alignment_ - 1)) & 99 | ~static_cast(alignment_ - 1)); 100 | capacity_ = size; 101 | cursize_ = 0; 102 | } 103 | // Used for write 104 | // Returns the number of bytes appended 105 | size_t Append(const char* src, size_t append_size) { 106 | size_t buffer_remaining = capacity_ - cursize_; 107 | size_t to_copy = std::min(append_size, buffer_remaining); 108 | 109 | if (to_copy > 0) { 110 | memcpy(bufstart_ + cursize_, src, to_copy); 111 | cursize_ += to_copy; 112 | } 113 | return to_copy; 114 | } 115 | 116 | size_t Read(char* dest, size_t offset, size_t read_size) const { 117 | assert(offset < cursize_); 118 | size_t to_read = std::min(cursize_ - offset, read_size); 119 | if (to_read > 0) { 120 | memcpy(dest, bufstart_ + offset, to_read); 121 | } 122 | return to_read; 123 | } 124 | 125 | /// Pad to alignment 126 | void PadToAlignmentWith(int padding) { 127 | size_t total_size = Roundup(cursize_, alignment_); 128 | size_t pad_size = total_size - cursize_; 129 | 130 | if (pad_size > 0) { 131 | assert((pad_size + cursize_) <= capacity_); 132 | memset(bufstart_ + cursize_, padding, pad_size); 133 | cursize_ += pad_size; 134 | } 135 | } 136 | 137 | // After a partial flush move the tail to the beginning of the buffer 138 | void RefitTail(size_t tail_offset, size_t tail_size) { 139 | if (tail_size > 0) { 140 | memmove(bufstart_, bufstart_ + tail_offset, tail_size); 141 | } 142 | cursize_ = tail_size; 143 | } 144 | 145 | // Returns place to start writing 146 | char* Destination() { 147 | return bufstart_ + cursize_; 148 | } 149 | 150 | void Size(size_t cursize) { 151 | cursize_ = cursize; 152 | } 153 | }; 154 | } 155 | -------------------------------------------------------------------------------- /rutil/auto_roll_logger.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | #include "rocksutil/auto_roll_logger.h" 7 | 8 | namespace rocksutil { 9 | 10 | std::string InfoLogFileName(const std::string& log_path) { 11 | return log_path + "/LOG"; 12 | } 13 | 14 | static std::string OldInfoLogFileName(const std::string& log_path, 15 | uint64_t ts) { 16 | char buf[50]; 17 | snprintf(buf, sizeof(buf), "%llu", static_cast(ts)); 18 | 19 | return log_path + "/LOG.old." + buf; 20 | } 21 | 22 | // -- AutoRollLogger 23 | Status AutoRollLogger::ResetLogger() { 24 | status_ = env_->NewLogger(log_fname_, 25 | flush_every_seconds_, &logger_); 26 | 27 | if (!status_.ok()) { 28 | return status_; 29 | } 30 | 31 | if (logger_->GetLogFileSize() == Logger::kDoNotSupportGetLogFileSize) { 32 | status_ = Status::NotSupported( 33 | "The underlying logger doesn't support GetLogFileSize()"); 34 | } 35 | if (status_.ok()) { 36 | cached_now = static_cast(env_->NowMicros() * 1e-6); 37 | ctime_ = cached_now; 38 | cached_now_access_count = 0; 39 | } 40 | 41 | return status_; 42 | } 43 | 44 | void AutoRollLogger::RollLogFile() { 45 | // This function is called when log is rotating. Two rotations 46 | // can happen quickly (NowMicro returns same value). To not overwrite 47 | // previous log file we increment by one micro second and try again. 48 | uint64_t now = env_->NowMicros(); 49 | std::string old_fname; 50 | do { 51 | old_fname = OldInfoLogFileName( 52 | log_path_, now); 53 | now++; 54 | } while (env_->FileExists(old_fname).ok()); 55 | env_->RenameFile(log_fname_, old_fname); 56 | } 57 | 58 | std::string AutoRollLogger::ValistToString(const char* format, 59 | va_list args) const { 60 | // Any log messages longer than 1024 will get truncated. 61 | // The user is responsible for chopping longer messages into multi line log 62 | static const int MAXBUFFERSIZE = 1024; 63 | char buffer[MAXBUFFERSIZE]; 64 | 65 | int count = vsnprintf(buffer, MAXBUFFERSIZE, format, args); 66 | (void) count; 67 | assert(count >= 0); 68 | 69 | return buffer; 70 | } 71 | 72 | void AutoRollLogger::LogInternal(const char* format, ...) { 73 | mutex_.AssertHeld(); 74 | va_list args; 75 | va_start(args, format); 76 | logger_->Logv(format, args); 77 | va_end(args); 78 | } 79 | 80 | void AutoRollLogger::Logv(const char* format, va_list ap) { 81 | assert(GetStatus().ok()); 82 | 83 | std::shared_ptr logger; 84 | { 85 | MutexLock l(&mutex_); 86 | if ((kLogFileTimeToRoll > 0 && LogExpired()) || 87 | (kMaxLogFileSize > 0 && logger_->GetLogFileSize() >= kMaxLogFileSize)) { 88 | RollLogFile(); 89 | Status s = ResetLogger(); 90 | if (!s.ok()) { 91 | // can't really log the error if creating a new LOG file failed 92 | return; 93 | } 94 | 95 | WriteHeaderInfo(); 96 | } 97 | 98 | // pin down the current logger_ instance before releasing the mutex. 99 | logger = logger_; 100 | } 101 | 102 | // Another thread could have put a new Logger instance into logger_ by now. 103 | // However, since logger is still hanging on to the previous instance 104 | // (reference count is not zero), we don't have to worry about it being 105 | // deleted while we are accessing it. 106 | // Note that logv itself is not mutex protected to allow maximum concurrency, 107 | // as thread safety should have been handled by the underlying logger. 108 | logger->Logv(format, ap); 109 | } 110 | 111 | void AutoRollLogger::WriteHeaderInfo() { 112 | mutex_.AssertHeld(); 113 | for (auto& header : headers_) { 114 | LogInternal("%s", header.c_str()); 115 | } 116 | } 117 | 118 | void AutoRollLogger::LogHeader(const char* format, va_list args) { 119 | // header message are to be retained in memory. Since we cannot make any 120 | // assumptions about the data contained in va_list, we will retain them as 121 | // strings 122 | va_list tmp; 123 | va_copy(tmp, args); 124 | std::string data = ValistToString(format, tmp); 125 | va_end(tmp); 126 | 127 | MutexLock l(&mutex_); 128 | headers_.push_back(data); 129 | 130 | // Log the original message to the current log 131 | logger_->Logv(format, args); 132 | } 133 | 134 | bool AutoRollLogger::LogExpired() { 135 | if (cached_now_access_count >= call_NowMicros_every_N_records_) { 136 | cached_now = static_cast(env_->NowMicros() * 1e-6); 137 | cached_now_access_count = 0; 138 | } 139 | 140 | ++cached_now_access_count; 141 | return cached_now >= ctime_ + kLogFileTimeToRoll; 142 | } 143 | 144 | //Status CreateLoggerFromOptions(Env* env, const std::string& log_path, 145 | Status CreateLogger(const std::string& log_path, 146 | std::shared_ptr* logger, Env* env, 147 | size_t log_max_size, size_t log_file_time_to_roll, 148 | const InfoLogLevel log_level, 149 | uint64_t flush_every_seconds) { 150 | 151 | std::string fname = 152 | InfoLogFileName(log_path); 153 | 154 | env->CreateDirIfMissing(log_path); // In case it does not exist 155 | // Currently we only support roll by time-to-roll and log size 156 | if (log_file_time_to_roll > 0 || log_max_size > 0) { 157 | AutoRollLogger* result = new AutoRollLogger( 158 | env, log_path, log_max_size, log_file_time_to_roll, 159 | log_level, flush_every_seconds); 160 | Status s = result->GetStatus(); 161 | if (!s.ok()) { 162 | delete result; 163 | } else { 164 | logger->reset(result); 165 | } 166 | return s; 167 | } else { 168 | // Open a log file in the same directory as the db 169 | env->RenameFile( 170 | fname, OldInfoLogFileName(log_path, env->NowMicros())); 171 | auto s = env->NewLogger(fname, flush_every_seconds, 172 | logger); 173 | if (logger->get() != nullptr) { 174 | (*logger)->SetInfoLogLevel(log_level); 175 | } 176 | return s; 177 | } 178 | } 179 | 180 | } // namespace rocksutil 181 | -------------------------------------------------------------------------------- /rutil/autovector.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace rocksutil { 15 | 16 | // A vector that leverages pre-allocated stack-based array to achieve better 17 | // performance for array with small amount of items. 18 | // 19 | // The interface resembles that of vector, but with less features since we aim 20 | // to solve the problem that we have in hand, rather than implementing a 21 | // full-fledged generic container. 22 | // 23 | // Currently we don't support: 24 | // * reserve()/shrink_to_fit() 25 | // If used correctly, in most cases, people should not touch the 26 | // underlying vector at all. 27 | // * random insert()/erase(), please only use push_back()/pop_back(). 28 | // * No move/swap operations. Each autovector instance has a 29 | // stack-allocated array and if we want support move/swap operations, we 30 | // need to copy the arrays other than just swapping the pointers. In this 31 | // case we'll just explicitly forbid these operations since they may 32 | // lead users to make false assumption by thinking they are inexpensive 33 | // operations. 34 | // 35 | // Naming style of public methods almost follows that of the STL's. 36 | template 37 | class autovector { 38 | public: 39 | // General STL-style container member types. 40 | typedef T value_type; 41 | typedef typename std::vector::difference_type difference_type; 42 | typedef typename std::vector::size_type size_type; 43 | typedef value_type& reference; 44 | typedef const value_type& const_reference; 45 | typedef value_type* pointer; 46 | typedef const value_type* const_pointer; 47 | 48 | // This class is the base for regular/const iterator 49 | template 50 | class iterator_impl { 51 | public: 52 | // -- iterator traits 53 | typedef iterator_impl self_type; 54 | typedef TValueType value_type; 55 | typedef TValueType& reference; 56 | typedef TValueType* pointer; 57 | typedef typename TAutoVector::difference_type difference_type; 58 | typedef std::random_access_iterator_tag iterator_category; 59 | 60 | iterator_impl(TAutoVector* vect, size_t index) 61 | : vect_(vect), index_(index) {}; 62 | iterator_impl(const iterator_impl&) = default; 63 | ~iterator_impl() {} 64 | iterator_impl& operator=(const iterator_impl&) = default; 65 | 66 | // -- Advancement 67 | // ++iterator 68 | self_type& operator++() { 69 | ++index_; 70 | return *this; 71 | } 72 | 73 | // iterator++ 74 | self_type operator++(int) { 75 | auto old = *this; 76 | ++index_; 77 | return old; 78 | } 79 | 80 | // --iterator 81 | self_type& operator--() { 82 | --index_; 83 | return *this; 84 | } 85 | 86 | // iterator-- 87 | self_type operator--(int) { 88 | auto old = *this; 89 | --index_; 90 | return old; 91 | } 92 | 93 | self_type operator-(difference_type len) { 94 | return self_type(vect_, index_ - len); 95 | } 96 | 97 | difference_type operator-(const self_type& other) { 98 | assert(vect_ == other.vect_); 99 | return index_ - other.index_; 100 | } 101 | 102 | self_type operator+(difference_type len) { 103 | return self_type(vect_, index_ + len); 104 | } 105 | 106 | self_type& operator+=(difference_type len) { 107 | index_ += len; 108 | return *this; 109 | } 110 | 111 | self_type& operator-=(difference_type len) { 112 | index_ -= len; 113 | return *this; 114 | } 115 | 116 | // -- Reference 117 | reference operator*() { 118 | assert(vect_->size() >= index_); 119 | return (*vect_)[index_]; 120 | } 121 | pointer operator->() { 122 | assert(vect_->size() >= index_); 123 | return &(*vect_)[index_]; 124 | } 125 | 126 | // -- Logical Operators 127 | bool operator==(const self_type& other) const { 128 | assert(vect_ == other.vect_); 129 | return index_ == other.index_; 130 | } 131 | 132 | bool operator!=(const self_type& other) const { return !(*this == other); } 133 | 134 | bool operator>(const self_type& other) const { 135 | assert(vect_ == other.vect_); 136 | return index_ > other.index_; 137 | } 138 | 139 | bool operator<(const self_type& other) const { 140 | assert(vect_ == other.vect_); 141 | return index_ < other.index_; 142 | } 143 | 144 | bool operator>=(const self_type& other) const { 145 | assert(vect_ == other.vect_); 146 | return index_ >= other.index_; 147 | } 148 | 149 | bool operator<=(const self_type& other) const { 150 | assert(vect_ == other.vect_); 151 | return index_ <= other.index_; 152 | } 153 | 154 | private: 155 | TAutoVector* vect_ = nullptr; 156 | size_t index_ = 0; 157 | }; 158 | 159 | typedef iterator_impl iterator; 160 | typedef iterator_impl const_iterator; 161 | typedef std::reverse_iterator reverse_iterator; 162 | typedef std::reverse_iterator const_reverse_iterator; 163 | 164 | autovector() = default; 165 | 166 | autovector(std::initializer_list init_list) { 167 | for (const T& item : init_list) { 168 | push_back(item); 169 | } 170 | } 171 | 172 | ~autovector() = default; 173 | 174 | // -- Immutable operations 175 | // Indicate if all data resides in in-stack data structure. 176 | bool only_in_stack() const { 177 | // If no element was inserted at all, the vector's capacity will be `0`. 178 | return vect_.capacity() == 0; 179 | } 180 | 181 | size_type size() const { return num_stack_items_ + vect_.size(); } 182 | 183 | // resize does not guarantee anything about the contents of the newly 184 | // available elements 185 | void resize(size_type n) { 186 | if (n > kSize) { 187 | vect_.resize(n - kSize); 188 | num_stack_items_ = kSize; 189 | } else { 190 | vect_.clear(); 191 | num_stack_items_ = n; 192 | } 193 | } 194 | 195 | bool empty() const { return size() == 0; } 196 | 197 | const_reference operator[](size_type n) const { 198 | assert(n < size()); 199 | return n < kSize ? values_[n] : vect_[n - kSize]; 200 | } 201 | 202 | reference operator[](size_type n) { 203 | assert(n < size()); 204 | return n < kSize ? values_[n] : vect_[n - kSize]; 205 | } 206 | 207 | const_reference at(size_type n) const { 208 | assert(n < size()); 209 | return (*this)[n]; 210 | } 211 | 212 | reference at(size_type n) { 213 | assert(n < size()); 214 | return (*this)[n]; 215 | } 216 | 217 | reference front() { 218 | assert(!empty()); 219 | return *begin(); 220 | } 221 | 222 | const_reference front() const { 223 | assert(!empty()); 224 | return *begin(); 225 | } 226 | 227 | reference back() { 228 | assert(!empty()); 229 | return *(end() - 1); 230 | } 231 | 232 | const_reference back() const { 233 | assert(!empty()); 234 | return *(end() - 1); 235 | } 236 | 237 | // -- Mutable Operations 238 | void push_back(T&& item) { 239 | if (num_stack_items_ < kSize) { 240 | values_[num_stack_items_++] = std::move(item); 241 | } else { 242 | vect_.push_back(item); 243 | } 244 | } 245 | 246 | void push_back(const T& item) { 247 | if (num_stack_items_ < kSize) { 248 | values_[num_stack_items_++] = item; 249 | } else { 250 | vect_.push_back(item); 251 | } 252 | } 253 | 254 | template 255 | void emplace_back(Args&&... args) { 256 | push_back(value_type(args...)); 257 | } 258 | 259 | void pop_back() { 260 | assert(!empty()); 261 | if (!vect_.empty()) { 262 | vect_.pop_back(); 263 | } else { 264 | --num_stack_items_; 265 | } 266 | } 267 | 268 | void clear() { 269 | num_stack_items_ = 0; 270 | vect_.clear(); 271 | } 272 | 273 | // -- Copy and Assignment 274 | autovector& assign(const autovector& other); 275 | 276 | autovector(const autovector& other) { assign(other); } 277 | 278 | autovector& operator=(const autovector& other) { return assign(other); } 279 | 280 | // move operation are disallowed since it is very hard to make sure both 281 | // autovectors are allocated from the same function stack. 282 | autovector& operator=(autovector&& other) = delete; 283 | autovector(autovector&& other) = delete; 284 | 285 | // -- Iterator Operations 286 | iterator begin() { return iterator(this, 0); } 287 | 288 | const_iterator begin() const { return const_iterator(this, 0); } 289 | 290 | iterator end() { return iterator(this, this->size()); } 291 | 292 | const_iterator end() const { return const_iterator(this, this->size()); } 293 | 294 | reverse_iterator rbegin() { return reverse_iterator(end()); } 295 | 296 | const_reverse_iterator rbegin() const { 297 | return const_reverse_iterator(end()); 298 | } 299 | 300 | reverse_iterator rend() { return reverse_iterator(begin()); } 301 | 302 | const_reverse_iterator rend() const { 303 | return const_reverse_iterator(begin()); 304 | } 305 | 306 | private: 307 | size_type num_stack_items_ = 0; // current number of items 308 | value_type values_[kSize]; // the first `kSize` items 309 | // used only if there are more than `kSize` items. 310 | std::vector vect_; 311 | }; 312 | 313 | template 314 | autovector& autovector::assign(const autovector& other) { 315 | // copy the internal vector 316 | vect_.assign(other.vect_.begin(), other.vect_.end()); 317 | 318 | // copy array 319 | num_stack_items_ = other.num_stack_items_; 320 | std::copy(other.values_, other.values_ + num_stack_items_, values_); 321 | 322 | return *this; 323 | } 324 | } // namespace rocksutil 325 | -------------------------------------------------------------------------------- /rutil/build_version.cc.in: -------------------------------------------------------------------------------- 1 | #include "build_version.h" 2 | const char* rocksutil_build_git_sha = "rocksutil_build_git_sha:@@GIT_SHA@@"; 3 | const char* rocksutil_build_git_date = "rocksutil_build_git_date:@@GIT_DATE_TIME@@"; 4 | const char* rocksutil_build_compile_date = __DATE__; 5 | -------------------------------------------------------------------------------- /rutil/build_version.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | #pragma once 7 | #if !defined(IOS_CROSS_COMPILE) 8 | // if we compile with Xcode, we don't run build_detect_vesion, so we don't 9 | // generate these variables 10 | // this variable tells us about the git revision 11 | extern const char* rocksutil_build_git_sha; 12 | 13 | // Date on which the code was compiled: 14 | extern const char* rocksutil_build_compile_date; 15 | #endif 16 | -------------------------------------------------------------------------------- /rutil/coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rocksutil/coding.h" 11 | 12 | #include 13 | #include "rocksutil/slice.h" 14 | 15 | namespace rocksutil { 16 | 17 | char* EncodeVarint32(char* dst, uint32_t v) { 18 | // Operate on characters as unsigneds 19 | unsigned char* ptr = reinterpret_cast(dst); 20 | static const int B = 128; 21 | if (v < (1 << 7)) { 22 | *(ptr++) = v; 23 | } else if (v < (1 << 14)) { 24 | *(ptr++) = v | B; 25 | *(ptr++) = v >> 7; 26 | } else if (v < (1 << 21)) { 27 | *(ptr++) = v | B; 28 | *(ptr++) = (v >> 7) | B; 29 | *(ptr++) = v >> 14; 30 | } else if (v < (1 << 28)) { 31 | *(ptr++) = v | B; 32 | *(ptr++) = (v >> 7) | B; 33 | *(ptr++) = (v >> 14) | B; 34 | *(ptr++) = v >> 21; 35 | } else { 36 | *(ptr++) = v | B; 37 | *(ptr++) = (v >> 7) | B; 38 | *(ptr++) = (v >> 14) | B; 39 | *(ptr++) = (v >> 21) | B; 40 | *(ptr++) = v >> 28; 41 | } 42 | return reinterpret_cast(ptr); 43 | } 44 | 45 | const char* GetVarint32PtrFallback(const char* p, const char* limit, 46 | uint32_t* value) { 47 | uint32_t result = 0; 48 | for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { 49 | uint32_t byte = *(reinterpret_cast(p)); 50 | p++; 51 | if (byte & 128) { 52 | // More bytes are present 53 | result |= ((byte & 127) << shift); 54 | } else { 55 | result |= (byte << shift); 56 | *value = result; 57 | return reinterpret_cast(p); 58 | } 59 | } 60 | return nullptr; 61 | } 62 | 63 | const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { 64 | uint64_t result = 0; 65 | for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { 66 | uint64_t byte = *(reinterpret_cast(p)); 67 | p++; 68 | if (byte & 128) { 69 | // More bytes are present 70 | result |= ((byte & 127) << shift); 71 | } else { 72 | result |= (byte << shift); 73 | *value = result; 74 | return reinterpret_cast(p); 75 | } 76 | } 77 | return nullptr; 78 | } 79 | 80 | } // namespace rocksutil 81 | -------------------------------------------------------------------------------- /rutil/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rocksutil/env.h" 11 | 12 | #include 13 | #include "rport/port.h" 14 | #include "rport/sys_time.h" 15 | 16 | #include "rutil/autovector.h" 17 | 18 | namespace rocksutil { 19 | 20 | Env::~Env() { 21 | } 22 | 23 | uint64_t Env::GetThreadID() const { 24 | std::hash hasher; 25 | return hasher(std::this_thread::get_id()); 26 | } 27 | 28 | Status Env::ReuseWritableFile(const std::string& fname, 29 | const std::string& old_fname, 30 | unique_ptr* result, 31 | const EnvOptions& options) { 32 | Status s = RenameFile(old_fname, fname); 33 | if (!s.ok()) { 34 | return s; 35 | } 36 | return NewWritableFile(fname, result, options); 37 | } 38 | 39 | Status Env::GetChildrenFileAttributes(const std::string& dir, 40 | std::vector* result) { 41 | assert(result != nullptr); 42 | std::vector child_fnames; 43 | Status s = GetChildren(dir, &child_fnames); 44 | if (!s.ok()) { 45 | return s; 46 | } 47 | result->resize(child_fnames.size()); 48 | size_t result_size = 0; 49 | for (size_t i = 0; i < child_fnames.size(); ++i) { 50 | const std::string path = dir + "/" + child_fnames[i]; 51 | if (!(s = GetFileSize(path, &(*result)[result_size].size_bytes)).ok()) { 52 | if (FileExists(path).IsNotFound()) { 53 | // The file may have been deleted since we listed the directory 54 | continue; 55 | } 56 | return s; 57 | } 58 | (*result)[result_size].name = std::move(child_fnames[i]); 59 | result_size++; 60 | } 61 | result->resize(result_size); 62 | return Status::OK(); 63 | } 64 | 65 | SequentialFile::~SequentialFile() { 66 | } 67 | 68 | RandomAccessFile::~RandomAccessFile() { 69 | } 70 | 71 | WritableFile::~WritableFile() { 72 | } 73 | 74 | Logger::~Logger() { 75 | } 76 | 77 | FileLock::~FileLock() { 78 | } 79 | 80 | void LogFlush(Logger *info_log) { 81 | if (info_log) { 82 | info_log->Flush(); 83 | } 84 | } 85 | 86 | void Log(Logger* info_log, const char* format, ...) { 87 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) { 88 | va_list ap; 89 | va_start(ap, format); 90 | info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); 91 | va_end(ap); 92 | } 93 | } 94 | 95 | void Logger::Logv(const InfoLogLevel log_level, const char* format, va_list ap) { 96 | static const char* kInfoLogLevelNames[5] = { "DEBUG", "INFO", "WARN", 97 | "ERROR", "FATAL" }; 98 | if (log_level < log_level_) { 99 | return; 100 | } 101 | 102 | if (log_level == InfoLogLevel::INFO_LEVEL) { 103 | // Doesn't print log level if it is INFO level. 104 | // This is to avoid unexpected performance regression after we add 105 | // the feature of log level. All the logs before we add the feature 106 | // are INFO level. We don't want to add extra costs to those existing 107 | // logging. 108 | Logv(format, ap); 109 | } else { 110 | char new_format[500]; 111 | snprintf(new_format, sizeof(new_format) - 1, "[%s] %s", 112 | kInfoLogLevelNames[log_level], format); 113 | Logv(new_format, ap); 114 | } 115 | } 116 | 117 | 118 | void Log(const InfoLogLevel log_level, Logger* info_log, const char* format, 119 | ...) { 120 | if (info_log && info_log->GetInfoLogLevel() <= log_level) { 121 | va_list ap; 122 | va_start(ap, format); 123 | 124 | if (log_level == InfoLogLevel::HEADER_LEVEL) { 125 | info_log->LogHeader(format, ap); 126 | } else { 127 | info_log->Logv(log_level, format, ap); 128 | } 129 | 130 | va_end(ap); 131 | } 132 | } 133 | 134 | void Header(Logger* info_log, const char* format, ...) { 135 | if (info_log) { 136 | va_list ap; 137 | va_start(ap, format); 138 | info_log->LogHeader(format, ap); 139 | va_end(ap); 140 | } 141 | } 142 | 143 | void Debug(Logger* info_log, const char* format, ...) { 144 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) { 145 | va_list ap; 146 | va_start(ap, format); 147 | info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap); 148 | va_end(ap); 149 | } 150 | } 151 | 152 | void Info(Logger* info_log, const char* format, ...) { 153 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) { 154 | va_list ap; 155 | va_start(ap, format); 156 | info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); 157 | va_end(ap); 158 | } 159 | } 160 | 161 | void Warn(Logger* info_log, const char* format, ...) { 162 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::WARN_LEVEL) { 163 | va_list ap; 164 | va_start(ap, format); 165 | info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap); 166 | va_end(ap); 167 | } 168 | } 169 | void Error(Logger* info_log, const char* format, ...) { 170 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::ERROR_LEVEL) { 171 | va_list ap; 172 | va_start(ap, format); 173 | info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap); 174 | va_end(ap); 175 | } 176 | } 177 | void Fatal(Logger* info_log, const char* format, ...) { 178 | if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::FATAL_LEVEL) { 179 | va_list ap; 180 | va_start(ap, format); 181 | info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap); 182 | va_end(ap); 183 | } 184 | } 185 | 186 | void LogFlush(const shared_ptr& info_log) { 187 | if (info_log) { 188 | info_log->Flush(); 189 | } 190 | } 191 | 192 | void Log(const InfoLogLevel log_level, const shared_ptr& info_log, 193 | const char* format, ...) { 194 | if (info_log) { 195 | va_list ap; 196 | va_start(ap, format); 197 | info_log->Logv(log_level, format, ap); 198 | va_end(ap); 199 | } 200 | } 201 | 202 | void Header(const shared_ptr& info_log, const char* format, ...) { 203 | if (info_log) { 204 | va_list ap; 205 | va_start(ap, format); 206 | info_log->LogHeader(format, ap); 207 | va_end(ap); 208 | } 209 | } 210 | 211 | void Debug(const shared_ptr& info_log, const char* format, ...) { 212 | if (info_log) { 213 | va_list ap; 214 | va_start(ap, format); 215 | info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap); 216 | va_end(ap); 217 | } 218 | } 219 | 220 | void Info(const shared_ptr& info_log, const char* format, ...) { 221 | if (info_log) { 222 | va_list ap; 223 | va_start(ap, format); 224 | info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); 225 | va_end(ap); 226 | } 227 | } 228 | 229 | void Warn(const shared_ptr& info_log, const char* format, ...) { 230 | if (info_log) { 231 | va_list ap; 232 | va_start(ap, format); 233 | info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap); 234 | va_end(ap); 235 | } 236 | } 237 | 238 | void Error(const shared_ptr& info_log, const char* format, ...) { 239 | if (info_log) { 240 | va_list ap; 241 | va_start(ap, format); 242 | info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap); 243 | va_end(ap); 244 | } 245 | } 246 | 247 | void Fatal(const shared_ptr& info_log, const char* format, ...) { 248 | if (info_log) { 249 | va_list ap; 250 | va_start(ap, format); 251 | info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap); 252 | va_end(ap); 253 | } 254 | } 255 | 256 | void Log(const shared_ptr& info_log, const char* format, ...) { 257 | if (info_log) { 258 | va_list ap; 259 | va_start(ap, format); 260 | info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap); 261 | va_end(ap); 262 | } 263 | } 264 | 265 | Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname, 266 | bool should_sync) { 267 | unique_ptr file; 268 | EnvOptions soptions; 269 | Status s = env->NewWritableFile(fname, &file, soptions); 270 | if (!s.ok()) { 271 | return s; 272 | } 273 | s = file->Append(data); 274 | if (s.ok() && should_sync) { 275 | s = file->Sync(); 276 | } 277 | if (!s.ok()) { 278 | env->DeleteFile(fname); 279 | } 280 | return s; 281 | } 282 | 283 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { 284 | EnvOptions soptions; 285 | data->clear(); 286 | unique_ptr file; 287 | Status s = env->NewSequentialFile(fname, &file, soptions); 288 | if (!s.ok()) { 289 | return s; 290 | } 291 | static const int kBufferSize = 8192; 292 | char* space = new char[kBufferSize]; 293 | while (true) { 294 | Slice fragment; 295 | s = file->Read(kBufferSize, &fragment, space); 296 | if (!s.ok()) { 297 | break; 298 | } 299 | data->append(fragment.data(), fragment.size()); 300 | if (fragment.empty()) { 301 | break; 302 | } 303 | } 304 | delete[] space; 305 | return s; 306 | } 307 | 308 | EnvWrapper::~EnvWrapper() { 309 | } 310 | 311 | EnvOptions Env::OptimizeForLogWrite(const EnvOptions& env_options) const { 312 | EnvOptions optimized_env_options(env_options); 313 | return optimized_env_options; 314 | } 315 | 316 | EnvOptions Env::OptimizeForManifestWrite(const EnvOptions& env_options) const { 317 | return env_options; 318 | } 319 | 320 | 321 | EnvOptions::EnvOptions() { 322 | } 323 | 324 | 325 | } // namespace rocksutil 326 | -------------------------------------------------------------------------------- /rutil/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include 11 | #include "rocksutil/coding.h" 12 | #include "rocksutil/hash.h" 13 | 14 | namespace rocksutil { 15 | 16 | // This function may intentionally do a left shift on a -ve number 17 | #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 9) 18 | __attribute__((__no_sanitize__("undefined"))) 19 | #elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9) 20 | __attribute__((__no_sanitize_undefined__)) 21 | #endif 22 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 23 | // Similar to murmur hash 24 | const uint32_t m = 0xc6a4a793; 25 | const uint32_t r = 24; 26 | const char* limit = data + n; 27 | uint32_t h = static_cast(seed ^ (n * m)); 28 | 29 | // Pick up four bytes at a time 30 | while (data + 4 <= limit) { 31 | uint32_t w = DecodeFixed32(data); 32 | data += 4; 33 | h += w; 34 | h *= m; 35 | h ^= (h >> 16); 36 | } 37 | 38 | // Pick up remaining bytes 39 | switch (limit - data) { 40 | // Note: It would be better if this was cast to unsigned char, but that 41 | // would be a disk format change since we previously didn't have any cast 42 | // at all (so gcc used signed char). 43 | // To understand the difference between shifting unsigned and signed chars, 44 | // let's use 250 as an example. unsigned char will be 250, while signed char 45 | // will be -6. Bit-wise, they are equivalent: 11111010. However, when 46 | // converting negative number (signed char) to int, it will be converted 47 | // into negative int (of equivalent value, which is -6), while converting 48 | // positive number (unsigned char) will be converted to 250. Bitwise, 49 | // this looks like this: 50 | // signed char 11111010 -> int 11111111111111111111111111111010 51 | // unsigned char 11111010 -> int 00000000000000000000000011111010 52 | case 3: 53 | h += static_cast(static_cast(data[2]) << 16); 54 | // fall through 55 | case 2: 56 | h += static_cast(static_cast(data[1]) << 8); 57 | // fall through 58 | case 1: 59 | h += static_cast(static_cast(data[0])); 60 | h *= m; 61 | h ^= (h >> r); 62 | break; 63 | } 64 | return h; 65 | } 66 | 67 | } // namespace rocksutil 68 | -------------------------------------------------------------------------------- /rutil/io_posix.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "rocksutil/env.h" 15 | 16 | // For non linux platform, the following macros are used only as place 17 | // holder. 18 | #if !(defined OS_LINUX) && !(defined CYGWIN) 19 | #define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */ 20 | #define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */ 21 | #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ 22 | #define POSIX_FADV_WILLNEED 3 /* [MC1] will need these pages */ 23 | #define POSIX_FADV_DONTNEED 4 /* [MC1] dont need these pages */ 24 | #endif 25 | 26 | namespace rocksutil { 27 | 28 | static Status IOError(const std::string& context, int err_number) { 29 | return (err_number == ENOSPC) ? 30 | Status::NoSpace(context, strerror(err_number)) : 31 | Status::IOError(context, strerror(err_number)); 32 | } 33 | 34 | class PosixHelper { 35 | public: 36 | static size_t GetUniqueIdFromFile(int fd, char* id, size_t max_size); 37 | }; 38 | 39 | class PosixSequentialFile : public SequentialFile { 40 | private: 41 | std::string filename_; 42 | FILE* file_; 43 | int fd_; 44 | bool use_os_buffer_; 45 | 46 | public: 47 | PosixSequentialFile(const std::string& fname, FILE* f, 48 | const EnvOptions& options); 49 | virtual ~PosixSequentialFile(); 50 | 51 | virtual Status Read(size_t n, Slice* result, char* scratch) override; 52 | virtual Status Skip(uint64_t n) override; 53 | virtual Status InvalidateCache(size_t offset, size_t length) override; 54 | }; 55 | 56 | class PosixDirectIOSequentialFile : public SequentialFile { 57 | public: 58 | explicit PosixDirectIOSequentialFile(const std::string& filename, int fd) 59 | : filename_(filename), fd_(fd) {} 60 | 61 | virtual ~PosixDirectIOSequentialFile() {} 62 | 63 | Status Read(size_t n, Slice* result, char* scratch) override; 64 | Status Skip(uint64_t n) override; 65 | Status InvalidateCache(size_t offset, size_t length) override; 66 | 67 | private: 68 | const std::string filename_; 69 | int fd_ = -1; 70 | std::atomic off_{0}; // read offset 71 | }; 72 | 73 | class PosixRandomAccessFile : public RandomAccessFile { 74 | protected: 75 | std::string filename_; 76 | int fd_; 77 | bool use_os_buffer_; 78 | 79 | public: 80 | PosixRandomAccessFile(const std::string& fname, int fd, 81 | const EnvOptions& options); 82 | virtual ~PosixRandomAccessFile(); 83 | 84 | virtual Status Read(uint64_t offset, size_t n, Slice* result, 85 | char* scratch) const override; 86 | #if defined(OS_LINUX) || defined(OS_MACOSX) 87 | virtual size_t GetUniqueId(char* id, size_t max_size) const override; 88 | #endif 89 | virtual void Hint(AccessPattern pattern) override; 90 | virtual Status InvalidateCache(size_t offset, size_t length) override; 91 | }; 92 | 93 | // Direct IO random access file direct IO implementation 94 | class PosixDirectIORandomAccessFile : public PosixRandomAccessFile { 95 | public: 96 | explicit PosixDirectIORandomAccessFile(const std::string& filename, int fd) 97 | : PosixRandomAccessFile(filename, fd, EnvOptions()) {} 98 | virtual ~PosixDirectIORandomAccessFile() {} 99 | 100 | Status Read(uint64_t offset, size_t n, Slice* result, 101 | char* scratch) const override; 102 | virtual void Hint(AccessPattern pattern) override {} 103 | Status InvalidateCache(size_t offset, size_t length) override { 104 | return Status::OK(); 105 | } 106 | }; 107 | 108 | class PosixWritableFile : public WritableFile { 109 | protected: 110 | const std::string filename_; 111 | int fd_; 112 | uint64_t filesize_; 113 | #ifdef ROCKSUTIL_FALLOCATE_PRESENT 114 | bool allow_fallocate_; 115 | bool fallocate_with_keep_size_; 116 | #endif 117 | 118 | public: 119 | explicit PosixWritableFile(const std::string& fname, int fd, 120 | const EnvOptions& options); 121 | virtual ~PosixWritableFile(); 122 | 123 | // Means Close() will properly take care of truncate 124 | // and it does not need any additional information 125 | virtual Status Truncate(uint64_t size) override { return Status::OK(); } 126 | virtual Status Close() override; 127 | virtual Status Append(const Slice& data) override; 128 | virtual Status PositionedAppend(const Slice& data, uint64_t offset) override; 129 | virtual Status Flush() override; 130 | virtual Status Sync() override; 131 | virtual Status Fsync() override; 132 | virtual bool IsSyncThreadSafe() const override; 133 | virtual uint64_t GetFileSize() override; 134 | virtual Status InvalidateCache(size_t offset, size_t length) override; 135 | #ifdef ROCKSUTIL_FALLOCATE_PRESENT 136 | virtual Status Allocate(uint64_t offset, uint64_t len) override; 137 | virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override; 138 | virtual size_t GetUniqueId(char* id, size_t max_size) const override; 139 | #endif 140 | }; 141 | 142 | class PosixDirectIOWritableFile : public PosixWritableFile { 143 | public: 144 | explicit PosixDirectIOWritableFile(const std::string& filename, int fd) 145 | : PosixWritableFile(filename, fd, EnvOptions()) {} 146 | virtual ~PosixDirectIOWritableFile() {} 147 | 148 | bool UseOSBuffer() const override { return false; } 149 | size_t GetRequiredBufferAlignment() const override { return 4 * 1024; } 150 | Status Append(const Slice& data) override; 151 | Status PositionedAppend(const Slice& data, uint64_t offset) override; 152 | bool UseDirectIO() const override { return true; } 153 | Status InvalidateCache(size_t offset, size_t length) override { 154 | return Status::OK(); 155 | } 156 | }; 157 | 158 | class PosixMmapReadableFile : public RandomAccessFile { 159 | private: 160 | int fd_; 161 | std::string filename_; 162 | void* mmapped_region_; 163 | size_t length_; 164 | 165 | public: 166 | PosixMmapReadableFile(const int fd, const std::string& fname, void* base, 167 | size_t length, const EnvOptions& options); 168 | virtual ~PosixMmapReadableFile(); 169 | virtual Status Read(uint64_t offset, size_t n, Slice* result, 170 | char* scratch) const override; 171 | virtual Status InvalidateCache(size_t offset, size_t length) override; 172 | }; 173 | 174 | class PosixMmapFile : public WritableFile { 175 | private: 176 | std::string filename_; 177 | int fd_; 178 | size_t page_size_; 179 | size_t map_size_; // How much extra memory to map at a time 180 | char* base_; // The mapped region 181 | char* limit_; // Limit of the mapped region 182 | char* dst_; // Where to write next (in range [base_,limit_]) 183 | char* last_sync_; // Where have we synced up to 184 | uint64_t file_offset_; // Offset of base_ in file 185 | #ifdef ROCKSUTIL_FALLOCATE_PRESENT 186 | bool allow_fallocate_; // If false, fallocate calls are bypassed 187 | bool fallocate_with_keep_size_; 188 | #endif 189 | 190 | // Roundup x to a multiple of y 191 | static size_t Roundup(size_t x, size_t y) { return ((x + y - 1) / y) * y; } 192 | 193 | size_t TruncateToPageBoundary(size_t s) { 194 | s -= (s & (page_size_ - 1)); 195 | assert((s % page_size_) == 0); 196 | return s; 197 | } 198 | 199 | Status MapNewRegion(); 200 | Status UnmapCurrentRegion(); 201 | Status Msync(); 202 | 203 | public: 204 | PosixMmapFile(const std::string& fname, int fd, size_t page_size, 205 | const EnvOptions& options); 206 | ~PosixMmapFile(); 207 | 208 | // Means Close() will properly take care of truncate 209 | // and it does not need any additional information 210 | virtual Status Truncate(uint64_t size) override { return Status::OK(); } 211 | virtual Status Close() override; 212 | virtual Status Append(const Slice& data) override; 213 | virtual Status Flush() override; 214 | virtual Status Sync() override; 215 | virtual Status Fsync() override; 216 | virtual uint64_t GetFileSize() override; 217 | virtual Status InvalidateCache(size_t offset, size_t length) override; 218 | #ifdef ROCKSUTIL_FALLOCATE_PRESENT 219 | virtual Status Allocate(uint64_t offset, uint64_t len) override; 220 | #endif 221 | }; 222 | 223 | class PosixRandomRWFile : public RandomRWFile { 224 | public: 225 | explicit PosixRandomRWFile(const std::string& fname, int fd, 226 | const EnvOptions& options); 227 | virtual ~PosixRandomRWFile(); 228 | 229 | virtual Status Write(uint64_t offset, const Slice& data) override; 230 | 231 | virtual Status Read(uint64_t offset, size_t n, Slice* result, 232 | char* scratch) const override; 233 | 234 | virtual Status Flush() override; 235 | virtual Status Sync() override; 236 | virtual Status Fsync() override; 237 | virtual Status Close() override; 238 | 239 | private: 240 | const std::string filename_; 241 | int fd_; 242 | }; 243 | 244 | class PosixDirectory : public Directory { 245 | public: 246 | explicit PosixDirectory(int fd) : fd_(fd) {} 247 | ~PosixDirectory(); 248 | virtual Status Fsync() override; 249 | 250 | private: 251 | int fd_; 252 | }; 253 | 254 | } // namespace rocksutil 255 | -------------------------------------------------------------------------------- /rutil/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // Log format information shared by reader and writer. 11 | // See ../doc/log_format.txt for more detail. 12 | 13 | #pragma once 14 | namespace rocksutil { 15 | namespace log { 16 | 17 | enum RecordType { 18 | // Zero is reserved for preallocated files 19 | kZeroType = 0, 20 | kFullType = 1, 21 | 22 | // For fragments 23 | kFirstType = 2, 24 | kMiddleType = 3, 25 | kLastType = 4, 26 | 27 | // For recycled log files 28 | kRecyclableFullType = 5, 29 | kRecyclableFirstType = 6, 30 | kRecyclableMiddleType = 7, 31 | kRecyclableLastType = 8, 32 | }; 33 | static const int kMaxRecordType = kRecyclableLastType; 34 | 35 | static const unsigned int kBlockSize = 32768; 36 | 37 | // Header is checksum (4 bytes), type (1 byte), length (2 bytes). 38 | static const int kHeaderSize = 4 + 1 + 2; 39 | 40 | // Recyclable header is checksum (4 bytes), type (1 byte), log number 41 | // (4 bytes), length (2 bytes). 42 | static const int kRecyclableHeaderSize = 4 + 1 + 4 + 2; 43 | 44 | } // namespace log 45 | } // namespace rocksutil 46 | -------------------------------------------------------------------------------- /rutil/log_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rocksutil/log_writer.h" 11 | 12 | #include 13 | #include "rocksutil/env.h" 14 | #include "rocksutil/coding.h" 15 | #include "rocksutil/crc32c.h" 16 | #include "rocksutil/file_reader_writer.h" 17 | 18 | namespace rocksutil { 19 | namespace log { 20 | 21 | Writer::Writer(unique_ptr&& dest, 22 | uint64_t log_number, bool recycle_log_files) 23 | : dest_(std::move(dest)), 24 | block_offset_(0), 25 | log_number_(log_number), 26 | recycle_log_files_(recycle_log_files) { 27 | for (int i = 0; i <= kMaxRecordType; i++) { 28 | char t = static_cast(i); 29 | type_crc_[i] = crc32c::Value(&t, 1); 30 | } 31 | } 32 | 33 | Writer::~Writer() { 34 | } 35 | 36 | Status Writer::AddRecord(const Slice& slice) { 37 | const char* ptr = slice.data(); 38 | size_t left = slice.size(); 39 | 40 | // Header size varies depending on whether we are recycling or not. 41 | const int header_size = 42 | recycle_log_files_ ? kRecyclableHeaderSize : kHeaderSize; 43 | 44 | // Fragment the record if necessary and emit it. Note that if slice 45 | // is empty, we still want to iterate once to emit a single 46 | // zero-length record 47 | Status s; 48 | bool begin = true; 49 | do { 50 | const int64_t leftover = kBlockSize - block_offset_; 51 | assert(leftover >= 0); 52 | if (leftover < header_size) { 53 | // Switch to a new block 54 | if (leftover > 0) { 55 | // Fill the trailer (literal below relies on kHeaderSize and 56 | // kRecyclableHeaderSize being <= 11) 57 | assert(header_size <= 11); 58 | dest_->Append( 59 | Slice("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", leftover)); 60 | } 61 | block_offset_ = 0; 62 | } 63 | 64 | // Invariant: we never leave < header_size bytes in a block. 65 | assert(static_cast(kBlockSize - block_offset_) >= header_size); 66 | 67 | const size_t avail = kBlockSize - block_offset_ - header_size; 68 | const size_t fragment_length = (left < avail) ? left : avail; 69 | 70 | RecordType type; 71 | const bool end = (left == fragment_length); 72 | if (begin && end) { 73 | type = recycle_log_files_ ? kRecyclableFullType : kFullType; 74 | } else if (begin) { 75 | type = recycle_log_files_ ? kRecyclableFirstType : kFirstType; 76 | } else if (end) { 77 | type = recycle_log_files_ ? kRecyclableLastType : kLastType; 78 | } else { 79 | type = recycle_log_files_ ? kRecyclableMiddleType : kMiddleType; 80 | } 81 | 82 | s = EmitPhysicalRecord(type, ptr, fragment_length); 83 | ptr += fragment_length; 84 | left -= fragment_length; 85 | begin = false; 86 | } while (s.ok() && left > 0); 87 | return s; 88 | } 89 | 90 | Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { 91 | assert(n <= 0xffff); // Must fit in two bytes 92 | 93 | size_t header_size; 94 | char buf[kRecyclableHeaderSize]; 95 | 96 | // Format the header 97 | buf[4] = static_cast(n & 0xff); 98 | buf[5] = static_cast(n >> 8); 99 | buf[6] = static_cast(t); 100 | 101 | uint32_t crc = type_crc_[t]; 102 | if (t < kRecyclableFullType) { 103 | // Legacy record format 104 | assert(block_offset_ + kHeaderSize + n <= kBlockSize); 105 | header_size = kHeaderSize; 106 | } else { 107 | // Recyclable record format 108 | assert(block_offset_ + kRecyclableHeaderSize + n <= kBlockSize); 109 | header_size = kRecyclableHeaderSize; 110 | 111 | // Only encode low 32-bits of the 64-bit log number. This means 112 | // we will fail to detect an old record if we recycled a log from 113 | // ~4 billion logs ago, but that is effectively impossible, and 114 | // even if it were we'dbe far more likely to see a false positive 115 | // on the 32-bit CRC. 116 | EncodeFixed32(buf + 7, static_cast(log_number_)); 117 | crc = crc32c::Extend(crc, buf + 7, 4); 118 | } 119 | 120 | // Compute the crc of the record type and the payload. 121 | crc = crc32c::Extend(crc, ptr, n); 122 | crc = crc32c::Mask(crc); // Adjust for storage 123 | EncodeFixed32(buf, crc); 124 | 125 | // Write the header and the payload 126 | Status s = dest_->Append(Slice(buf, header_size)); 127 | if (s.ok()) { 128 | s = dest_->Append(Slice(ptr, n)); 129 | if (s.ok()) { 130 | s = dest_->Flush(); 131 | } 132 | } 133 | block_offset_ += header_size + n; 134 | return s; 135 | } 136 | 137 | } // namespace log 138 | } // namespace rocksutil 139 | -------------------------------------------------------------------------------- /rutil/lru_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | 11 | #include "rutil/sharded_cache.h" 12 | 13 | #include "rport/port.h" 14 | #include "rutil/autovector.h" 15 | 16 | namespace rocksutil { 17 | 18 | // LRU cache implementation 19 | 20 | // An entry is a variable length heap-allocated structure. 21 | // Entries are referenced by cache and/or by any external entity. 22 | // The cache keeps all its entries in table. Some elements 23 | // are also stored on LRU list. 24 | // 25 | // LRUHandle can be in these states: 26 | // 1. Referenced externally AND in hash table. 27 | // In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true) 28 | // 2. Not referenced externally and in hash table. In that case the entry is 29 | // in the LRU and can be freed. (refs == 1 && in_cache == true) 30 | // 3. Referenced externally and not in hash table. In that case the entry is 31 | // in not on LRU and not in table. (refs >= 1 && in_cache == false) 32 | // 33 | // All newly created LRUHandles are in state 1. If you call 34 | // LRUCacheShard::Release 35 | // on entry in state 1, it will go into state 2. To move from state 1 to 36 | // state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the 37 | // same key. 38 | // To move from state 2 to state 1, use LRUCacheShard::Lookup. 39 | // Before destruction, make sure that no handles are in state 1. This means 40 | // that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a 41 | // matching 42 | // RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3) 43 | 44 | struct LRUHandle { 45 | void* value; 46 | void (*deleter)(const Slice&, void* value); 47 | LRUHandle* next_hash; 48 | LRUHandle* next; 49 | LRUHandle* prev; 50 | size_t charge; // TODO(opt): Only allow uint32_t? 51 | size_t key_length; 52 | uint32_t refs; // a number of refs to this entry 53 | // cache itself is counted as 1 54 | 55 | // Include the following flags: 56 | // in_cache: whether this entry is referenced by the hash table. 57 | // is_high_pri: whether this entry is high priority entry. 58 | // in_high_pro_pool: whether this entry is in high-pri pool. 59 | char flags; 60 | 61 | uint32_t hash; // Hash of key(); used for fast sharding and comparisons 62 | 63 | char key_data[1]; // Beginning of key 64 | 65 | Slice key() const { 66 | // For cheaper lookups, we allow a temporary Handle object 67 | // to store a pointer to a key in "value". 68 | if (next == this) { 69 | return *(reinterpret_cast(value)); 70 | } else { 71 | return Slice(key_data, key_length); 72 | } 73 | } 74 | 75 | bool InCache() { return flags & 1; } 76 | bool IsHighPri() { return flags & 2; } 77 | bool InHighPriPool() { return flags & 4; } 78 | 79 | void SetInCache(bool in_cache) { 80 | if (in_cache) { 81 | flags |= 1; 82 | } else { 83 | flags &= ~1; 84 | } 85 | } 86 | 87 | void SetPriority(Cache::Priority priority) { 88 | if (priority == Cache::Priority::HIGH) { 89 | flags |= 2; 90 | } else { 91 | flags &= ~2; 92 | } 93 | } 94 | 95 | void SetInHighPriPool(bool in_high_pri_pool) { 96 | if (in_high_pri_pool) { 97 | flags |= 4; 98 | } else { 99 | flags &= ~4; 100 | } 101 | } 102 | 103 | void Free() { 104 | assert((refs == 1 && InCache()) || (refs == 0 && !InCache())); 105 | if (deleter) { 106 | (*deleter)(key(), value); 107 | } 108 | delete[] reinterpret_cast(this); 109 | } 110 | }; 111 | 112 | // We provide our own simple hash table since it removes a whole bunch 113 | // of porting hacks and is also faster than some of the built-in hash 114 | // table implementations in some of the compiler/runtime combinations 115 | // we have tested. E.g., readrandom speeds up by ~5% over the g++ 116 | // 4.4.3's builtin hashtable. 117 | class LRUHandleTable { 118 | public: 119 | LRUHandleTable(); 120 | ~LRUHandleTable(); 121 | 122 | LRUHandle* Lookup(const Slice& key, uint32_t hash); 123 | LRUHandle* Insert(LRUHandle* h); 124 | LRUHandle* Remove(const Slice& key, uint32_t hash); 125 | 126 | template 127 | void ApplyToAllCacheEntries(T func) { 128 | for (uint32_t i = 0; i < length_; i++) { 129 | LRUHandle* h = list_[i]; 130 | while (h != nullptr) { 131 | auto n = h->next_hash; 132 | assert(h->InCache()); 133 | func(h); 134 | h = n; 135 | } 136 | } 137 | } 138 | 139 | private: 140 | // Return a pointer to slot that points to a cache entry that 141 | // matches key/hash. If there is no such cache entry, return a 142 | // pointer to the trailing slot in the corresponding linked list. 143 | LRUHandle** FindPointer(const Slice& key, uint32_t hash); 144 | 145 | void Resize(); 146 | 147 | // The table consists of an array of buckets where each bucket is 148 | // a linked list of cache entries that hash into the bucket. 149 | uint32_t length_; 150 | uint32_t elems_; 151 | LRUHandle** list_; 152 | }; 153 | 154 | // A single shard of sharded cache. 155 | class LRUCacheShard : public CacheShard { 156 | public: 157 | LRUCacheShard(); 158 | virtual ~LRUCacheShard(); 159 | 160 | // Separate from constructor so caller can easily make an array of LRUCache 161 | // if current usage is more than new capacity, the function will attempt to 162 | // free the needed space 163 | virtual void SetCapacity(size_t capacity) override; 164 | 165 | // Set the flag to reject insertion if cache if full. 166 | virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override; 167 | 168 | // Set percentage of capacity reserved for high-pri cache entries. 169 | void SetHighPriorityPoolRatio(double high_pri_pool_ratio); 170 | 171 | // Like Cache methods, but with an extra "hash" parameter. 172 | virtual Status Insert(const Slice& key, uint32_t hash, void* value, 173 | size_t charge, 174 | void (*deleter)(const Slice& key, void* value), 175 | Cache::Handle** handle, 176 | Cache::Priority priority) override; 177 | virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override; 178 | virtual void Release(Cache::Handle* handle) override; 179 | virtual void Erase(const Slice& key, uint32_t hash) override; 180 | 181 | // Although in some platforms the update of size_t is atomic, to make sure 182 | // GetUsage() and GetPinnedUsage() work correctly under any platform, we'll 183 | // protect them with mutex_. 184 | 185 | virtual size_t GetUsage() const override; 186 | virtual size_t GetPinnedUsage() const override; 187 | 188 | virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t), 189 | bool thread_safe) override; 190 | 191 | virtual void EraseUnRefEntries() override; 192 | 193 | void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri); 194 | 195 | private: 196 | void LRU_Remove(LRUHandle* e); 197 | void LRU_Insert(LRUHandle* e); 198 | 199 | // Overflow the last entry in high-pri pool to low-pri pool until size of 200 | // high-pri pool is no larger than the size specify by high_pri_pool_pct. 201 | void MaintainPoolSize(); 202 | 203 | // Just reduce the reference count by 1. 204 | // Return true if last reference 205 | bool Unref(LRUHandle* e); 206 | 207 | // Free some space following strict LRU policy until enough space 208 | // to hold (usage_ + charge) is freed or the lru list is empty 209 | // This function is not thread safe - it needs to be executed while 210 | // holding the mutex_ 211 | void EvictFromLRU(size_t charge, autovector* deleted); 212 | 213 | // Initialized before use. 214 | size_t capacity_; 215 | 216 | // Memory size for entries residing in the cache 217 | size_t usage_; 218 | 219 | // Memory size for entries residing only in the LRU list 220 | size_t lru_usage_; 221 | 222 | // Memory size for entries in high-pri pool. 223 | size_t high_pri_pool_usage_; 224 | 225 | // Whether to reject insertion if cache reaches its full capacity. 226 | bool strict_capacity_limit_; 227 | 228 | // Ratio of capacity reserved for high priority cache entries. 229 | double high_pri_pool_ratio_; 230 | 231 | // High-pri pool size, equals to capacity * high_pri_pool_ratio. 232 | // Remember the value to avoid recomputing each time. 233 | double high_pri_pool_capacity_; 234 | 235 | // mutex_ protects the following state. 236 | // We don't count mutex_ as the cache's internal state so semantically we 237 | // don't mind mutex_ invoking the non-const actions. 238 | mutable port::Mutex mutex_; 239 | 240 | // Dummy head of LRU list. 241 | // lru.prev is newest entry, lru.next is oldest entry. 242 | // LRU contains items which can be evicted, ie reference only by cache 243 | LRUHandle lru_; 244 | 245 | // Pointer to head of low-pri pool in LRU list. 246 | LRUHandle* lru_low_pri_; 247 | 248 | LRUHandleTable table_; 249 | }; 250 | 251 | class LRUCache : public ShardedCache { 252 | public: 253 | LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit, 254 | double high_pri_pool_ratio); 255 | virtual ~LRUCache(); 256 | virtual const char* Name() const override { return "LRUCache"; } 257 | virtual CacheShard* GetShard(int shard) override; 258 | virtual const CacheShard* GetShard(int shard) const override; 259 | virtual void* Value(Handle* handle) override; 260 | virtual size_t GetCharge(Handle* handle) const override; 261 | virtual uint32_t GetHash(Handle* handle) const override; 262 | virtual void DisownData() override; 263 | 264 | private: 265 | LRUCacheShard* shards_; 266 | }; 267 | 268 | } // namespace rocksutil 269 | -------------------------------------------------------------------------------- /rutil/posix_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | // 10 | // Logger implementation that can be shared by all environments 11 | // where enough posix functionality is available. 12 | 13 | #pragma once 14 | #include 15 | #include 16 | #include "rport/sys_time.h" 17 | #include 18 | #include 19 | 20 | #ifdef OS_LINUX 21 | #ifndef FALLOC_FL_KEEP_SIZE 22 | #include 23 | #endif 24 | #endif 25 | 26 | #include "rocksutil/env.h" 27 | #include 28 | 29 | namespace rocksutil { 30 | 31 | class PosixLogger : public Logger { 32 | private: 33 | FILE* file_; 34 | uint64_t (*gettid_)(); // Return the thread id for the current thread 35 | std::atomic_size_t log_size_; 36 | int fd_; 37 | std::atomic_uint_fast64_t last_flush_micros_; 38 | Env* env_; 39 | bool flush_pending_; 40 | public: 41 | PosixLogger(FILE* f, uint64_t (*gettid)(), Env* env, 42 | uint64_t flush_every_seconds = 5, 43 | const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL) 44 | : Logger(flush_every_seconds, log_level), 45 | file_(f), 46 | gettid_(gettid), 47 | log_size_(0), 48 | fd_(fileno(f)), 49 | last_flush_micros_(0), 50 | env_(env), 51 | flush_pending_(false) {} 52 | virtual ~PosixLogger() { 53 | fclose(file_); 54 | } 55 | virtual void Flush() override { 56 | if (flush_pending_) { 57 | flush_pending_ = false; 58 | fflush(file_); 59 | } 60 | if (flush_every_seconds_) { 61 | last_flush_micros_ = env_->NowMicros(); 62 | } 63 | } 64 | 65 | using Logger::Logv; 66 | virtual void Logv(const char* format, va_list ap) override { 67 | 68 | const uint64_t thread_id = (*gettid_)(); 69 | 70 | // We try twice: the first time with a fixed-size stack allocated buffer, 71 | // and the second time with a much larger dynamically allocated buffer. 72 | char buffer[500]; 73 | for (int iter = 0; iter < 2; iter++) { 74 | char* base; 75 | int bufsize; 76 | if (iter == 0) { 77 | bufsize = sizeof(buffer); 78 | base = buffer; 79 | } else { 80 | bufsize = 65536; 81 | base = new char[bufsize]; 82 | } 83 | char* p = base; 84 | char* limit = base + bufsize; 85 | 86 | struct timeval now_tv; 87 | gettimeofday(&now_tv, nullptr); 88 | const time_t seconds = now_tv.tv_sec; 89 | struct tm t; 90 | localtime_r(&seconds, &t); 91 | p += snprintf(p, limit - p, 92 | "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", 93 | t.tm_year + 1900, 94 | t.tm_mon + 1, 95 | t.tm_mday, 96 | t.tm_hour, 97 | t.tm_min, 98 | t.tm_sec, 99 | static_cast(now_tv.tv_usec), 100 | static_cast(thread_id)); 101 | 102 | // Print the message 103 | if (p < limit) { 104 | va_list backup_ap; 105 | va_copy(backup_ap, ap); 106 | p += vsnprintf(p, limit - p, format, backup_ap); 107 | va_end(backup_ap); 108 | } 109 | 110 | // Truncate to available space if necessary 111 | if (p >= limit) { 112 | if (iter == 0) { 113 | continue; // Try again with larger buffer 114 | } else { 115 | p = limit - 1; 116 | } 117 | } 118 | 119 | // Add newline if necessary 120 | if (p == base || p[-1] != '\n') { 121 | *p++ = '\n'; 122 | } 123 | 124 | assert(p <= limit); 125 | const size_t write_size = p - base; 126 | 127 | #ifdef ROCKSUTIL_FALLOCATE_PRESENT 128 | const int kDebugLogChunkSize = 128 * 1024; 129 | 130 | // If this write would cross a boundary of kDebugLogChunkSize 131 | // space, pre-allocate more space to avoid overly large 132 | // allocations from filesystem allocsize options. 133 | const size_t log_size = log_size_; 134 | const size_t last_allocation_chunk = 135 | ((kDebugLogChunkSize - 1 + log_size) / kDebugLogChunkSize); 136 | const size_t desired_allocation_chunk = 137 | ((kDebugLogChunkSize - 1 + log_size + write_size) / 138 | kDebugLogChunkSize); 139 | if (last_allocation_chunk != desired_allocation_chunk) { 140 | fallocate( 141 | fd_, FALLOC_FL_KEEP_SIZE, 0, 142 | static_cast(desired_allocation_chunk * kDebugLogChunkSize)); 143 | } 144 | #endif 145 | 146 | size_t sz = fwrite(base, 1, write_size, file_); 147 | flush_pending_ = true; 148 | assert(sz == write_size); 149 | if (sz > 0) { 150 | log_size_ += write_size; 151 | } 152 | // Always flush 153 | if (flush_every_seconds_ == 0) { 154 | Flush(); 155 | } else { 156 | uint64_t now_micros = static_cast(now_tv.tv_sec) * 1000000 + 157 | now_tv.tv_usec; 158 | if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) { 159 | Flush(); 160 | } 161 | } 162 | if (base != buffer) { 163 | delete[] base; 164 | } 165 | break; 166 | } 167 | } 168 | size_t GetLogFileSize() const override { return log_size_; } 169 | }; 170 | 171 | } // namespace rocksutil 172 | -------------------------------------------------------------------------------- /rutil/random.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | 7 | #include "rutil/random.h" 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "rport/likely.h" 15 | #include "rocksutil/thread_local.h" 16 | 17 | #if ROCKSUTIL_SUPPORT_THREAD_LOCAL 18 | #define STORAGE_DECL static __thread 19 | #else 20 | #define STORAGE_DECL static 21 | #endif 22 | 23 | namespace rocksutil { 24 | 25 | Random* Random::GetTLSInstance() { 26 | STORAGE_DECL Random* tls_instance; 27 | STORAGE_DECL std::aligned_storage::type tls_instance_bytes; 28 | 29 | auto rv = tls_instance; 30 | if (UNLIKELY(rv == nullptr)) { 31 | size_t seed = std::hash()(std::this_thread::get_id()); 32 | rv = new (&tls_instance_bytes) Random((uint32_t)seed); 33 | tls_instance = rv; 34 | } 35 | return rv; 36 | } 37 | 38 | } // namespace rocksutil 39 | -------------------------------------------------------------------------------- /rutil/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | #include 12 | #include 13 | 14 | namespace rocksutil { 15 | 16 | // A very simple random number generator. Not especially good at 17 | // generating truly random bits, but good enough for our needs in this 18 | // package. 19 | class Random { 20 | private: 21 | enum : uint32_t { 22 | M = 2147483647L // 2^31-1 23 | }; 24 | enum : uint64_t { 25 | A = 16807 // bits 14, 8, 7, 5, 2, 1, 0 26 | }; 27 | 28 | uint32_t seed_; 29 | 30 | static uint32_t GoodSeed(uint32_t s) { return (s & M) != 0 ? (s & M) : 1; } 31 | 32 | public: 33 | // This is the largest value that can be returned from Next() 34 | enum : uint32_t { kMaxNext = M }; 35 | 36 | explicit Random(uint32_t s) : seed_(GoodSeed(s)) {} 37 | 38 | void Reset(uint32_t s) { seed_ = GoodSeed(s); } 39 | 40 | uint32_t Next() { 41 | // We are computing 42 | // seed_ = (seed_ * A) % M, where M = 2^31-1 43 | // 44 | // seed_ must not be zero or M, or else all subsequent computed values 45 | // will be zero or M respectively. For all other values, seed_ will end 46 | // up cycling through every number in [1,M-1] 47 | uint64_t product = seed_ * A; 48 | 49 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 50 | seed_ = static_cast((product >> 31) + (product & M)); 51 | // The first reduction may overflow by 1 bit, so we may need to 52 | // repeat. mod == M is not possible; using > allows the faster 53 | // sign-bit-based test. 54 | if (seed_ > M) { 55 | seed_ -= M; 56 | } 57 | return seed_; 58 | } 59 | 60 | // Returns a uniformly distributed value in the range [0..n-1] 61 | // REQUIRES: n > 0 62 | uint32_t Uniform(int n) { return Next() % n; } 63 | 64 | // Randomly returns true ~"1/n" of the time, and false otherwise. 65 | // REQUIRES: n > 0 66 | bool OneIn(int n) { return (Next() % n) == 0; } 67 | 68 | // Skewed: pick "base" uniformly from range [0,max_log] and then 69 | // return "base" random bits. The effect is to pick a number in the 70 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 71 | uint32_t Skewed(int max_log) { 72 | return Uniform(1 << Uniform(max_log + 1)); 73 | } 74 | 75 | // Returns a Random instance for use by the current thread without 76 | // additional locking 77 | static Random* GetTLSInstance(); 78 | }; 79 | 80 | // A simple 64bit random number generator based on std::mt19937_64 81 | class Random64 { 82 | private: 83 | std::mt19937_64 generator_; 84 | 85 | public: 86 | explicit Random64(uint64_t s) : generator_(s) { } 87 | 88 | // Generates the next random number 89 | uint64_t Next() { return generator_(); } 90 | 91 | // Returns a uniformly distributed value in the range [0..n-1] 92 | // REQUIRES: n > 0 93 | uint64_t Uniform(uint64_t n) { 94 | return std::uniform_int_distribution(0, n - 1)(generator_); 95 | } 96 | 97 | // Randomly returns true ~"1/n" of the time, and false otherwise. 98 | // REQUIRES: n > 0 99 | bool OneIn(uint64_t n) { return Uniform(n) == 0; } 100 | 101 | // Skewed: pick "base" uniformly from range [0,max_log] and then 102 | // return "base" random bits. The effect is to pick a number in the 103 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 104 | uint64_t Skewed(int max_log) { 105 | return Uniform(uint64_t(1) << Uniform(max_log + 1)); 106 | } 107 | }; 108 | 109 | } // namespace rocksutil 110 | -------------------------------------------------------------------------------- /rutil/sharded_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rutil/sharded_cache.h" 11 | #include "rocksutil/mutexlock.h" 12 | 13 | namespace rocksutil { 14 | 15 | ShardedCache::ShardedCache(size_t capacity, int num_shard_bits, 16 | bool strict_capacity_limit) 17 | : num_shard_bits_(num_shard_bits), 18 | capacity_(capacity), 19 | strict_capacity_limit_(strict_capacity_limit), 20 | last_id_(1) {} 21 | 22 | void ShardedCache::SetCapacity(size_t capacity) { 23 | int num_shards = 1 << num_shard_bits_; 24 | const size_t per_shard = (capacity + (num_shards - 1)) / num_shards; 25 | MutexLock l(&capacity_mutex_); 26 | for (int s = 0; s < num_shards; s++) { 27 | GetShard(s)->SetCapacity(per_shard); 28 | } 29 | capacity_ = capacity; 30 | } 31 | 32 | void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) { 33 | int num_shards = 1 << num_shard_bits_; 34 | MutexLock l(&capacity_mutex_); 35 | for (int s = 0; s < num_shards; s++) { 36 | GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit); 37 | } 38 | strict_capacity_limit_ = strict_capacity_limit; 39 | } 40 | 41 | Status ShardedCache::Insert(const Slice& key, void* value, size_t charge, 42 | void (*deleter)(const Slice& key, void* value), 43 | Handle** handle, Priority priority) { 44 | uint32_t hash = HashSlice(key); 45 | return GetShard(Shard(hash)) 46 | ->Insert(key, hash, value, charge, deleter, handle, priority); 47 | } 48 | 49 | Cache::Handle* ShardedCache::Lookup(const Slice& key) { 50 | uint32_t hash = HashSlice(key); 51 | return GetShard(Shard(hash))->Lookup(key, hash); 52 | } 53 | 54 | void ShardedCache::Release(Handle* handle) { 55 | uint32_t hash = GetHash(handle); 56 | GetShard(Shard(hash))->Release(handle); 57 | } 58 | 59 | void ShardedCache::Erase(const Slice& key) { 60 | uint32_t hash = HashSlice(key); 61 | GetShard(Shard(hash))->Erase(key, hash); 62 | } 63 | 64 | uint64_t ShardedCache::NewId() { 65 | return last_id_.fetch_add(1, std::memory_order_relaxed); 66 | } 67 | 68 | size_t ShardedCache::GetCapacity() const { 69 | MutexLock l(&capacity_mutex_); 70 | return capacity_; 71 | } 72 | 73 | bool ShardedCache::HasStrictCapacityLimit() const { 74 | MutexLock l(&capacity_mutex_); 75 | return strict_capacity_limit_; 76 | } 77 | 78 | size_t ShardedCache::GetUsage() const { 79 | // We will not lock the cache when getting the usage from shards. 80 | int num_shards = 1 << num_shard_bits_; 81 | size_t usage = 0; 82 | for (int s = 0; s < num_shards; s++) { 83 | usage += GetShard(s)->GetUsage(); 84 | } 85 | return usage; 86 | } 87 | 88 | size_t ShardedCache::GetUsage(Handle* handle) const { 89 | return GetCharge(handle); 90 | } 91 | 92 | size_t ShardedCache::GetPinnedUsage() const { 93 | // We will not lock the cache when getting the usage from shards. 94 | int num_shards = 1 << num_shard_bits_; 95 | size_t usage = 0; 96 | for (int s = 0; s < num_shards; s++) { 97 | usage += GetShard(s)->GetPinnedUsage(); 98 | } 99 | return usage; 100 | } 101 | 102 | void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t), 103 | bool thread_safe) { 104 | int num_shards = 1 << num_shard_bits_; 105 | for (int s = 0; s < num_shards; s++) { 106 | GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe); 107 | } 108 | } 109 | 110 | void ShardedCache::EraseUnRefEntries() { 111 | int num_shards = 1 << num_shard_bits_; 112 | for (int s = 0; s < num_shards; s++) { 113 | GetShard(s)->EraseUnRefEntries(); 114 | } 115 | } 116 | 117 | } // namespace rocksutil 118 | -------------------------------------------------------------------------------- /rutil/sharded_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #pragma once 11 | 12 | #include 13 | 14 | #include "rport/port.h" 15 | #include "rocksutil/cache.h" 16 | #include "rocksutil/hash.h" 17 | 18 | namespace rocksutil { 19 | 20 | // Single cache shard interface. 21 | class CacheShard { 22 | public: 23 | CacheShard() = default; 24 | virtual ~CacheShard() = default; 25 | 26 | virtual Status Insert(const Slice& key, uint32_t hash, void* value, 27 | size_t charge, 28 | void (*deleter)(const Slice& key, void* value), 29 | Cache::Handle** handle, Cache::Priority priority) = 0; 30 | virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0; 31 | virtual void Release(Cache::Handle* handle) = 0; 32 | virtual void Erase(const Slice& key, uint32_t hash) = 0; 33 | virtual void SetCapacity(size_t capacity) = 0; 34 | virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0; 35 | virtual size_t GetUsage() const = 0; 36 | virtual size_t GetPinnedUsage() const = 0; 37 | virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t), 38 | bool thread_safe) = 0; 39 | virtual void EraseUnRefEntries() = 0; 40 | }; 41 | 42 | // Generic cache interface which shards cache by hash of keys. 2^num_shard_bits 43 | // shards will be created, with capacity split evenly to each of the shards. 44 | // Keys are sharded by the highest num_shard_bits bits of hash value. 45 | class ShardedCache : public Cache { 46 | public: 47 | ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit); 48 | virtual ~ShardedCache() = default; 49 | virtual const char* Name() const override = 0; 50 | virtual CacheShard* GetShard(int shard) = 0; 51 | virtual const CacheShard* GetShard(int shard) const = 0; 52 | virtual void* Value(Handle* handle) override = 0; 53 | virtual size_t GetCharge(Handle* handle) const = 0; 54 | virtual uint32_t GetHash(Handle* handle) const = 0; 55 | virtual void DisownData() override = 0; 56 | 57 | virtual void SetCapacity(size_t capacity) override; 58 | virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override; 59 | 60 | virtual Status Insert(const Slice& key, void* value, size_t charge, 61 | void (*deleter)(const Slice& key, void* value), 62 | Handle** handle, Priority priority) override; 63 | virtual Handle* Lookup(const Slice& key) override; 64 | virtual void Release(Handle* handle) override; 65 | virtual void Erase(const Slice& key) override; 66 | virtual uint64_t NewId() override; 67 | virtual size_t GetCapacity() const override; 68 | virtual bool HasStrictCapacityLimit() const override; 69 | virtual size_t GetUsage() const override; 70 | virtual size_t GetUsage(Handle* handle) const override; 71 | virtual size_t GetPinnedUsage() const override; 72 | virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t), 73 | bool thread_safe) override; 74 | virtual void EraseUnRefEntries() override; 75 | 76 | private: 77 | static inline uint32_t HashSlice(const Slice& s) { 78 | return Hash(s.data(), s.size(), 0); 79 | } 80 | 81 | uint32_t Shard(uint32_t hash) { 82 | // Note, hash >> 32 yields hash in gcc, not the zero we expect! 83 | return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0; 84 | } 85 | 86 | int num_shard_bits_; 87 | mutable port::Mutex capacity_mutex_; 88 | size_t capacity_; 89 | bool strict_capacity_limit_; 90 | std::atomic last_id_; 91 | }; 92 | 93 | } // namespace rocksutil 94 | -------------------------------------------------------------------------------- /rutil/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include 11 | #include "rport/port.h" 12 | #include "rocksutil/status.h" 13 | 14 | namespace rocksutil { 15 | 16 | const char* Status::CopyState(const char* state) { 17 | uint32_t size; 18 | memcpy(&size, state, sizeof(size)); 19 | char* result = new char[size + 4]; 20 | memcpy(result, state, size + 4); 21 | return result; 22 | } 23 | 24 | Status::Status(Code _code, SubCode _subcode, const Slice& msg, const Slice& msg2) 25 | : code_(_code), subcode_(_subcode) { 26 | assert(code_ != kOk); 27 | assert(subcode_ != kMaxSubCode); 28 | const uint32_t len1 = static_cast(msg.size()); 29 | const uint32_t len2 = static_cast(msg2.size()); 30 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 31 | char* result = new char[size + 4]; 32 | memcpy(result, &size, sizeof(size)); 33 | memcpy(result + 4, msg.data(), len1); 34 | if (len2) { 35 | result[4 + len1] = ':'; 36 | result[5 + len1] = ' '; 37 | memcpy(result + 6 + len1, msg2.data(), len2); 38 | } 39 | state_ = result; 40 | } 41 | 42 | std::string Status::ToString() const { 43 | char tmp[30]; 44 | const char* type; 45 | switch (code_) { 46 | case kOk: 47 | return "OK"; 48 | case kNotFound: 49 | type = "NotFound: "; 50 | break; 51 | case kCorruption: 52 | type = "Corruption: "; 53 | break; 54 | case kNotSupported: 55 | type = "Not implemented: "; 56 | break; 57 | case kInvalidArgument: 58 | type = "Invalid argument: "; 59 | break; 60 | case kIOError: 61 | type = "IO error: "; 62 | break; 63 | case kMergeInProgress: 64 | type = "Merge in progress: "; 65 | break; 66 | case kIncomplete: 67 | type = "Result incomplete: "; 68 | break; 69 | case kShutdownInProgress: 70 | type = "Shutdown in progress: "; 71 | break; 72 | case kTimedOut: 73 | type = "Operation timed out: "; 74 | break; 75 | case kAborted: 76 | type = "Operation aborted: "; 77 | break; 78 | case kBusy: 79 | type = "Resource busy: "; 80 | break; 81 | case kExpired: 82 | type = "Operation expired: "; 83 | break; 84 | case kTryAgain: 85 | type = "Operation failed. Try again.: "; 86 | break; 87 | default: 88 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 89 | static_cast(code())); 90 | type = tmp; 91 | break; 92 | } 93 | std::string result(type); 94 | if (subcode_ != kNone) { 95 | uint32_t index = static_cast(subcode_); 96 | assert(sizeof(msgs) > index); 97 | result.append(msgs[index]); 98 | } 99 | 100 | if (state_ != nullptr) { 101 | uint32_t length; 102 | memcpy(&length, state_, sizeof(length)); 103 | result.append(state_ + 4, length); 104 | } 105 | return result; 106 | } 107 | 108 | } // namespace rocksutil 109 | -------------------------------------------------------------------------------- /rutil/status_message.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | 6 | #include "rocksutil/status.h" 7 | 8 | namespace rocksutil { 9 | 10 | const char* Status::msgs[] = { 11 | "", // kNone 12 | "Timeout Acquiring Mutex", // kMutexTimeout 13 | "Timeout waiting to lock key", // kLockTimeout 14 | "Failed to acquire lock due to max_num_locks limit", // kLockLimit 15 | "No space left on device" // kNoSpace 16 | }; 17 | 18 | } // namespace rocksutil 19 | -------------------------------------------------------------------------------- /rutil/string_util.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | #include 7 | #include 8 | #include 9 | #include "rutil/string_util.h" 10 | 11 | namespace rocksutil { 12 | 13 | std::vector StringSplit(const std::string& arg, char delim) { 14 | std::vector splits; 15 | std::stringstream ss(arg); 16 | std::string item; 17 | while (std::getline(ss, item, delim)) { 18 | splits.push_back(item); 19 | } 20 | return splits; 21 | } 22 | 23 | } // namespace rocksutil 24 | -------------------------------------------------------------------------------- /rutil/string_util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | 7 | #pragma once 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace rocksutil { 14 | 15 | extern std::vector StringSplit(const std::string& arg, char delim); 16 | 17 | template 18 | inline std::string ToString(T value) { 19 | #if !(defined OS_ANDROID) && !(defined CYGWIN) && !(defined OS_FREEBSD) 20 | return std::to_string(value); 21 | #else 22 | // Andorid or cygwin doesn't support all of C++11, std::to_string() being 23 | // one of the not supported features. 24 | std::ostringstream os; 25 | os << value; 26 | return os.str(); 27 | #endif 28 | } 29 | 30 | } // namespace rocksutil 31 | -------------------------------------------------------------------------------- /rutil/threadpool.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | 11 | namespace rocksutil { 12 | 13 | /* 14 | * ThreadPool is a component that will spawn N background threads that will 15 | * be used to execute scheduled work, The number of background threads could 16 | * be modified by calling SetBackgroundThreads(). 17 | * */ 18 | class ThreadPool { 19 | public: 20 | virtual ~ThreadPool() {} 21 | 22 | // Wait for all threads to finish. 23 | virtual void JoinAllThreads() = 0; 24 | 25 | // Set the number of background threads that will be executing the 26 | // scheduled jobs. 27 | virtual void SetBackgroundThreads(int num) = 0; 28 | 29 | // Get the number of jobs scheduled in the ThreadPool queue. 30 | virtual unsigned int GetQueueLen() const = 0; 31 | }; 32 | 33 | // NewThreadPool() is a function that could be used to create a ThreadPool 34 | // with `num_threads` background threads. 35 | extern ThreadPool* NewThreadPool(int num_threads); 36 | 37 | } // namespace rocksutil 38 | -------------------------------------------------------------------------------- /rutil/threadpool_imp.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | 10 | #include "rutil/threadpool_imp.h" 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #ifdef OS_LINUX 17 | # include 18 | #endif 19 | 20 | #ifdef OS_FREEBSD 21 | # include 22 | #endif 23 | 24 | 25 | namespace rocksutil { 26 | 27 | void ThreadPoolImpl::PthreadCall(const char* label, int result) { 28 | if (result != 0) { 29 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 30 | abort(); 31 | } 32 | } 33 | 34 | namespace { 35 | 36 | using Lock = pthread_mutex_t&; 37 | using Condition = pthread_cond_t&; 38 | 39 | inline int ThreadPoolMutexLock(Lock mutex) { 40 | return pthread_mutex_lock(&mutex); 41 | } 42 | 43 | inline 44 | int ConditionWait(Condition condition, Lock lock) { 45 | return pthread_cond_wait(&condition, &lock); 46 | } 47 | 48 | inline 49 | int ConditionSignalAll(Condition condition) { 50 | return pthread_cond_broadcast(&condition); 51 | } 52 | 53 | inline 54 | int ConditionSignal(Condition condition) { 55 | return pthread_cond_signal(&condition); 56 | } 57 | 58 | inline 59 | int MutexUnlock(Lock mutex) { 60 | return pthread_mutex_unlock(&mutex); 61 | } 62 | 63 | inline 64 | void ThreadJoin(pthread_t& thread) { 65 | pthread_join(thread, nullptr); 66 | } 67 | 68 | inline 69 | int ThreadDetach(pthread_t& thread) { 70 | return pthread_detach(thread); 71 | } 72 | } 73 | 74 | ThreadPoolImpl::ThreadPoolImpl() 75 | : total_threads_limit_(1), 76 | bgthreads_(0), 77 | queue_(), 78 | queue_len_(), 79 | exit_all_threads_(false), 80 | low_io_priority_(false), 81 | env_(nullptr) { 82 | PthreadCall("mutex_init", pthread_mutex_init(&mu_, nullptr)); 83 | PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, nullptr)); 84 | } 85 | 86 | ThreadPoolImpl::~ThreadPoolImpl() { assert(bgthreads_.size() == 0U); } 87 | 88 | void ThreadPoolImpl::JoinAllThreads() { 89 | Lock lock(mu_); 90 | PthreadCall("lock", ThreadPoolMutexLock(lock)); 91 | assert(!exit_all_threads_); 92 | exit_all_threads_ = true; 93 | PthreadCall("signalall", ConditionSignalAll(bgsignal_)); 94 | PthreadCall("unlock", MutexUnlock(lock)); 95 | 96 | for (auto& th : bgthreads_) { 97 | ThreadJoin(th); 98 | } 99 | 100 | bgthreads_.clear(); 101 | } 102 | 103 | void ThreadPoolImpl::LowerIOPriority() { 104 | #ifdef OS_LINUX 105 | PthreadCall("lock", pthread_mutex_lock(&mu_)); 106 | low_io_priority_ = true; 107 | PthreadCall("unlock", pthread_mutex_unlock(&mu_)); 108 | #endif 109 | } 110 | 111 | void ThreadPoolImpl::BGThread(size_t thread_id) { 112 | bool low_io_priority = false; 113 | while (true) { 114 | // Wait until there is an item that is ready to run 115 | Lock uniqueLock(mu_); 116 | PthreadCall("lock", ThreadPoolMutexLock(uniqueLock)); 117 | // Stop waiting if the thread needs to do work or needs to terminate. 118 | while (!exit_all_threads_ && !IsLastExcessiveThread(thread_id) && 119 | (queue_.empty() || IsExcessiveThread(thread_id))) { 120 | PthreadCall("wait", ConditionWait(bgsignal_, uniqueLock)); 121 | } 122 | 123 | if (exit_all_threads_) { // mechanism to let BG threads exit safely 124 | PthreadCall("unlock", MutexUnlock(uniqueLock)); 125 | break; 126 | } 127 | 128 | if (IsLastExcessiveThread(thread_id)) { 129 | // Current thread is the last generated one and is excessive. 130 | // We always terminate excessive thread in the reverse order of 131 | // generation time. 132 | auto& terminating_thread = bgthreads_.back(); 133 | PthreadCall("detach", ThreadDetach(terminating_thread)); 134 | bgthreads_.pop_back(); 135 | if (HasExcessiveThread()) { 136 | // There is still at least more excessive thread to terminate. 137 | WakeUpAllThreads(); 138 | } 139 | PthreadCall("unlock", MutexUnlock(uniqueLock)); 140 | break; 141 | } 142 | 143 | void (*function)(void*) = queue_.front().function; 144 | void* arg = queue_.front().arg; 145 | queue_.pop_front(); 146 | queue_len_.store(static_cast(queue_.size()), 147 | std::memory_order_relaxed); 148 | 149 | bool decrease_io_priority = (low_io_priority != low_io_priority_); 150 | PthreadCall("unlock", MutexUnlock(uniqueLock)); 151 | 152 | #ifdef OS_LINUX 153 | if (decrease_io_priority) { 154 | #define IOPRIO_CLASS_SHIFT (13) 155 | #define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) 156 | // Put schedule into IOPRIO_CLASS_IDLE class (lowest) 157 | // These system calls only have an effect when used in conjunction 158 | // with an I/O scheduler that supports I/O priorities. As at 159 | // kernel 2.6.17 the only such scheduler is the Completely 160 | // Fair Queuing (CFQ) I/O scheduler. 161 | // To change scheduler: 162 | // echo cfq > /sys/block//queue/schedule 163 | // Tunables to consider: 164 | // /sys/block//queue/slice_idle 165 | // /sys/block//queue/slice_sync 166 | syscall(SYS_ioprio_set, 1, // IOPRIO_WHO_PROCESS 167 | 0, // current thread 168 | IOPRIO_PRIO_VALUE(3, 0)); 169 | low_io_priority = true; 170 | } 171 | #else 172 | (void)decrease_io_priority; // avoid 'unused variable' error 173 | #endif 174 | (*function)(arg); 175 | } 176 | } 177 | 178 | // Helper struct for passing arguments when creating threads. 179 | struct BGThreadMetadata { 180 | ThreadPoolImpl* thread_pool_; 181 | size_t thread_id_; // Thread count in the thread. 182 | BGThreadMetadata(ThreadPoolImpl* thread_pool, size_t thread_id) 183 | : thread_pool_(thread_pool), thread_id_(thread_id) {} 184 | }; 185 | 186 | static void* BGThreadWrapper(void* arg) { 187 | BGThreadMetadata* meta = reinterpret_cast(arg); 188 | size_t thread_id = meta->thread_id_; 189 | ThreadPoolImpl* tp = meta->thread_pool_; 190 | delete meta; 191 | tp->BGThread(thread_id); 192 | return nullptr; 193 | } 194 | 195 | void ThreadPoolImpl::WakeUpAllThreads() { 196 | PthreadCall("signalall", ConditionSignalAll(bgsignal_)); 197 | } 198 | 199 | void ThreadPoolImpl::SetBackgroundThreadsInternal(int num, bool allow_reduce) { 200 | Lock lock(mu_); 201 | PthreadCall("lock", ThreadPoolMutexLock(lock)); 202 | if (exit_all_threads_) { 203 | PthreadCall("unlock", MutexUnlock(lock)); 204 | return; 205 | } 206 | if (num > total_threads_limit_ || 207 | (num < total_threads_limit_ && allow_reduce)) { 208 | total_threads_limit_ = std::max(1, num); 209 | WakeUpAllThreads(); 210 | StartBGThreads(); 211 | } 212 | PthreadCall("unlock", MutexUnlock(lock)); 213 | } 214 | 215 | void ThreadPoolImpl::IncBackgroundThreadsIfNeeded(int num) { 216 | SetBackgroundThreadsInternal(num, false); 217 | } 218 | 219 | void ThreadPoolImpl::SetBackgroundThreads(int num) { 220 | SetBackgroundThreadsInternal(num, true); 221 | } 222 | 223 | void ThreadPoolImpl::StartBGThreads() { 224 | // Start background thread if necessary 225 | while ((int)bgthreads_.size() < total_threads_limit_) { 226 | pthread_t t; 227 | PthreadCall("create thread", 228 | pthread_create(&t, nullptr, &BGThreadWrapper, 229 | new BGThreadMetadata(this, bgthreads_.size()))); 230 | // Set the thread name to aid debugging 231 | #if defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ) 232 | #if __GLIBC_PREREQ(2, 12) 233 | char name_buf[16]; 234 | snprintf(name_buf, sizeof name_buf, "rocksutil:bg%" ROCKSUTIL_PRIszt, 235 | bgthreads_.size()); 236 | name_buf[sizeof name_buf - 1] = '\0'; 237 | pthread_setname_np(t, name_buf); 238 | #endif 239 | #endif 240 | bgthreads_.push_back(t); 241 | } 242 | } 243 | 244 | void ThreadPoolImpl::Schedule(void (*function)(void* arg1), void* arg, 245 | void* tag, void (*unschedFunction)(void* arg)) { 246 | Lock lock(mu_); 247 | PthreadCall("lock", ThreadPoolMutexLock(lock)); 248 | 249 | if (exit_all_threads_) { 250 | PthreadCall("unlock", MutexUnlock(lock)); 251 | return; 252 | } 253 | 254 | StartBGThreads(); 255 | 256 | // Add to priority queue 257 | queue_.push_back(BGItem()); 258 | queue_.back().function = function; 259 | queue_.back().arg = arg; 260 | queue_.back().tag = tag; 261 | queue_.back().unschedFunction = unschedFunction; 262 | queue_len_.store(static_cast(queue_.size()), 263 | std::memory_order_relaxed); 264 | 265 | if (!HasExcessiveThread()) { 266 | // Wake up at least one waiting thread. 267 | PthreadCall("signal", ConditionSignal(bgsignal_)); 268 | } else { 269 | // Need to wake up all threads to make sure the one woken 270 | // up is not the one to terminate. 271 | WakeUpAllThreads(); 272 | } 273 | 274 | PthreadCall("unlock", MutexUnlock(lock)); 275 | } 276 | 277 | int ThreadPoolImpl::UnSchedule(void* arg) { 278 | int count = 0; 279 | 280 | Lock lock(mu_); 281 | PthreadCall("lock", ThreadPoolMutexLock(lock)); 282 | 283 | // Remove from priority queue 284 | BGQueue::iterator it = queue_.begin(); 285 | while (it != queue_.end()) { 286 | if (arg == (*it).tag) { 287 | void (*unschedFunction)(void*) = (*it).unschedFunction; 288 | void* arg1 = (*it).arg; 289 | if (unschedFunction != nullptr) { 290 | (*unschedFunction)(arg1); 291 | } 292 | it = queue_.erase(it); 293 | count++; 294 | } else { 295 | ++it; 296 | } 297 | } 298 | queue_len_.store(static_cast(queue_.size()), 299 | std::memory_order_relaxed); 300 | PthreadCall("unlock", MutexUnlock(lock)); 301 | return count; 302 | } 303 | 304 | ThreadPool* NewThreadPool(int num_threads) { 305 | ThreadPoolImpl* thread_pool = new ThreadPoolImpl(); 306 | thread_pool->SetBackgroundThreads(num_threads); 307 | return thread_pool; 308 | } 309 | 310 | } // namespace rocksutil 311 | -------------------------------------------------------------------------------- /rutil/threadpool_imp.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. 2 | // This source code is licensed under the BSD-style license found in the 3 | // LICENSE file in the root directory of this source tree. An additional grant 4 | // of patent rights can be found in the PATENTS file in the same directory. 5 | // 6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style license that can be 8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 9 | #pragma once 10 | 11 | #include "rocksutil/env.h" 12 | #include "rutil/threadpool.h" 13 | #include "rport/port.h" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace rocksutil { 20 | 21 | class ThreadPoolImpl : public ThreadPool { 22 | public: 23 | ThreadPoolImpl(); 24 | ~ThreadPoolImpl(); 25 | 26 | void JoinAllThreads() override; 27 | void LowerIOPriority(); 28 | void BGThread(size_t thread_id); 29 | void WakeUpAllThreads(); 30 | void IncBackgroundThreadsIfNeeded(int num); 31 | void SetBackgroundThreads(int num) override; 32 | void StartBGThreads(); 33 | void Schedule(void (*function)(void* arg1), void* arg, void* tag, 34 | void (*unschedFunction)(void* arg)); 35 | int UnSchedule(void* arg); 36 | 37 | unsigned int GetQueueLen() const override { 38 | return queue_len_.load(std::memory_order_relaxed); 39 | } 40 | 41 | void SetHostEnv(Env* env) { env_ = env; } 42 | Env* GetHostEnv() const { return env_; } 43 | 44 | // Return true if there is at least one thread needs to terminate. 45 | bool HasExcessiveThread() const { 46 | return static_cast(bgthreads_.size()) > total_threads_limit_; 47 | } 48 | 49 | // Return true iff the current thread is the excessive thread to terminate. 50 | // Always terminate the running thread that is added last, even if there are 51 | // more than one thread to terminate. 52 | bool IsLastExcessiveThread(size_t thread_id) const { 53 | return HasExcessiveThread() && thread_id == bgthreads_.size() - 1; 54 | } 55 | 56 | // Is one of the threads to terminate. 57 | bool IsExcessiveThread(size_t thread_id) const { 58 | return static_cast(thread_id) >= total_threads_limit_; 59 | } 60 | 61 | // Return the thread priority. 62 | // This would allow its member-thread to know its priority. 63 | Env::Priority GetThreadPriority() const { return priority_; } 64 | 65 | // Set the thread priority. 66 | void SetThreadPriority(Env::Priority priority) { priority_ = priority; } 67 | 68 | static void PthreadCall(const char* label, int result); 69 | 70 | private: 71 | // Entry per Schedule() call 72 | struct BGItem { 73 | void* arg; 74 | void (*function)(void*); 75 | void* tag; 76 | void (*unschedFunction)(void*); 77 | }; 78 | 79 | typedef std::deque BGQueue; 80 | 81 | int total_threads_limit_; 82 | 83 | pthread_mutex_t mu_; 84 | pthread_cond_t bgsignal_; 85 | std::vector bgthreads_; 86 | BGQueue queue_; 87 | std::atomic_uint queue_len_; // Queue length. Used for stats reporting 88 | bool exit_all_threads_; 89 | bool low_io_priority_; 90 | Env::Priority priority_; 91 | Env* env_; 92 | 93 | void SetBackgroundThreadsInternal(int num, bool allow_reduce); 94 | }; 95 | 96 | } // namespace rocksutil 97 | -------------------------------------------------------------------------------- /src.mk: -------------------------------------------------------------------------------- 1 | LIB_SOURCES = \ 2 | rutil/build_version.cc \ 3 | rutil/coding.cc \ 4 | rutil/env.cc \ 5 | rutil/env_posix.cc \ 6 | rutil/io_posix.cc \ 7 | rutil/random.cc \ 8 | rutil/status.cc \ 9 | rutil/status_message.cc \ 10 | rutil/string_util.cc \ 11 | rutil/thread_local.cc \ 12 | rutil/threadpool_imp.cc \ 13 | rutil/auto_roll_logger.cc \ 14 | rutil/sharded_cache.cc \ 15 | rutil/lru_cache.cc \ 16 | rutil/hash.cc \ 17 | rutil/file_reader_writer.cc \ 18 | rutil/crc32c.cc \ 19 | rutil/log_writer.cc \ 20 | rutil/log_reader.cc \ 21 | rport/port_posix.cc 22 | 23 | EXAMPLE_SOURCE = \ 24 | examples/log_example.cc \ 25 | examples/thread_local_example.cc \ 26 | examples/mutexlock_example.cc \ 27 | examples/thread_pool_example.cc 28 | --------------------------------------------------------------------------------