├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── Makefile ├── README.md ├── db ├── db_impl.cc └── db_impl.h ├── file ├── file.h ├── posix_file.cc └── posix_file.h ├── include ├── comparator.h ├── db.h ├── env.h ├── index.h ├── options.h └── skiplist.h ├── port └── atomic_pointer.h ├── test ├── CMakeLists.txt └── dbimpl_test.cc └── util ├── arena.cc ├── arena.h ├── coding.cc ├── coding.h ├── hash.h ├── lock.h └── random.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs 2 | *.so 3 | .idea/ 4 | *mydb/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | lanauage: cpp 2 | 3 | install: 4 | - sudo apt-get install -y libgtest-dev cmake 5 | - cd /usr/src/gtest 6 | - sudo cmake . 7 | - sudo make 8 | - sudo mv libgtest* /usr/lib/ 9 | - cd "${TRAVIS_BUILD_DIR}" 10 | 11 | os: 12 | - linux 13 | 14 | compiler: 15 | - gcc 16 | 17 | script: 18 | - cmake . 19 | - make 20 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | 3 | project(atendb CXX) 4 | 5 | if(NOT CMAKE_BUILD_TYPE) 6 | set(CMAKE_BUILD_TYPE "Debug") 7 | endif() 8 | 9 | set(CXX_FLAGS 10 | -g 11 | -Wall 12 | -march=native 13 | -rdynamic 14 | -std=c++0x 15 | ) 16 | 17 | string(REPLACE ";" " " CMAKE_CXX_FLAGS "${CXX_FLAGS}") 18 | 19 | set(CMAKE_CXX_COMPILER "g++") 20 | set(CMAKE_CXX_FLAGS_DEBUG "-std=c++11 -g -O0") 21 | 22 | string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE) 23 | message(STATUS "CXX_FLAGS = " ${CMAKE_CXX_FLAGS} " " ${CMAKE_CXX_FLAGS_${BUILD_TYPE}}) 24 | 25 | find_package(GTest REQUIRED) 26 | 27 | include_directories(${PROJECT_SOURCE_DIR}) 28 | include_directories(${PROJECT_SOURCE_DIR}/include) 29 | 30 | set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib) 31 | set(EXECUTABLE_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/bin) 32 | 33 | file(GLOB_RECURSE SOURCE 34 | ${CMAKE_CURRENT_SOURCE_DIR} 35 | ${CMAKE_CURRENT_SOURCE_DIR}/db/*.cc 36 | ${CMAKE_CURRENT_SOURCE_DIR}/port/*.cc 37 | ${CMAKE_CURRENT_SOURCE_DIR}/util/*.cc 38 | ${CMAKE_CURRENT_SOURCE_DIR}/file/*.cc 39 | ) 40 | 41 | add_library(atendb ${SOURCE}) 42 | target_link_libraries(atendb pthread rt) 43 | set_target_properties(atendb PROPERTIES OUTPUT_NAME atendb) 44 | install(TARGETS atendb DESTINATION lib) 45 | 46 | add_subdirectory(test) 47 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # CMAKE generated file: DO NOT EDIT! 2 | # Generated by "Unix Makefiles" Generator, CMake Version 3.12 3 | 4 | # Default target executed when no arguments are given to make. 5 | default_target: all 6 | 7 | .PHONY : default_target 8 | 9 | # Allow only one "make -f Makefile2" at a time, but pass parallelism. 10 | .NOTPARALLEL: 11 | 12 | 13 | #============================================================================= 14 | # Special targets provided by cmake. 15 | 16 | # Disable implicit rules so canonical targets will work. 17 | .SUFFIXES: 18 | 19 | 20 | # Remove some rules from gmake that .SUFFIXES does not remove. 21 | SUFFIXES = 22 | 23 | .SUFFIXES: .hpux_make_needs_suffix_list 24 | 25 | 26 | # Suppress display of executed commands. 27 | $(VERBOSE).SILENT: 28 | 29 | 30 | # A target that is always out of date. 31 | cmake_force: 32 | 33 | .PHONY : cmake_force 34 | 35 | #============================================================================= 36 | # Set environment variables for the build. 37 | 38 | # The shell in which to execute make rules. 39 | SHELL = /bin/sh 40 | 41 | # The CMake executable. 42 | CMAKE_COMMAND = /usr/bin/cmake 43 | 44 | # The command to remove a file. 45 | RM = /usr/bin/cmake -E remove -f 46 | 47 | # Escaping for special characters. 48 | EQUALS = = 49 | 50 | # The top-level source directory on which CMake was run. 51 | CMAKE_SOURCE_DIR = /home/leviathan/dev/Biu 52 | 53 | # The top-level build directory on which CMake was run. 54 | CMAKE_BINARY_DIR = /home/leviathan/dev/Biu 55 | 56 | #============================================================================= 57 | # Targets provided globally by CMake. 58 | 59 | # Special rule for the target install/strip 60 | install/strip: preinstall 61 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..." 62 | /usr/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake 63 | .PHONY : install/strip 64 | 65 | # Special rule for the target install/strip 66 | install/strip/fast: preinstall/fast 67 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..." 68 | /usr/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake 69 | .PHONY : install/strip/fast 70 | 71 | # Special rule for the target edit_cache 72 | edit_cache: 73 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake cache editor..." 74 | /usr/bin/ccmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) 75 | .PHONY : edit_cache 76 | 77 | # Special rule for the target edit_cache 78 | edit_cache/fast: edit_cache 79 | 80 | .PHONY : edit_cache/fast 81 | 82 | # Special rule for the target rebuild_cache 83 | rebuild_cache: 84 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..." 85 | /usr/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) 86 | .PHONY : rebuild_cache 87 | 88 | # Special rule for the target rebuild_cache 89 | rebuild_cache/fast: rebuild_cache 90 | 91 | .PHONY : rebuild_cache/fast 92 | 93 | # Special rule for the target list_install_components 94 | list_install_components: 95 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Available install components are: \"Unspecified\"" 96 | .PHONY : list_install_components 97 | 98 | # Special rule for the target list_install_components 99 | list_install_components/fast: list_install_components 100 | 101 | .PHONY : list_install_components/fast 102 | 103 | # Special rule for the target install/local 104 | install/local: preinstall 105 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..." 106 | /usr/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake 107 | .PHONY : install/local 108 | 109 | # Special rule for the target install/local 110 | install/local/fast: preinstall/fast 111 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..." 112 | /usr/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake 113 | .PHONY : install/local/fast 114 | 115 | # Special rule for the target install 116 | install: preinstall 117 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..." 118 | /usr/bin/cmake -P cmake_install.cmake 119 | .PHONY : install 120 | 121 | # Special rule for the target install 122 | install/fast: preinstall/fast 123 | @$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..." 124 | /usr/bin/cmake -P cmake_install.cmake 125 | .PHONY : install/fast 126 | 127 | # The main all target 128 | all: cmake_check_build_system 129 | $(CMAKE_COMMAND) -E cmake_progress_start /home/leviathan/dev/Biu/CMakeFiles /home/leviathan/dev/Biu/CMakeFiles/progress.marks 130 | $(MAKE) -f CMakeFiles/Makefile2 all 131 | $(CMAKE_COMMAND) -E cmake_progress_start /home/leviathan/dev/Biu/CMakeFiles 0 132 | .PHONY : all 133 | 134 | # The main clean target 135 | clean: 136 | $(MAKE) -f CMakeFiles/Makefile2 clean 137 | .PHONY : clean 138 | 139 | # The main clean target 140 | clean/fast: clean 141 | 142 | .PHONY : clean/fast 143 | 144 | # Prepare targets for installation. 145 | preinstall: all 146 | $(MAKE) -f CMakeFiles/Makefile2 preinstall 147 | .PHONY : preinstall 148 | 149 | # Prepare targets for installation. 150 | preinstall/fast: 151 | $(MAKE) -f CMakeFiles/Makefile2 preinstall 152 | .PHONY : preinstall/fast 153 | 154 | # clear depends 155 | depend: 156 | $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 1 157 | .PHONY : depend 158 | 159 | #============================================================================= 160 | # Target rules for targets named atendb 161 | 162 | # Build rule for target. 163 | atendb: cmake_check_build_system 164 | $(MAKE) -f CMakeFiles/Makefile2 atendb 165 | .PHONY : atendb 166 | 167 | # fast build rule for target. 168 | atendb/fast: 169 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/build 170 | .PHONY : atendb/fast 171 | 172 | #============================================================================= 173 | # Target rules for targets named db 174 | 175 | # Build rule for target. 176 | db: cmake_check_build_system 177 | $(MAKE) -f CMakeFiles/Makefile2 db 178 | .PHONY : db 179 | 180 | # fast build rule for target. 181 | db/fast: 182 | $(MAKE) -f test/CMakeFiles/db.dir/build.make test/CMakeFiles/db.dir/build 183 | .PHONY : db/fast 184 | 185 | db/db_impl.o: db/db_impl.cc.o 186 | 187 | .PHONY : db/db_impl.o 188 | 189 | # target to build an object file 190 | db/db_impl.cc.o: 191 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/db/db_impl.cc.o 192 | .PHONY : db/db_impl.cc.o 193 | 194 | db/db_impl.i: db/db_impl.cc.i 195 | 196 | .PHONY : db/db_impl.i 197 | 198 | # target to preprocess a source file 199 | db/db_impl.cc.i: 200 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/db/db_impl.cc.i 201 | .PHONY : db/db_impl.cc.i 202 | 203 | db/db_impl.s: db/db_impl.cc.s 204 | 205 | .PHONY : db/db_impl.s 206 | 207 | # target to generate assembly for a file 208 | db/db_impl.cc.s: 209 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/db/db_impl.cc.s 210 | .PHONY : db/db_impl.cc.s 211 | 212 | file/posix_file.o: file/posix_file.cc.o 213 | 214 | .PHONY : file/posix_file.o 215 | 216 | # target to build an object file 217 | file/posix_file.cc.o: 218 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/file/posix_file.cc.o 219 | .PHONY : file/posix_file.cc.o 220 | 221 | file/posix_file.i: file/posix_file.cc.i 222 | 223 | .PHONY : file/posix_file.i 224 | 225 | # target to preprocess a source file 226 | file/posix_file.cc.i: 227 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/file/posix_file.cc.i 228 | .PHONY : file/posix_file.cc.i 229 | 230 | file/posix_file.s: file/posix_file.cc.s 231 | 232 | .PHONY : file/posix_file.s 233 | 234 | # target to generate assembly for a file 235 | file/posix_file.cc.s: 236 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/file/posix_file.cc.s 237 | .PHONY : file/posix_file.cc.s 238 | 239 | util/arena.o: util/arena.cc.o 240 | 241 | .PHONY : util/arena.o 242 | 243 | # target to build an object file 244 | util/arena.cc.o: 245 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/arena.cc.o 246 | .PHONY : util/arena.cc.o 247 | 248 | util/arena.i: util/arena.cc.i 249 | 250 | .PHONY : util/arena.i 251 | 252 | # target to preprocess a source file 253 | util/arena.cc.i: 254 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/arena.cc.i 255 | .PHONY : util/arena.cc.i 256 | 257 | util/arena.s: util/arena.cc.s 258 | 259 | .PHONY : util/arena.s 260 | 261 | # target to generate assembly for a file 262 | util/arena.cc.s: 263 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/arena.cc.s 264 | .PHONY : util/arena.cc.s 265 | 266 | util/coding.o: util/coding.cc.o 267 | 268 | .PHONY : util/coding.o 269 | 270 | # target to build an object file 271 | util/coding.cc.o: 272 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/coding.cc.o 273 | .PHONY : util/coding.cc.o 274 | 275 | util/coding.i: util/coding.cc.i 276 | 277 | .PHONY : util/coding.i 278 | 279 | # target to preprocess a source file 280 | util/coding.cc.i: 281 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/coding.cc.i 282 | .PHONY : util/coding.cc.i 283 | 284 | util/coding.s: util/coding.cc.s 285 | 286 | .PHONY : util/coding.s 287 | 288 | # target to generate assembly for a file 289 | util/coding.cc.s: 290 | $(MAKE) -f CMakeFiles/atendb.dir/build.make CMakeFiles/atendb.dir/util/coding.cc.s 291 | .PHONY : util/coding.cc.s 292 | 293 | # Help Target 294 | help: 295 | @echo "The following are some of the valid targets for this Makefile:" 296 | @echo "... all (the default if no target is provided)" 297 | @echo "... clean" 298 | @echo "... depend" 299 | @echo "... install/strip" 300 | @echo "... edit_cache" 301 | @echo "... rebuild_cache" 302 | @echo "... atendb" 303 | @echo "... list_install_components" 304 | @echo "... install/local" 305 | @echo "... install" 306 | @echo "... db" 307 | @echo "... db/db_impl.o" 308 | @echo "... db/db_impl.i" 309 | @echo "... db/db_impl.s" 310 | @echo "... file/posix_file.o" 311 | @echo "... file/posix_file.i" 312 | @echo "... file/posix_file.s" 313 | @echo "... util/arena.o" 314 | @echo "... util/arena.i" 315 | @echo "... util/arena.s" 316 | @echo "... util/coding.o" 317 | @echo "... util/coding.i" 318 | @echo "... util/coding.s" 319 | .PHONY : help 320 | 321 | 322 | 323 | #============================================================================= 324 | # Special targets to cleanup operation of make. 325 | 326 | # Special rule to run CMake to check the build system integrity. 327 | # No rule that depends on this can have commands that come from listfiles 328 | # because they might be regenerated. 329 | cmake_check_build_system: 330 | $(CMAKE_COMMAND) -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR) --check-build-system CMakeFiles/Makefile.cmake 0 331 | .PHONY : cmake_check_build_system 332 | 333 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Atendb 2 | [![Build Status](https://travis-ci.org/Leviathan1995/Atendb.svg?branch=master)](https://travis-ci.org/Leviathan1995/Atendb) 3 | 4 | Atendb is a key-value storage that provides an API for storing and retrieve key/value data. The design of Atendb was inspired by [Bitcask](http://basho.com/wp-content/uploads/2015/05/bitcask-intro.pdf) 5 | 6 | ## TODO: 7 | * Range() 8 | * Compact() 9 | 10 | ## License: 11 | 12 | MIT 13 | -------------------------------------------------------------------------------- /db/db_impl.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #include 4 | #include 5 | 6 | #include "db/db_impl.h" 7 | #include "util/coding.h" 8 | #include "util/hash.h" 9 | 10 | namespace atendb { 11 | 12 | DBImpl::DBImpl(const std::string& dbname, const Options& options) : 13 | dbname_(dbname), 14 | options_(options), 15 | cursor_(0), 16 | max_file_(0) { } 17 | 18 | DBImpl::~DBImpl() { 19 | for (auto file : data_files_) { 20 | delete file; 21 | } 22 | for (auto file : index_files_) { 23 | delete file; 24 | } 25 | for (auto table: tables_) { 26 | delete table; 27 | } 28 | } 29 | 30 | bool DBImpl::Put(const std::string& key, const std::string& value) { 31 | uint32_t cursor = cursor_++; 32 | uint8_t file_index = (cursor & (max_file_ - 1)); 33 | File* data_file = data_files_[file_index]; 34 | uint64_t file_offset; 35 | auto s = data_file->AppendData(key, value, &file_offset); 36 | if (likely(s)) { 37 | File* index_file = index_files_[file_index]; 38 | s = index_file->AppendIndex(key, file_index, file_offset, 39 | key.size(), value.size()); 40 | if (likely(s)) { 41 | Index index(key, file_index, file_offset, key.size(), value.size()); 42 | int32_t slot = (Hash(key.c_str()) & (table_size_ - 1)); 43 | tables_[slot]->Insert(index); 44 | } 45 | } 46 | return s; 47 | } 48 | 49 | bool DBImpl::Get(const std::string& key, std::string* value) { 50 | int32_t slot = (Hash(key.data()) & (table_size_ - 1)); 51 | Index search_index(key), internal_index; 52 | auto s = tables_[slot]->Get(search_index, &internal_index); 53 | if (likely(s)) { 54 | if (unlikely(internal_index.KeySize() == 0 && 55 | internal_index.ValueSize() == 0)) { 56 | // The key has been deleted. 57 | *value = ""; 58 | return false; 59 | } 60 | char buf[internal_index.DataSize()]; 61 | auto s = data_files_[internal_index.FileIndex()]->Read(internal_index.FileOffset(), 62 | internal_index.DataSize(), 63 | buf); 64 | if (likely(s)) { 65 | DecodeData(buf, 66 | internal_index.key_size_, 67 | internal_index.value_size_, 68 | value); 69 | return true; 70 | } else { 71 | return s; 72 | } 73 | } else { 74 | return s; 75 | } 76 | } 77 | 78 | bool DBImpl::Delete(const std::string& key) { 79 | int32_t slot = (Hash(key.data()) & (table_size_ - 1)); 80 | Index search_index(key), internal_index; 81 | auto s = tables_[slot]->Get(search_index, &internal_index); 82 | if (likely(s)) { 83 | internal_index.SetDeleted(); 84 | tables_[slot]->Insert(internal_index); 85 | return s; 86 | } else { 87 | return s; 88 | } 89 | } 90 | 91 | void DBImpl::IndexCallback(File* file) { 92 | uint64_t file_offset = file->FileOffset(); 93 | if (file_offset != 0) { // Load index files into skiplist 94 | uint64_t pos = 0; 95 | while(pos < file_offset) { 96 | char size_buf[sizeof(uint32_t)]; 97 | auto s = file->Read(pos, sizeof(uint32_t), size_buf); 98 | if (s) { 99 | pos += sizeof(uint32_t); 100 | uint32_t index_size = DecodeFixed32(size_buf); 101 | char index_buf[index_size]; 102 | auto s = file->Read(pos, index_size, index_buf); 103 | if (s) { 104 | pos += index_size; 105 | Index index; 106 | DecodeIndex(index_buf, 107 | &index.key_, 108 | &index.file_index_, 109 | &index.file_offset_, 110 | &index.key_size_, 111 | &index.value_size_); 112 | int32_t slot = (Hash(index.key_.data()) & (table_size_ - 1)); 113 | tables_[slot]->Insert(index); 114 | } 115 | } 116 | } 117 | } 118 | } 119 | 120 | bool DBImpl::Recover() { 121 | // Recover 122 | std::vector threads; 123 | for (uint32_t i = 0; i < max_file_; i++) { 124 | std::thread recover(&DBImpl::IndexCallback, this, index_files_[i]); 125 | threads.push_back(std::move(recover)); 126 | } 127 | 128 | for (uint32_t i = 0; i < max_file_; i++) { 129 | threads[i].join(); 130 | } 131 | return true; 132 | } 133 | 134 | 135 | bool DB::Open(const std::string& dbname, 136 | const Options& options, 137 | DB** db) { 138 | *db = nullptr; 139 | DBImpl* impl = new DBImpl(dbname, options); 140 | 141 | // Get the number of processor 142 | int32_t num_processor = sysconf(_SC_NPROCESSORS_CONF); 143 | if (num_processor == -1) { 144 | return false; 145 | } 146 | 147 | impl->max_file_ = num_processor * 2; 148 | impl->table_size_ = num_processor; 149 | // Create skiplist tables 150 | for (uint32_t i = 0; i < impl->table_size_; ++i) { 151 | Table* table = new Table(Compare(options.comparator_)); 152 | impl->tables_.push_back(table); 153 | } 154 | 155 | // Create data directory 156 | std::string current_dir; 157 | if (!Env::GetCurrentDir(¤t_dir)) { 158 | return false; 159 | } 160 | 161 | std::string data_dir = current_dir + "/" + dbname; 162 | if (!Env::FileExists(data_dir)) { 163 | auto s = Env::CreateDir(data_dir); 164 | if (!s) { 165 | return false; 166 | } 167 | } 168 | 169 | // Open data 170 | for (uint32_t i = 1; i <= impl->max_file_; i++) { 171 | std::string data_file = data_dir + DataFileName + std::to_string(i); 172 | File* file; 173 | auto s = Env::NewFile(data_file, &file); 174 | if (s) { 175 | impl->data_files_.push_back(file); 176 | } else { 177 | return false; 178 | } 179 | } 180 | 181 | // Open index 182 | for (uint32_t i = 1; i <= impl->max_file_; i++) { 183 | std::string index_file = data_dir + IndexFileName + std::to_string(i); 184 | File* file; 185 | auto s = Env::NewFile(index_file, &file); 186 | if (s) { 187 | impl->index_files_.push_back(file); 188 | } else { 189 | return false; 190 | } 191 | } 192 | 193 | // Recover 194 | impl->Recover(); 195 | 196 | *db = impl; 197 | return true; 198 | } 199 | 200 | } // namespace atendb 201 | -------------------------------------------------------------------------------- /db/db_impl.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_DB_DB_IMPL_H_ 4 | #define ATENDB_DB_DB_IMPL_H_ 5 | 6 | #include 7 | #include 8 | 9 | #include "include/db.h" 10 | #include "include/options.h" 11 | #include "include/comparator.h" 12 | #include "include/skiplist.h" 13 | #include "util/arena.h" 14 | #include "file/file.h" 15 | #include "util/lock.h" 16 | 17 | namespace atendb { 18 | 19 | const std::string IndexFileName = "/INDEX-"; 20 | const std::string DataFileName = "/DATA-"; 21 | 22 | typedef SkipList Table; 23 | 24 | class DBImpl : public DB { 25 | public: 26 | DBImpl(const std::string& dbname, const Options& options); 27 | 28 | virtual ~DBImpl(); 29 | 30 | virtual bool Put(const std::string& key, const std::string& value); 31 | 32 | virtual bool Get(const std::string& key, std::string* value); 33 | 34 | virtual bool Delete(const std::string& key); 35 | 36 | void IndexCallback(File* file); 37 | 38 | bool Recover(); 39 | 40 | private: 41 | friend DB; 42 | 43 | std::string dbname_; 44 | Options options_; 45 | std::atomic cursor_; 46 | 47 | uint32_t max_file_; 48 | std::vector data_files_; 49 | std::vector index_files_; 50 | 51 | 52 | uint32_t table_size_; 53 | std::vector tables_; 54 | 55 | // No copying allowed 56 | void operator=(const DBImpl&); 57 | DBImpl(const DBImpl&); 58 | }; 59 | 60 | 61 | } // namespace atendb 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /file/file.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_FILE_FILE_H_ 4 | #define ATENDB_FILE_FILE_H_ 5 | 6 | #include 7 | 8 | namespace atendb { 9 | 10 | class File { 11 | public: 12 | File() { } 13 | virtual ~File() {} 14 | 15 | virtual bool Read(uint64_t offset, uint32_t n, char* buf) = 0; 16 | 17 | virtual bool AppendData(const std::string& key, 18 | const std::string& value, 19 | uint64_t* file_offset) = 0; 20 | 21 | virtual bool AppendIndex(const std::string& key, 22 | uint32_t file_index, 23 | uint64_t file_offset, 24 | uint32_t key_size_, 25 | uint32_t value_size_) = 0; 26 | 27 | virtual bool Close() = 0; 28 | 29 | virtual uint64_t FileOffset() = 0; 30 | }; 31 | 32 | } // namespace atendb 33 | 34 | #endif // ATENDB_FILE_FILE_H_ 35 | 36 | -------------------------------------------------------------------------------- /file/posix_file.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "posix_file.h" 12 | 13 | namespace atendb { 14 | 15 | PosixFile::~PosixFile() { 16 | if (fd_ > 0) { 17 | Close(); 18 | } 19 | } 20 | 21 | bool PosixFile::Read(uint64_t offset, uint32_t n, char* buf) { 22 | if (offset > file_offset_) { 23 | uint32_t pos = offset - file_offset_; 24 | if (pos > map_size_) { 25 | return false; 26 | } else { 27 | memcpy(buf, base_buf_+pos, n); 28 | } 29 | } else { 30 | size_t read = pread(fd_, buf, n, offset); 31 | if (read < 0) { 32 | return false; 33 | } 34 | } 35 | return true; 36 | } 37 | 38 | bool PosixFile::AppendData(const std::string& key, 39 | const std::string& value, 40 | uint64_t* file_offset) { 41 | uint32_t data_size = key.size() + value.size(); 42 | char data_buf[data_size]; 43 | EncodeData(data_buf, key, value); 44 | 45 | *file_offset = file_offset_.fetch_add(data_size); 46 | int64_t cnt = 0; 47 | while (cnt < data_size) { 48 | ssize_t n = pwrite(fd_, data_buf, data_size - cnt, *file_offset + cnt); 49 | if (n > 0) { 50 | cnt += n; 51 | } else if (n < 0) { 52 | if (errno != EINTR) { 53 | cnt = -1; 54 | break; 55 | } 56 | } else { 57 | break; 58 | } 59 | } 60 | 61 | if (cnt == -1) { 62 | return false; 63 | } else { 64 | return true; 65 | } 66 | } 67 | 68 | bool PosixFile::AppendIndex(const std::string& key, 69 | uint32_t file_index, 70 | uint64_t file_offset, 71 | uint32_t key_size, 72 | uint32_t value_size) { 73 | uint32_t index_size = key.size() + sizeof(uint32_t) * 4 + sizeof(uint64_t); 74 | char index_buf[index_size]; 75 | EncodeIndex(index_buf, key, file_index, 76 | file_offset, key_size, value_size); 77 | 78 | uint64_t offset = file_offset_.fetch_add(index_size); 79 | int64_t cnt = 0; 80 | while (cnt < index_size) { 81 | ssize_t n = pwrite(fd_, index_buf, index_size - cnt, offset + cnt); 82 | if (n > 0) { 83 | cnt += n; 84 | } else if (n < 0) { 85 | if (errno != EINTR) { 86 | cnt = -1; 87 | break; 88 | } 89 | } else { 90 | break; 91 | } 92 | } 93 | 94 | if (cnt == -1) { 95 | return false; 96 | } else { 97 | return true; 98 | } 99 | } 100 | 101 | bool PosixFile::Close() { 102 | ::close(fd_); 103 | fd_ = -1; 104 | return true; 105 | } 106 | 107 | } // namespace atendb 108 | -------------------------------------------------------------------------------- /file/posix_file.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_FILE_POSIX_FILE_H_ 4 | #define ATENDB_FILE_POSIX_FILE_H_ 5 | 6 | #include 7 | 8 | #include "file/file.h" 9 | #include "util/coding.h" 10 | #include "util/lock.h" 11 | 12 | namespace atendb { 13 | 14 | class PosixFile : public File { 15 | public: 16 | PosixFile(int32_t fd, const std::string& fname, size_t file_offset) : 17 | fd_(fd), 18 | fname_(fname), 19 | file_offset_(file_offset), 20 | buf_(nullptr), 21 | base_buf_(nullptr), 22 | map_size_(4096 * 10), 23 | map_pos_(0), 24 | map_avail_(0) { } 25 | 26 | virtual ~PosixFile(); 27 | 28 | bool Read(uint64_t offset, uint32_t n, char* buf); 29 | 30 | bool AppendData(const std::string& key, 31 | const std::string& value, 32 | uint64_t* file_offset); 33 | 34 | bool AppendIndex(const std::string& key, 35 | uint32_t file_index, 36 | uint64_t file_offset, 37 | uint32_t key_size_, 38 | uint32_t value_size_); 39 | 40 | bool Close(); 41 | 42 | inline uint64_t FileOffset() { return file_offset_; } 43 | 44 | private: 45 | uint32_t fd_; 46 | std::string fname_; 47 | std::atomic file_offset_; 48 | 49 | char* buf_; 50 | char* base_buf_; 51 | uint32_t map_size_; 52 | uint32_t map_pos_; 53 | uint32_t map_avail_; 54 | 55 | SpinLock file_lock_; 56 | }; 57 | 58 | } // namespace atendb 59 | 60 | #endif // ATENDB_FILE_POSIX_FILE_H_ 61 | -------------------------------------------------------------------------------- /include/comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_INCLUDE_COMPARATOR_H_ 4 | #define ATENDB_INCLUDE_COMPARATOR_H_ 5 | 6 | #include 7 | #include "include/index.h" 8 | 9 | namespace atendb { 10 | 11 | class Comparator { 12 | public: 13 | virtual ~Comparator() {} 14 | virtual int compare(const std::string& a, const std::string& b) const = 0; 15 | }; 16 | 17 | class BytewiseComparator : public Comparator { 18 | public: 19 | int compare(const std::string& a, const std::string& b) const { 20 | return a.compare(b); 21 | } 22 | }; 23 | 24 | class Compare { 25 | public: 26 | Compare(Comparator* comparator) 27 | : comparator_(comparator) {} 28 | 29 | int operator()(const Index& a, const Index& b) const { 30 | return comparator_->compare(a.Key(), b.Key()); 31 | } 32 | 33 | int operator()(const std::string& a, const std::string& b) const { 34 | return strcmp(a.c_str(), b.c_str()); 35 | } 36 | 37 | private: 38 | Comparator* comparator_; 39 | }; 40 | 41 | } // namespace atendb 42 | 43 | #endif // ATENDB_INCLUDE_COMPARATOR_H_ 44 | -------------------------------------------------------------------------------- /include/db.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_INCLUDE_DB_H_ 4 | #define ATENDB_INCLUDE_DB_H_ 5 | 6 | #include "include/options.h" 7 | #include "include/env.h" 8 | #include "include/comparator.h" 9 | 10 | namespace atendb { 11 | 12 | class DB { 13 | public: 14 | DB() { } 15 | virtual ~DB() {} 16 | 17 | static bool Open(const std::string& dbname, const Options& options, DB** dbptr); 18 | 19 | virtual bool Put(const std::string& key, const std::string& value) = 0; 20 | 21 | virtual bool Get(const std::string& key, std::string* value) = 0; 22 | 23 | virtual bool Delete(const std::string& key) = 0; 24 | 25 | private: 26 | // No copying allowed 27 | void operator=(const DB&); 28 | DB(const DB&); 29 | }; 30 | 31 | } // namespace atendb 32 | 33 | #endif // ATENDB_INCLUDE_DB_H_ 34 | -------------------------------------------------------------------------------- /include/env.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_INCLUDE_ENV_H_ 4 | #define ATENDB_INCLUDE_ENV_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "file/file.h" 13 | #include "file/posix_file.h" 14 | 15 | namespace atendb { 16 | 17 | class Env { 18 | public: 19 | Env() {} 20 | virtual ~Env() {} 21 | 22 | static bool NewFile(const std::string& fname, File** file) { 23 | int32_t fd = ::open(fname.c_str(), O_RDWR | O_CREAT | O_APPEND, 0645); 24 | if (fd < 0) { 25 | file = nullptr; 26 | return false; 27 | } else { 28 | uint64_t offset; 29 | GetFileSize(fname, &offset); 30 | *file = new PosixFile(fd, fname, offset); 31 | return true; 32 | } 33 | } 34 | 35 | static bool FileExists(const std::string& fname) { 36 | if (access(fname.c_str(), F_OK) == 0) { 37 | return true; 38 | } else{ 39 | return false; 40 | } 41 | } 42 | 43 | static bool CreateDir(const std::string& dirname) { 44 | if (mkdir(dirname.c_str(), 0755) != 0) { 45 | return false; 46 | } 47 | return true; 48 | } 49 | 50 | static bool GetFileSize(const std::string& fname, uint64_t* size) { 51 | struct stat sbuf; 52 | if (stat(fname.c_str(), &sbuf) != 0) { 53 | *size = 0; 54 | return false; 55 | } else { 56 | *size = sbuf.st_size; 57 | } 58 | return true; 59 | } 60 | 61 | static bool GetCurrentDir(std::string* dir) { 62 | char cwd[125]; 63 | if (getcwd(cwd, sizeof(cwd)) != nullptr) { 64 | *dir = std::string(cwd); 65 | return true; 66 | } 67 | return false; 68 | } 69 | 70 | }; 71 | 72 | } // namespace atendb 73 | 74 | #endif // ATENDB_INCLUDE_ENV_H_ 75 | -------------------------------------------------------------------------------- /include/index.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_INCLUDE_INDEX_H_ 4 | #define ATENDB_INCLUDE_INDEX_H_ 5 | 6 | #include 7 | 8 | namespace atendb { 9 | 10 | 11 | class Index { 12 | public: 13 | Index() {} 14 | 15 | Index(std::string key) : 16 | key_(key), 17 | file_index_(0), 18 | file_offset_(0), 19 | key_size_(0), 20 | value_size_(0) {} 21 | 22 | Index(std::string key, uint32_t file_index, uint64_t file_offset, uint32_t key_size, uint32_t value_size) : 23 | key_(key), 24 | file_index_(file_index), 25 | file_offset_(file_offset), 26 | key_size_(key_size), 27 | value_size_(value_size) {} 28 | 29 | ~Index() {} 30 | 31 | std::string Key() const { return key_; } 32 | 33 | uint32_t FileIndex() const { return file_index_; } 34 | 35 | uint64_t FileOffset() const { return file_offset_; } 36 | 37 | uint32_t KeySize() const { return key_size_; } 38 | 39 | uint32_t ValueSize() const { return value_size_; } 40 | 41 | uint32_t DataSize() const { return key_size_ + value_size_; } 42 | 43 | uint32_t IndexSize() const { 44 | return key_size_ + sizeof(uint32_t) * 3 + sizeof(uint64_t); 45 | } 46 | 47 | void SetDeleted() { 48 | this->key_size_ = 0; 49 | this->value_size_ = 0; 50 | } 51 | 52 | void operator=(const Index& index) { 53 | this->key_ = index.Key(); 54 | this->file_index_ = index.FileIndex(); 55 | this->file_offset_ = index.FileOffset(); 56 | this->key_size_ = index.KeySize(); 57 | this->value_size_ = index.ValueSize(); 58 | } 59 | 60 | private: 61 | friend class DB; 62 | friend class DBImpl; 63 | 64 | std::string key_; 65 | uint32_t file_index_; 66 | uint64_t file_offset_; 67 | 68 | uint32_t key_size_; 69 | uint32_t value_size_; 70 | }; 71 | 72 | } // namespace atendb 73 | 74 | #endif // ATENDB_INCLUDE_INDEX_H_ 75 | 76 | -------------------------------------------------------------------------------- /include/options.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_INCLUDE_OPTIONS_H_ 4 | #define ATENDB_INCLUDE_OPTIONS_H_ 5 | 6 | #include "include/comparator.h" 7 | #include "include/env.h" 8 | 9 | namespace atendb { 10 | 11 | class Options { 12 | public: 13 | Options() : 14 | comparator_(nullptr) { } 15 | 16 | ~Options() {} 17 | 18 | Comparator* comparator_; 19 | }; 20 | 21 | } // namespace atendb 22 | 23 | #endif // ATENDB_INCLUDE_OPTIONS_H_ 24 | -------------------------------------------------------------------------------- /include/skiplist.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_ 6 | #define STORAGE_LEVELDB_DB_SKIPLIST_H_ 7 | 8 | // Thread safety 9 | // ------------- 10 | // 11 | // Writes require external synchronization, most likely a mutex. 12 | // Reads require a guarantee that the SkipList will not be destroyed 13 | // while the read is in progress. Apart from that, reads progress 14 | // without any internal locking or synchronization. 15 | // 16 | // Invariants: 17 | // 18 | // (1) Allocated nodes are never deleted until the SkipList is 19 | // destroyed. This is trivially guaranteed by the code since we 20 | // never delete any skip list nodes. 21 | // 22 | // (2) The contents of a Node except for the next/prev pointers are 23 | // immutable after the Node has been linked into the SkipList. 24 | // Only Insert() modifies the list, and it is careful to initialize 25 | // a node and use release-stores to publish the nodes in one or 26 | // more lists. 27 | // 28 | // ... prev vs. next pointer ordering ... 29 | 30 | #include 31 | #include 32 | #include "util/arena.h" 33 | #include "util/random.h" 34 | 35 | namespace atendb { 36 | 37 | class Arena; 38 | 39 | template 40 | class SkipList { 41 | private: 42 | struct Node; 43 | 44 | public: 45 | // Create a new SkipList object that will use "cmp" for comparing keys, 46 | // and will allocate memory using "*arena". Objects allocated in the arena 47 | // must remain allocated for the lifetime of the skiplist object. 48 | explicit SkipList(Comparator cmp); 49 | 50 | // Insert key into the list. 51 | // REQUIRES: nothing that compares equal to key is currently in the list. 52 | void Insert(const Key& key); 53 | 54 | // Returns true iff an entry that compares equal to key is in the list. 55 | bool Contains(const Key& key) const; 56 | 57 | bool Get(const Key& key, Key* result) const; 58 | 59 | // Iteration over the contents of a skip list 60 | class Iterator { 61 | public: 62 | // Initialize an iterator over the specified list. 63 | // The returned iterator is not valid. 64 | explicit Iterator(const SkipList* list); 65 | 66 | // Returns true iff the iterator is positioned at a valid node. 67 | bool Valid() const; 68 | 69 | // Returns the key at the current position. 70 | // REQUIRES: Valid() 71 | const Key& key() const; 72 | 73 | // Advances to the next position. 74 | // REQUIRES: Valid() 75 | void Next(); 76 | 77 | // Advances to the previous position. 78 | // REQUIRES: Valid() 79 | void Prev(); 80 | 81 | // Advance to the first entry with a key >= target 82 | void Seek(const Key& target); 83 | 84 | // Position at the first entry in list. 85 | // Final state of iterator is Valid() iff list is not empty. 86 | void SeekToFirst(); 87 | 88 | // Position at the last entry in list. 89 | // Final state of iterator is Valid() iff list is not empty. 90 | void SeekToLast(); 91 | 92 | private: 93 | const SkipList* list_; 94 | Node* node_; 95 | // Intentionally copyable 96 | }; 97 | 98 | private: 99 | enum { kMaxHeight = 12 }; 100 | 101 | // Immutable after construction 102 | Comparator const compare_; 103 | Arena arena_; 104 | SpinLock lock_; 105 | 106 | Node* const head_; 107 | 108 | // Modified only by Insert(). Read racily by readers, but stale 109 | // values are ok. 110 | port::AtomicPointer max_height_; // Height of the entire list 111 | 112 | inline int GetMaxHeight() const { 113 | return static_cast( 114 | reinterpret_cast(max_height_.NoBarrier_Load())); 115 | } 116 | 117 | // Read/written only by Insert(). 118 | Random rnd_; 119 | 120 | Node* NewNode(const Key& key, int height); 121 | int RandomHeight(); 122 | bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } 123 | 124 | // Return true if key is greater than the data stored in "n" 125 | bool KeyIsAfterNode(const Key& key, Node* n) const; 126 | 127 | // Return the earliest node that comes at or after key. 128 | // Return nullptr if there is no such node. 129 | // 130 | // If prev is non-null, fills prev[level] with pointer to previous 131 | // node at "level" for every level in [0..max_height_-1]. 132 | Node* FindGreaterOrEqual(const Key& key, Node** prev) const; 133 | 134 | // Return the latest node with a key < key. 135 | // Return head_ if there is no such node. 136 | Node* FindLessThan(const Key& key) const; 137 | 138 | // Return the last node in the list. 139 | // Return head_ if list is empty. 140 | Node* FindLast() const; 141 | 142 | // No copying allowed 143 | SkipList(const SkipList&); 144 | void operator=(const SkipList&); 145 | }; 146 | 147 | // Implementation details follow 148 | template 149 | struct SkipList::Node { 150 | explicit Node(const Key& k) : key(k) { } 151 | 152 | Key key; 153 | 154 | // Accessors/mutators for links. Wrapped in methods so we can 155 | // add the appropriate barriers as necessary. 156 | Node* Next(int n) { 157 | assert(n >= 0); 158 | // Use an 'acquire load' so that we observe a fully initialized 159 | // version of the returned Node. 160 | return reinterpret_cast(next_[n].Acquire_Load()); 161 | } 162 | void SetNext(int n, Node* x) { 163 | assert(n >= 0); 164 | // Use a 'release store' so that anybody who reads through this 165 | // pointer observes a fully initialized version of the inserted node. 166 | next_[n].Release_Store(x); 167 | } 168 | 169 | // No-barrier variants that can be safely used in a few locations. 170 | Node* NoBarrier_Next(int n) { 171 | assert(n >= 0); 172 | return reinterpret_cast(next_[n].NoBarrier_Load()); 173 | } 174 | void NoBarrier_SetNext(int n, Node* x) { 175 | assert(n >= 0); 176 | next_[n].NoBarrier_Store(x); 177 | } 178 | 179 | private: 180 | // Array of length equal to the node height. next_[0] is lowest level link. 181 | port::AtomicPointer next_[1]; 182 | }; 183 | 184 | template 185 | typename SkipList::Node* 186 | SkipList::NewNode(const Key& key, int height) { 187 | char* mem = arena_.AllocateAligned( 188 | sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1)); 189 | return new (mem) Node(key); 190 | } 191 | 192 | template 193 | inline SkipList::Iterator::Iterator(const SkipList* list) { 194 | list_ = list; 195 | node_ = nullptr; 196 | } 197 | 198 | template 199 | inline bool SkipList::Iterator::Valid() const { 200 | return node_ != nullptr; 201 | } 202 | 203 | template 204 | inline const Key& SkipList::Iterator::key() const { 205 | assert(Valid()); 206 | return node_->key; 207 | } 208 | 209 | template 210 | inline void SkipList::Iterator::Next() { 211 | assert(Valid()); 212 | node_ = node_->Next(0); 213 | } 214 | 215 | template 216 | inline void SkipList::Iterator::Prev() { 217 | // Instead of using explicit "prev" links, we just search for the 218 | // last node that falls before key. 219 | assert(Valid()); 220 | node_ = list_->FindLessThan(node_->key); 221 | if (node_ == list_->head_) { 222 | node_ = nullptr; 223 | } 224 | } 225 | 226 | template 227 | inline void SkipList::Iterator::Seek(const Key& target) { 228 | node_ = list_->FindGreaterOrEqual(target, nullptr); 229 | } 230 | 231 | template 232 | inline void SkipList::Iterator::SeekToFirst() { 233 | node_ = list_->head_->Next(0); 234 | } 235 | 236 | template 237 | inline void SkipList::Iterator::SeekToLast() { 238 | node_ = list_->FindLast(); 239 | if (node_ == list_->head_) { 240 | node_ = nullptr; 241 | } 242 | } 243 | 244 | template 245 | int SkipList::RandomHeight() { 246 | // Increase height with probability 1 in kBranching 247 | static const unsigned int kBranching = 4; 248 | int height = 1; 249 | while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) { 250 | height++; 251 | } 252 | assert(height > 0); 253 | assert(height <= kMaxHeight); 254 | return height; 255 | } 256 | 257 | template 258 | bool SkipList::KeyIsAfterNode(const Key& key, Node* n) const { 259 | // null n is considered infinite 260 | return (n != nullptr) && (compare_(n->key, key) < 0); 261 | } 262 | 263 | template 264 | typename SkipList::Node* SkipList::FindGreaterOrEqual(const Key& key, Node** prev) 265 | const { 266 | Node* x = head_; 267 | int level = GetMaxHeight() - 1; 268 | while (true) { 269 | Node* next = x->Next(level); 270 | if (KeyIsAfterNode(key, next)) { 271 | // Keep searching in this list 272 | x = next; 273 | } else { 274 | if (prev != nullptr) prev[level] = x; 275 | if (level == 0) { 276 | return next; 277 | } else { 278 | // Switch to next list 279 | level--; 280 | } 281 | } 282 | } 283 | } 284 | 285 | template 286 | typename SkipList::Node* 287 | SkipList::FindLessThan(const Key& key) const { 288 | Node* x = head_; 289 | int level = GetMaxHeight() - 1; 290 | while (true) { 291 | assert(x == head_ || compare_(x->key, key) < 0); 292 | Node* next = x->Next(level); 293 | if (next == nullptr || compare_(next->key, key) >= 0) { 294 | if (level == 0) { 295 | return x; 296 | } else { 297 | // Switch to next list 298 | level--; 299 | } 300 | } else { 301 | x = next; 302 | } 303 | } 304 | } 305 | 306 | template 307 | typename SkipList::Node* SkipList::FindLast() 308 | const { 309 | Node* x = head_; 310 | int level = GetMaxHeight() - 1; 311 | while (true) { 312 | Node* next = x->Next(level); 313 | if (next == nullptr) { 314 | if (level == 0) { 315 | return x; 316 | } else { 317 | // Switch to next list 318 | level--; 319 | } 320 | } else { 321 | x = next; 322 | } 323 | } 324 | } 325 | 326 | template 327 | SkipList::SkipList(Comparator cmp) 328 | : compare_(cmp), 329 | head_(NewNode(Key() /* any key will do */, kMaxHeight)), 330 | max_height_(reinterpret_cast(1)), 331 | rnd_(0xdeadbeef) { 332 | for (int i = 0; i < kMaxHeight; i++) { 333 | head_->SetNext(i, nullptr); 334 | } 335 | } 336 | 337 | template 338 | void SkipList::Insert(const Key& key) { 339 | // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual() 340 | // here since Insert() is externally synchronized. 341 | Node* prev[kMaxHeight]; 342 | 343 | lock_.Lock(); 344 | Node* x = FindGreaterOrEqual(key, prev); 345 | 346 | int height = RandomHeight(); 347 | if (height > GetMaxHeight()) { 348 | for (int i = GetMaxHeight(); i < height; i++) { 349 | prev[i] = head_; 350 | } 351 | //fprintf(stderr, "Change height from %d to %d\n", max_height_, height); 352 | 353 | // It is ok to mutate max_height_ without any synchronization 354 | // with concurrent readers. A concurrent reader that observes 355 | // the new value of max_height_ will see either the old value of 356 | // new level pointers from head_ (nullptr), or a new value set in 357 | // the loop below. In the former case the reader will 358 | // immediately drop to the next level since nullptr sorts after all 359 | // keys. In the latter case the reader will use the new node. 360 | max_height_.NoBarrier_Store(reinterpret_cast(height)); 361 | } 362 | 363 | if (x != nullptr && Equal(key, x->key)) { 364 | x->key = key; 365 | lock_.Unlock(); 366 | return; 367 | } 368 | 369 | x = NewNode(key, height); 370 | for (int i = 0; i < height; i++) { 371 | // NoBarrier_SetNext() suffices since we will add a barrier when 372 | // we publish a pointer to "x" in prev[i]. 373 | x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i)); 374 | prev[i]->SetNext(i, x); 375 | } 376 | lock_.Unlock(); 377 | } 378 | 379 | template 380 | bool SkipList::Contains(const Key& key) const { 381 | Node* x = FindGreaterOrEqual(key, nullptr); 382 | if (x != nullptr && Equal(key, x->key)) { 383 | return true; 384 | } else { 385 | return false; 386 | } 387 | } 388 | 389 | template 390 | bool SkipList::Get(const Key& key, Key* result) const { 391 | Node* x = FindGreaterOrEqual(key, nullptr); 392 | if (x != nullptr && Equal(key, x->key)) { 393 | *result = x->key; 394 | return true; 395 | } else { 396 | return false; 397 | } 398 | } 399 | 400 | } // namespace atendb 401 | 402 | #endif // STORAGE_LEVELDB_DB_SKIPLIST_H_ 403 | -------------------------------------------------------------------------------- /port/atomic_pointer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 6 | 7 | // AtomicPointer provides storage for a lock-free pointer. 8 | // Platform-dependent implementation of AtomicPointer: 9 | // - If the platform provides a cheap barrier, we use it with raw pointers 10 | // - If is present (on newer versions of gcc, it is), we use 11 | // a -based AtomicPointer. However we prefer the memory 12 | // barrier based version, because at least on a gcc 4.4 32-bit build 13 | // on linux, we have encountered a buggy implementation. 14 | // Also, some implementations are much slower than a memory-barrier 15 | // based implementation (~16ns for based acquire-load vs. ~1ns for 16 | // a barrier based acquire-load). 17 | // This code is based on atomicops-internals-* in Google's perftools: 18 | // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase 19 | 20 | #ifndef PORT_ATOMIC_POINTER_H_ 21 | #define PORT_ATOMIC_POINTER_H_ 22 | 23 | #include 24 | 25 | #include 26 | 27 | #ifdef OS_WIN 28 | #include 29 | #endif 30 | 31 | #if defined(_M_X64) || defined(__x86_64__) 32 | #define ARCH_CPU_X86_FAMILY 1 33 | #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) 34 | #define ARCH_CPU_X86_FAMILY 1 35 | #elif defined(__ARMEL__) 36 | #define ARCH_CPU_ARM_FAMILY 1 37 | #elif defined(__aarch64__) 38 | #define ARCH_CPU_ARM64_FAMILY 1 39 | #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) 40 | #define ARCH_CPU_PPC_FAMILY 1 41 | #elif defined(__mips__) 42 | #define ARCH_CPU_MIPS_FAMILY 1 43 | #endif 44 | 45 | namespace atendb { 46 | namespace port { 47 | 48 | // Define MemoryBarrier() if available 49 | // Windows on x86 50 | #if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY) 51 | // windows.h already provides a MemoryBarrier(void) macro 52 | // http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx 53 | #define LEVELDB_HAVE_MEMORY_BARRIER 54 | 55 | // Mac OS 56 | #elif defined(__APPLE__) 57 | inline void MemoryBarrier() { 58 | std::atomic_thread_fence(std::memory_order_seq_cst); 59 | } 60 | #define LEVELDB_HAVE_MEMORY_BARRIER 61 | 62 | // Gcc on x86 63 | #elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__) 64 | inline void MemoryBarrier() { 65 | // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 66 | // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 67 | __asm__ __volatile__("" : : : "memory"); 68 | } 69 | #define LEVELDB_HAVE_MEMORY_BARRIER 70 | 71 | // Sun Studio 72 | #elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC) 73 | inline void MemoryBarrier() { 74 | // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 75 | // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 76 | asm volatile("" : : : "memory"); 77 | } 78 | #define LEVELDB_HAVE_MEMORY_BARRIER 79 | 80 | // ARM Linux 81 | #elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__) 82 | typedef void (*LinuxKernelMemoryBarrierFunc)(void); 83 | // The Linux ARM kernel provides a highly optimized device-specific memory 84 | // barrier function at a fixed memory address that is mapped in every 85 | // user-level process. 86 | // 87 | // This beats using CPU-specific instructions which are, on single-core 88 | // devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more 89 | // than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking 90 | // shows that the extra function call cost is completely negligible on 91 | // multi-core devices. 92 | // 93 | inline void MemoryBarrier() { 94 | (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)(); 95 | } 96 | #define LEVELDB_HAVE_MEMORY_BARRIER 97 | 98 | // ARM64 99 | #elif defined(ARCH_CPU_ARM64_FAMILY) 100 | inline void MemoryBarrier() { 101 | asm volatile("dmb sy" : : : "memory"); 102 | } 103 | #define LEVELDB_HAVE_MEMORY_BARRIER 104 | 105 | // PPC 106 | #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) 107 | inline void MemoryBarrier() { 108 | // TODO for some powerpc expert: is there a cheaper suitable variant? 109 | // Perhaps by having separate barriers for acquire and release ops. 110 | asm volatile("sync" : : : "memory"); 111 | } 112 | #define LEVELDB_HAVE_MEMORY_BARRIER 113 | 114 | // MIPS 115 | #elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__) 116 | inline void MemoryBarrier() { 117 | __asm__ __volatile__("sync" : : : "memory"); 118 | } 119 | #define LEVELDB_HAVE_MEMORY_BARRIER 120 | 121 | #endif 122 | 123 | // AtomicPointer built using platform-specific MemoryBarrier(). 124 | #if defined(LEVELDB_HAVE_MEMORY_BARRIER) 125 | class AtomicPointer { 126 | private: 127 | void* rep_; 128 | public: 129 | AtomicPointer() { } 130 | explicit AtomicPointer(void* p) : rep_(p) {} 131 | inline void* NoBarrier_Load() const { return rep_; } 132 | inline void NoBarrier_Store(void* v) { rep_ = v; } 133 | inline void* Acquire_Load() const { 134 | void* result = rep_; 135 | MemoryBarrier(); 136 | return result; 137 | } 138 | inline void Release_Store(void* v) { 139 | MemoryBarrier(); 140 | rep_ = v; 141 | } 142 | }; 143 | 144 | // AtomicPointer based on C++11 . 145 | #else 146 | class AtomicPointer { 147 | private: 148 | std::atomic rep_; 149 | public: 150 | AtomicPointer() { } 151 | explicit AtomicPointer(void* v) : rep_(v) { } 152 | inline void* Acquire_Load() const { 153 | return rep_.load(std::memory_order_acquire); 154 | } 155 | inline void Release_Store(void* v) { 156 | rep_.store(v, std::memory_order_release); 157 | } 158 | inline void* NoBarrier_Load() const { 159 | return rep_.load(std::memory_order_relaxed); 160 | } 161 | inline void NoBarrier_Store(void* v) { 162 | rep_.store(v, std::memory_order_relaxed); 163 | } 164 | }; 165 | 166 | #endif 167 | 168 | #undef LEVELDB_HAVE_MEMORY_BARRIER 169 | #undef ARCH_CPU_X86_FAMILY 170 | #undef ARCH_CPU_ARM_FAMILY 171 | #undef ARCH_CPU_ARM64_FAMILY 172 | #undef ARCH_CPU_PPC_FAMILY 173 | 174 | } // namespace port 175 | } // namespace atendb 176 | 177 | #endif // PORT_ATOMIC_POINTER_H_ 178 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(db dbimpl_test.cc) 2 | target_link_libraries(db atendb) 3 | -------------------------------------------------------------------------------- /test/dbimpl_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "include/options.h" 12 | #include "include/comparator.h" 13 | #include "include/db.h" 14 | 15 | using namespace atendb; 16 | 17 | std::string random_string( size_t length ) { 18 | auto randchar = []() -> char { 19 | const char charset[] = "0123456789" 20 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 21 | "abcdefghijklmnopqrstuvwxyz"; 22 | const size_t max_index = (sizeof(charset) - 1); 23 | return charset[ rand() % max_index ]; 24 | }; 25 | std::string str(length,0); 26 | std::generate_n( str.begin(), length, randchar ); 27 | return str; 28 | } 29 | 30 | int main() { 31 | Options opts; 32 | opts.comparator_ = new BytewiseComparator(); 33 | std::string db_name = "data"; 34 | 35 | DB* db; 36 | DB::Open(db_name, opts, &db); 37 | std::map kvs; 38 | for (int32_t i = 0; i < 1000; i++) { 39 | std::string key = random_string(8); 40 | std::string value = random_string(4096); 41 | kvs[key] = value; 42 | auto s = db->Put(key, value); 43 | if (!s) { 44 | std::cout << "Write failed." << key << std::endl; 45 | } 46 | 47 | std::string read_value; 48 | s = db->Get(key, &read_value); 49 | if (!s || value != read_value) { 50 | std::cout << "Read value failed: " << key << std::endl; 51 | } 52 | } 53 | delete db; 54 | 55 | DB* reopen_db; 56 | DB::Open(db_name, opts, &reopen_db); 57 | for (auto it = kvs.begin(); it != kvs.end(); ++it) { 58 | std::string read_value; 59 | auto s = reopen_db->Get(it->first, &read_value); 60 | if (!s || it->second != read_value) { 61 | std::cout << "Reread value failed: " << it->first << std::endl; 62 | } 63 | } 64 | 65 | // Delete 66 | for (auto it = kvs.begin(); it != kvs.end(); ++it) { 67 | auto s = reopen_db->Delete(it->first); 68 | if (!s) { 69 | std::cout << "Delete value failed: " << it->first << std::endl; 70 | } 71 | } 72 | 73 | for (auto it = kvs.begin(); it != kvs.end(); ++it) { 74 | std::string read_value; 75 | auto s = reopen_db->Get(it->first, &read_value); 76 | if (s || read_value != "") { 77 | std::cout << "Delete value failed: " << it->first << std::endl; 78 | } 79 | } 80 | delete reopen_db; 81 | 82 | return 0; 83 | } 84 | -------------------------------------------------------------------------------- /util/arena.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 6 | 7 | #include 8 | #include 9 | 10 | #include "util/arena.h" 11 | 12 | namespace atendb { 13 | 14 | static const int kBlockSize = 4096; 15 | 16 | Arena::Arena() : memory_usage_(0) { 17 | alloc_ptr_ = nullptr; // First allocation will allocate a block 18 | alloc_bytes_remaining_ = 0; 19 | } 20 | 21 | Arena::~Arena() { 22 | for (size_t i = 0; i < blocks_.size(); i++) { 23 | delete[] blocks_[i]; 24 | } 25 | } 26 | 27 | char* Arena::AllocateFallback(size_t bytes) { 28 | if (bytes > kBlockSize / 4) { 29 | // Object is more than a quarter of our block size. Allocate it separately 30 | // to avoid wasting too much space in leftover bytes. 31 | char* result = AllocateNewBlock(bytes); 32 | return result; 33 | } 34 | 35 | // We waste the remaining space in the current block. 36 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 37 | alloc_bytes_remaining_ = kBlockSize; 38 | 39 | char* result = alloc_ptr_; 40 | alloc_ptr_ += bytes; 41 | alloc_bytes_remaining_ -= bytes; 42 | return result; 43 | } 44 | 45 | char* Arena::AllocateAligned(size_t bytes) { 46 | const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; 47 | assert((align & (align-1)) == 0); // Pointer size should be a power of 2 48 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); 49 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 50 | size_t needed = bytes + slop; 51 | char* result; 52 | if (needed <= alloc_bytes_remaining_) { 53 | result = alloc_ptr_ + slop; 54 | alloc_ptr_ += needed; 55 | alloc_bytes_remaining_ -= needed; 56 | } else { 57 | // AllocateFallback always returned aligned memory 58 | result = AllocateFallback(bytes); 59 | } 60 | assert((reinterpret_cast(result) & (align-1)) == 0); 61 | return result; 62 | } 63 | 64 | char* Arena::AllocateNewBlock(size_t block_bytes) { 65 | char* result = new char[block_bytes]; 66 | blocks_.push_back(result); 67 | memory_usage_.NoBarrier_Store( 68 | reinterpret_cast(MemoryUsage() + block_bytes + sizeof(char*))); 69 | return result; 70 | } 71 | 72 | } // namespace atendb 73 | -------------------------------------------------------------------------------- /util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 6 | 7 | #ifndef ATENDB_UTIL_ARENA_H_ 8 | #define ATENDB_UTIL_ARENA_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "port/atomic_pointer.h" 15 | 16 | namespace atendb { 17 | 18 | class Arena { 19 | public: 20 | Arena(); 21 | ~Arena(); 22 | 23 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 24 | char* Allocate(size_t bytes); 25 | 26 | // Allocate memory with the normal alignment guarantees provided by malloc 27 | char* AllocateAligned(size_t bytes); 28 | 29 | // Returns an estimate of the total memory usage of data allocated 30 | // by the arena. 31 | size_t MemoryUsage() const { 32 | return reinterpret_cast(memory_usage_.NoBarrier_Load()); 33 | } 34 | 35 | private: 36 | char* AllocateFallback(size_t bytes); 37 | char* AllocateNewBlock(size_t block_bytes); 38 | 39 | // Allocation state 40 | char* alloc_ptr_; 41 | size_t alloc_bytes_remaining_; 42 | 43 | // Array of new[] allocated memory blocks 44 | std::vector blocks_; 45 | 46 | // Total memory usage of the arena. 47 | port::AtomicPointer memory_usage_; 48 | 49 | // No copying allowed 50 | Arena(const Arena&); 51 | void operator=(const Arena&); 52 | }; 53 | 54 | inline char* Arena::Allocate(size_t bytes) { 55 | // The semantics of what to return are a bit messy if we allow 56 | // 0-byte allocations, so we disallow them here (we don't need 57 | // them for our internal use). 58 | assert(bytes > 0); 59 | if (bytes <= alloc_bytes_remaining_) { 60 | char* result = alloc_ptr_; 61 | alloc_ptr_ += bytes; 62 | alloc_bytes_remaining_ -= bytes; 63 | return result; 64 | } 65 | return AllocateFallback(bytes); 66 | } 67 | 68 | } // namespace atendb 69 | 70 | #endif // ATENDB_UTIL_ARENA_H_ 71 | -------------------------------------------------------------------------------- /util/coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #include 4 | #include 5 | 6 | #include "coding.h" 7 | 8 | namespace atendb { 9 | 10 | void EncodeData(char* buf, const std::string& key, const std::string& value) { 11 | memcpy(buf, key.data(), key.size()); 12 | memcpy(buf+key.size(), value.data(), value.size()); 13 | return; 14 | } 15 | 16 | void DecodeData(char* buf, 17 | uint32_t key_size, 18 | uint32_t value_size, 19 | std::string* value) { 20 | *value = std::string(buf+key_size, value_size); 21 | return; 22 | } 23 | 24 | void EncodeIndex(char* buf, 25 | const std::string& key, 26 | uint32_t file_index, 27 | uint64_t file_offset, 28 | uint32_t key_size, 29 | uint32_t value_size) { 30 | assert(buf); 31 | size_t pos = 0; 32 | 33 | uint32_t index_size = key_size + sizeof(uint32_t) * 3 + sizeof(uint64_t); 34 | // The size of index 35 | memcpy(buf, &index_size, sizeof(uint32_t)); 36 | pos += sizeof(uint32_t); 37 | // File index 38 | memcpy(buf+pos, &file_index, sizeof(uint32_t)); 39 | pos += sizeof(uint32_t); 40 | // File offset 41 | memcpy(buf+pos, &file_offset, sizeof(uint64_t)); 42 | pos += sizeof(uint64_t); 43 | // The size of key 44 | memcpy(buf+pos, &key_size, sizeof(uint32_t)); 45 | pos += sizeof(uint32_t); 46 | // The key 47 | memcpy(buf+pos, key.data(), key_size); 48 | pos += key_size; 49 | // The size of value 50 | memcpy(buf+pos, &value_size, sizeof(uint32_t)); 51 | 52 | return; 53 | } 54 | 55 | void DecodeIndex(char* buf, 56 | std::string* key, 57 | uint32_t* file_index, 58 | uint64_t* file_offset, 59 | uint32_t* key_size, 60 | uint32_t* value_size) { 61 | assert(buf); 62 | 63 | size_t pos = 0; 64 | 65 | *file_index = DecodeFixed32(buf); 66 | pos += sizeof(uint32_t); 67 | 68 | *file_offset = DecodeFixed64(buf+pos); 69 | pos += sizeof(uint64_t); 70 | 71 | *key_size = DecodeFixed32(buf+pos); 72 | pos += sizeof(uint32_t); 73 | 74 | *key = std::string(buf+pos, *key_size); 75 | pos += *key_size; 76 | 77 | *value_size = DecodeFixed32(buf+pos); 78 | 79 | return; 80 | } 81 | 82 | uint32_t DecodeFixed32(const char* ptr) { 83 | uint32_t result; 84 | memcpy(&result, ptr, sizeof(result)); 85 | return result; 86 | } 87 | 88 | uint64_t DecodeFixed64(const char* ptr) { 89 | uint64_t result; 90 | memcpy(&result, ptr, sizeof(result)); 91 | return result; 92 | } 93 | 94 | } // namespace atendb 95 | -------------------------------------------------------------------------------- /util/coding.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_UTIL_CODING_H_ 4 | #define ATENDB_UTIL_CODING_H_ 5 | 6 | #include 7 | 8 | namespace atendb { 9 | 10 | #define likely(x) __builtin_expect(!!(x), 1) 11 | #define unlikely(x) __builtin_expect(!!(x), 0) 12 | 13 | void EncodeData(char* buf, 14 | const std::string& key, 15 | const std::string& value); 16 | 17 | void DecodeData(char* buf, 18 | uint32_t key_size, 19 | uint32_t value_size, 20 | std::string* value); 21 | 22 | void EncodeIndex(char* buf, 23 | const std::string& key, 24 | uint32_t file_index, 25 | uint64_t file_offset, 26 | uint32_t key_size, 27 | uint32_t value_size); 28 | 29 | void DecodeIndex(char* buf, 30 | std::string* key, 31 | uint32_t* file_index, 32 | uint64_t* file_offset, 33 | uint32_t* key_size, 34 | uint32_t* value_size); 35 | 36 | uint32_t DecodeFixed32(const char* buf); 37 | 38 | uint64_t DecodeFixed64(const char* buf); 39 | 40 | } // namespace atendb 41 | 42 | #endif // ATENDB_UTIL_CODING_H_ 43 | -------------------------------------------------------------------------------- /util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_UTIL_HASH_H_ 4 | #define ATENDB_UTIL_HASH_H_ 5 | 6 | namespace atendb { 7 | 8 | unsigned int Hash(const char* str) { 9 | unsigned int seed = 131; 10 | unsigned int hash = 0; 11 | 12 | while (*str) { 13 | hash = hash * seed + (*str++); 14 | } 15 | return (hash & 0x7FFFFFFF); 16 | } 17 | 18 | } // namespace atendb 19 | 20 | #endif // ATENDB_UTIL_HASH_H_ 21 | -------------------------------------------------------------------------------- /util/lock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | #ifndef ATENDB_UTIL_LOCK_H_ 4 | #define ATENDB_UTIL_LOCK_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace atendb { 11 | 12 | class SpinLock { 13 | public: 14 | void Lock() { 15 | while(lck.test_and_set(std::memory_order_acquire)){} 16 | } 17 | 18 | void Unlock() { 19 | lck.clear(std::memory_order_release); 20 | } 21 | 22 | private: 23 | std::atomic_flag lck = ATOMIC_FLAG_INIT; 24 | }; 25 | 26 | } // namespace atendb 27 | 28 | #endif // ATENDB_UTIL_LOCK_H_ 29 | -------------------------------------------------------------------------------- /util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The atendb Authors. All rights reserved. 2 | 3 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 6 | 7 | #ifndef ATENDB_UTIL_RANDOM_H_ 8 | #define ATENDB_UTIL_RANDOM_H_ 9 | 10 | #include 11 | 12 | namespace atendb { 13 | 14 | // A very simple random number generator. Not especially good at 15 | // generating truly random bits, but good enough for our needs in this 16 | // package. 17 | class Random { 18 | private: 19 | uint32_t seed_; 20 | public: 21 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { 22 | // Avoid bad seeds. 23 | if (seed_ == 0 || seed_ == 2147483647L) { 24 | seed_ = 1; 25 | } 26 | } 27 | uint32_t Next() { 28 | static const uint32_t M = 2147483647L; // 2^31-1 29 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 30 | // We are computing 31 | // seed_ = (seed_ * A) % M, where M = 2^31-1 32 | // 33 | // seed_ must not be zero or M, or else all subsequent computed values 34 | // will be zero or M respectively. For all other values, seed_ will end 35 | // up cycling through every number in [1,M-1] 36 | uint64_t product = seed_ * A; 37 | 38 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 39 | seed_ = static_cast((product >> 31) + (product & M)); 40 | // The first reduction may overflow by 1 bit, so we may need to 41 | // repeat. mod == M is not possible; using > allows the faster 42 | // sign-bit-based test. 43 | if (seed_ > M) { 44 | seed_ -= M; 45 | } 46 | return seed_; 47 | } 48 | // Returns a uniformly distributed value in the range [0..n-1] 49 | // REQUIRES: n > 0 50 | uint32_t Uniform(int n) { return Next() % n; } 51 | 52 | // Randomly returns true ~"1/n" of the time, and false otherwise. 53 | // REQUIRES: n > 0 54 | bool OneIn(int n) { return (Next() % n) == 0; } 55 | 56 | // Skewed: pick "base" uniformly from range [0,max_log] and then 57 | // return "base" random bits. The effect is to pick a number in the 58 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 59 | uint32_t Skewed(int max_log) { 60 | return Uniform(1 << Uniform(max_log + 1)); 61 | } 62 | }; 63 | 64 | } // namespace atendb 65 | 66 | #endif // ATENDB_UTIL_RANDOM_H_ 67 | --------------------------------------------------------------------------------