├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── benchmark.md ├── contributing.md └── src ├── AUTHORS ├── LICENSE ├── Makefile.am ├── README ├── benchmark.cc ├── collate.sh ├── config.h.in.forcmake ├── configure.ac ├── db ├── autocompact_test.cc ├── builder.cc ├── builder.h ├── c.cc ├── c_test.c ├── corruption_test.cc ├── db_bench.cc ├── db_impl.cc ├── db_impl.h ├── db_iter.cc ├── db_iter.h ├── db_test.cc ├── dbformat.cc ├── dbformat.h ├── dbformat_test.cc ├── filename.cc ├── filename.h ├── filename_test.cc ├── leveldb_main.cc ├── log_format.h ├── log_reader.cc ├── log_reader.h ├── log_test.cc ├── log_writer.cc ├── log_writer.h ├── memtable.cc ├── memtable.h ├── murmurhash3.cc ├── murmurhash3.h ├── repair.cc ├── replay_iterator.cc ├── replay_iterator.h ├── skiplist.h ├── skiplist_test.cc ├── snapshot.h ├── table_cache.cc ├── table_cache.h ├── version_edit.cc ├── version_edit.h ├── version_edit_test.cc ├── version_set.cc ├── version_set.h ├── version_set_test.cc ├── write_batch.cc ├── write_batch_internal.h └── write_batch_test.cc ├── demo ├── README.md └── installation_test.cc ├── design-sentinels.txt ├── doc ├── bench │ ├── db_bench_sqlite3.cc │ └── db_bench_tree_db.cc ├── benchmark.html ├── doc.css ├── impl.html ├── index.html ├── log_format.txt └── table_format.txt ├── graphs ├── hyperdex-multi.png ├── micro-all.png ├── mongo.png └── multi-ycsb.png ├── helpers └── memenv │ ├── memenv.cc │ ├── memenv.h │ └── memenv_test.cc ├── include └── pebblesdb │ ├── c.h │ ├── cache.h │ ├── comparator.h │ ├── db.h │ ├── env.h │ ├── filter_policy.h │ ├── iterator.h │ ├── options.h │ ├── replay_iterator.h │ ├── slice.h │ ├── status.h │ ├── table.h │ ├── table_builder.h │ └── write_batch.h ├── issues ├── issue178_test.cc └── issue200_test.cc ├── leveldb-dump-all.cc ├── leveldb-verify.cc ├── libpebblesdb.pc.in ├── m4 ├── anal_warnings.m4 ├── ax_check_compile_flag.m4 ├── ax_check_link_flag.m4 └── ax_check_preproc_flag.m4 ├── port ├── README ├── atomic_pointer.h ├── port.h ├── port_example.h ├── port_posix.cc ├── port_posix.h ├── thread_annotations.h └── win │ └── stdint.h ├── reliability.sh ├── scripts ├── script_background_compaction_time_measure.sh ├── script_memtable_time_measure.sh ├── script_readrandom_time_measure.sh └── script_write_time_measure.sh ├── table ├── block.cc ├── block.h ├── block_builder.cc ├── block_builder.h ├── filter_block.cc ├── filter_block.h ├── filter_block_test.cc ├── format.cc ├── format.h ├── iterator.cc ├── iterator_wrapper.h ├── merger.cc ├── merger.h ├── table.cc ├── table_builder.cc ├── table_test.cc ├── two_level_iterator.cc └── two_level_iterator.h └── util ├── arena.cc ├── arena.h ├── arena_test.cc ├── atomic.cc ├── atomic.h ├── bloom.cc ├── bloom_test.cc ├── cache.cc ├── cache_test.cc ├── coding.cc ├── coding.h ├── coding_test.cc ├── comparator.cc ├── crc32c.cc ├── crc32c.h ├── crc32c_test.cc ├── env.cc ├── env_posix.cc ├── env_test.cc ├── filter_policy.cc ├── hash.cc ├── hash.h ├── histogram.cc ├── histogram.h ├── logging.cc ├── logging.h ├── mutexlock.h ├── options.cc ├── posix_logger.h ├── random.h ├── status.cc ├── string_builder.h ├── testharness.cc ├── testharness.h ├── testutil.cc ├── testutil.h └── timer.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | # wildcards 35 | .deps 36 | .dirstamp 37 | *.la 38 | .libs 39 | *.lo 40 | *.o 41 | *_test 42 | # specific files 43 | /aclocal.m4 44 | /autom4te.cache/ 45 | /benchmark 46 | /compile 47 | /config.guess 48 | /config.h 49 | /config.h.in 50 | /config.h.in~ 51 | /config.log 52 | /config.status 53 | /config.sub 54 | /configure 55 | /db_bench 56 | /depcomp 57 | /install-sh 58 | /leveldb-dump-all 59 | /leveldbutil 60 | /leveldb-verify 61 | /libpebblesdb.pc 62 | /libtool 63 | /ltmain.sh 64 | /m4/ 65 | /Makefile 66 | /Makefile.in 67 | /Makefile.old 68 | /missing 69 | /stamp-h1 70 | config.h* 71 | 72 | .idea/ 73 | .vscode/ 74 | cmake-build-debug/ 75 | testPebblesDB 76 | gdb.txt 77 | .autotools 78 | .cproject 79 | .project 80 | .settings/ 81 | nbproject/ 82 | CMakeFiles 83 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Build matrix / environment variable are explained on: 2 | # http://about.travis-ci.org/docs/user/build-configuration/ 3 | # This file can be validated on: http://lint.travis-ci.org/ 4 | 5 | sudo: false 6 | dist: trusty 7 | language: cpp 8 | 9 | compiler: 10 | - gcc 11 | 12 | os: 13 | - linux 14 | - osx 15 | 16 | git: 17 | submodules: false 18 | 19 | addons: 20 | apt: 21 | # List of whitelisted in travis packages for ubuntu-trusty can be found here: 22 | # https://github.com/travis-ci/apt-package-whitelist/blob/master/ubuntu-trusty 23 | # List of whitelisted in travis apt-sources: 24 | # https://github.com/travis-ci/apt-source-whitelist/blob/master/ubuntu.json 25 | sources: 26 | - ubuntu-toolchain-r-test 27 | packages: 28 | - g++-5 29 | - libsnappy-dev 30 | - libtool 31 | 32 | before_install: 33 | echo $TRAVIS_OS_NAME 34 | - if [ "$TRAVIS_OS_NAME" != "osx" ]; then 35 | sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 762E3157; 36 | fi 37 | 38 | install: 39 | 40 | - if [ "$TRAVIS_OS_NAME" == "osx" ]; then 41 | brew update; 42 | if [ -L /usr/local/include/c++ ]; then rm /usr/local/include/c++; fi; 43 | brew install gcc@5; 44 | brew install snappy; 45 | fi 46 | 47 | # /usr/bin/gcc is stuck to old versions on both Linux and OSX. 48 | - if [ "$CXX" = "g++" ]; then export CXX="g++-5"; fi 49 | - echo ${CXX} 50 | - ${CXX} --version 51 | 52 | before_script: 53 | - cmake . 54 | 55 | script: 56 | # `make all` still break 57 | - make db_test 58 | - ./db_test 59 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at vijay@cs.utexas.edu. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 4 | Copyright (c) 2013-2014 The HyperLevelDB Authors. All rights reserved. (HyperLevelDB changes) 5 | Copyright (c) 2017, The University of Texas at Austin (PebblesDB changes) 6 | All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | * Neither the name of the copyright holder nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | ### Contributing to PebblesDB 2 | ----- 3 | Are you an undergraduate student looking to contribute to an interesting systems research project? 4 | 5 | We could always use extra help in maintaining and extending PebblesDB! 6 | 7 | List of features we would like added: 8 | 9 | 1. Extending PebblesDB to use SURF filters (https://github.com/efficient/SuRF) 10 | 2. Allowing PebblesDB to run with a limited amount of memory (currently it uses all memory available) 11 | 3. Implementing the ideas behind Monkey in PebblesDB (https://stratos.seas.harvard.edu/files/stratos/files/monkeykeyvaluestore.pdf) 12 | 13 | If you are interested in implementing any of the above, please email vijay@cs.utexas.edu 14 | -------------------------------------------------------------------------------- /src/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | Johan Bilien 13 | 14 | # HyperLevelDB authors: 15 | Robert Escriva 16 | 17 | # PebblesDB authors: 18 | Pandian Raju 19 | Vijay Chidambaram 20 | Jayashree Mohan 21 | Zeyuan Hu 22 | 23 | -------------------------------------------------------------------------------- /src/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | Copyright (c) 2013-2014 The HyperLevelDB Authors. All rights reserved. (HyperLevelDB changes) 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /src/README: -------------------------------------------------------------------------------- 1 | leveldb: A key-value store 2 | Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) 3 | 4 | The code under this directory implements a system for maintaining a 5 | persistent key/value store. 6 | 7 | See doc/index.html for more explanation. 8 | See doc/impl.html for a brief overview of the implementation. 9 | 10 | The public interface is in include/*.h. Callers should not include or 11 | rely on the details of any other header files in this package. Those 12 | internal APIs may be changed without warning. 13 | 14 | Guide to header files: 15 | 16 | include/db.h 17 | Main interface to the DB: Start here 18 | 19 | include/options.h 20 | Control over the behavior of an entire database, and also 21 | control over the behavior of individual reads and writes. 22 | 23 | include/comparator.h 24 | Abstraction for user-specified comparison function. If you want 25 | just bytewise comparison of keys, you can use the default comparator, 26 | but clients can write their own comparator implementations if they 27 | want custom ordering (e.g. to handle different character 28 | encodings, etc.) 29 | 30 | include/iterator.h 31 | Interface for iterating over data. You can get an iterator 32 | from a DB object. 33 | 34 | include/write_batch.h 35 | Interface for atomically applying multiple updates to a database. 36 | 37 | include/slice.h 38 | A simple module for maintaining a pointer and a length into some 39 | other byte array. 40 | 41 | include/status.h 42 | Status is returned from many of the public interfaces and is used 43 | to report success and various kinds of errors. 44 | 45 | include/env.h 46 | Abstraction of the OS environment. A posix implementation of 47 | this interface is in util/env_posix.cc 48 | 49 | include/table.h 50 | include/table_builder.h 51 | Lower-level modules that most clients probably won't use directly 52 | 53 | Install 54 | ======= 55 | 56 | Get up and running quickly: 57 | 58 | $ autoreconf -i 59 | $ ./configure 60 | $ make 61 | # make install 62 | # ldconfig 63 | -------------------------------------------------------------------------------- /src/config.h.in.forcmake: -------------------------------------------------------------------------------- 1 | /* Define to 1 if you have the `alarm' function. */ 2 | #cmakedefine HAVE_ALARM 3 | 4 | /* Define to 1 if you have the `clock_gettime' function. */ 5 | #cmakedefine HAVE_CLOCK_GETTIME 6 | 7 | /* Define to 1 if you have the declaration of `fdatasync', and to 0 if you 8 | don't. */ 9 | #cmakedefine01 HAVE_DECL_FDATASYNC 10 | 11 | /* Define to 1 if you have the header file. */ 12 | #cmakedefine HAVE_DLFCN_H 13 | 14 | /* Define to 1 if you have the header file. */ 15 | #cmakedefine HAVE_ENDIAN_H 16 | 17 | /* Define to 1 if you have the `fflush' function. */ 18 | #cmakedefine01 HAVE_FFLUSH 19 | 20 | /* Define to 1 if you have the `fflush_unlocked' function. */ 21 | #cmakedefine HAVE_FFLUSH_UNLOCKED 22 | 23 | /* Define to 1 if you have the `fread' function. */ 24 | #cmakedefine01 HAVE_FREAD 25 | 26 | /* Define to 1 if you have the `fread_unlocked' function. */ 27 | #cmakedefine HAVE_FREAD_UNLOCKED 28 | 29 | /* Define to 1 if you have the `fsync' function. */ 30 | #cmakedefine01 HAVE_FSYNC 31 | 32 | /* Define to 1 if you have the `ftruncate' function. */ 33 | #cmakedefine HAVE_FTRUNCATE 34 | 35 | /* Define to 1 if you have the `fwrite' function. */ 36 | #cmakedefine01 HAVE_FWRITE 37 | 38 | /* Define to 1 if you have the `fwrite_unlocked' function. */ 39 | #cmakedefine HAVE_FWRITE_UNLOCKED 40 | 41 | /* Define to 1 if you have the `getpagesize' function. */ 42 | #cmakedefine HAVE_GETPAGESIZE 43 | 44 | /* Define to 1 if you have the header file. */ 45 | #cmakedefine HAVE_INTTYPES_H 46 | 47 | /* Define to 1 if you have the header file. */ 48 | #cmakedefine HAVE_MACHINE_ENDIAN_H 49 | 50 | /* Define to 1 if you have the `mach_absolute_time' function. */ 51 | #cmakedefine HAVE_MACH_ABSOLUTE_TIME 52 | 53 | /* Define to 1 if you have the `memmove' function. */ 54 | #cmakedefine HAVE_MEMMOVE 55 | 56 | /* Define to 1 if you have the header file. */ 57 | #cmakedefine HAVE_MEMORY_H 58 | 59 | /* Define to 1 if you have the `mkdir' function. */ 60 | #cmakedefine HAVE_MKDIR 61 | 62 | /* Define to 1 if you have a working `mmap' system call. */ 63 | #cmakedefine HAVE_MMAP 64 | 65 | /* Define to 1 if you have the `munmap' function. */ 66 | #cmakedefine HAVE_MUNMAP 67 | 68 | /* Define to 1 if you have the `rmdir' function. */ 69 | #cmakedefine HAVE_RMDIR 70 | 71 | /* Define to 1 if you have the `socket' function. */ 72 | #cmakedefine HAVE_SOCKET 73 | 74 | /* Define to 1 if you have the header file. */ 75 | #cmakedefine HAVE_STDINT_H 76 | 77 | /* Define to 1 if you have the header file. */ 78 | #cmakedefine HAVE_STDLIB_H 79 | 80 | /* Define to 1 if you have the header file. */ 81 | #cmakedefine HAVE_STRINGS_H 82 | 83 | /* Define to 1 if you have the header file. */ 84 | #cmakedefine HAVE_STRING_H 85 | 86 | /* Define to 1 if you have the header file. */ 87 | #cmakedefine HAVE_SYS_ENDIAN_H 88 | 89 | /* Define to 1 if you have the header file. */ 90 | #cmakedefine HAVE_SYS_ISA_DEFS_H 91 | 92 | /* Define to 1 if you have the header file. */ 93 | #cmakedefine HAVE_SYS_PARAM_H 94 | 95 | /* Define to 1 if you have the header file. */ 96 | #cmakedefine HAVE_SYS_STAT_H 97 | 98 | /* Define to 1 if you have the header file. */ 99 | #cmakedefine HAVE_SYS_TYPES_H 100 | 101 | /* Define to 1 if you have the header file. */ 102 | #cmakedefine HAVE_UNISTD_H 103 | 104 | /* Define to the sub-directory where libtool stores uninstalled libraries. */ 105 | #cmakedefine LT_OBJDIR 106 | 107 | /* 'Disable debug' */ 108 | #cmakedefine NDEBUG 109 | 110 | /* Name of package */ 111 | #cmakedefine PACKAGE 112 | 113 | /* Define to the address where bug reports for this package should be sent. */ 114 | #cmakedefine PACKAGE_BUGREPORT 115 | 116 | /* Define to the full name of this package. */ 117 | #cmakedefine PACKAGE_NAME 118 | 119 | /* Define to the full name and version of this package. */ 120 | #cmakedefine PACKAGE_STRING 121 | 122 | /* Define to the one symbol short name of this package. */ 123 | #cmakedefine PACKAGE_TARNAME 124 | 125 | /* Define to the home page for this package. */ 126 | #cmakedefine PACKAGE_URL 127 | 128 | /* Define to the version of this package. */ 129 | #cmakedefine PACKAGE_VERSION 130 | 131 | /* Define to 1 if you have the ANSI C header files. */ 132 | #cmakedefine STDC_HEADERS 133 | 134 | /* 'Start timer log' */ 135 | #cmakedefine TIMER_LOG 136 | 137 | /* Version number of package */ 138 | #cmakedefine VERSION 139 | 140 | /* 'Disable Mac-incompatible trace function' */ 141 | #cmakedefine __APPLE__ 142 | -------------------------------------------------------------------------------- /src/db/autocompact_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #define __STDC_LIMIT_MACROS 6 | 7 | #include "pebblesdb/db.h" 8 | #include "db/db_impl.h" 9 | #include "pebblesdb/cache.h" 10 | #include "util/testharness.h" 11 | #include "util/testutil.h" 12 | 13 | namespace leveldb { 14 | 15 | class AutoCompactTest { 16 | public: 17 | std::string dbname_; 18 | Cache* tiny_cache_; 19 | Options options_; 20 | DB* db_; 21 | 22 | AutoCompactTest() { 23 | dbname_ = test::TmpDir() + "/autocompact_test"; 24 | tiny_cache_ = NewLRUCache(100); 25 | options_.block_cache = tiny_cache_; 26 | DestroyDB(dbname_, options_); 27 | options_.create_if_missing = true; 28 | options_.compression = kNoCompression; 29 | ASSERT_OK(DB::Open(options_, dbname_, &db_)); 30 | } 31 | 32 | ~AutoCompactTest() { 33 | delete db_; 34 | DestroyDB(dbname_, Options()); 35 | delete tiny_cache_; 36 | } 37 | 38 | std::string Key(int i) { 39 | char buf[100]; 40 | snprintf(buf, sizeof(buf), "key%06d", i); 41 | return std::string(buf); 42 | } 43 | 44 | uint64_t Size(const Slice& start, const Slice& limit) { 45 | Range r(start, limit); 46 | uint64_t size; 47 | db_->GetApproximateSizes(&r, 1, &size); 48 | return size; 49 | } 50 | 51 | void DoReads(int n); 52 | }; 53 | 54 | static const int kValueSize = 200 * 1024; 55 | static const int kTotalSize = 100 * 1024 * 1024; 56 | static const int kCount = kTotalSize / kValueSize; 57 | 58 | // Read through the first n keys repeatedly and check that they get 59 | // compacted (verified by checking the size of the key space). 60 | void AutoCompactTest::DoReads(int n) { 61 | std::string value(kValueSize, 'x'); 62 | DBImpl* dbi = reinterpret_cast(db_); 63 | 64 | // Fill database 65 | for (int i = 0; i < kCount; i++) { 66 | ASSERT_OK(db_->Put(WriteOptions(), Key(i), value)); 67 | } 68 | ASSERT_OK(dbi->TEST_CompactMemTable()); 69 | 70 | // Delete everything 71 | for (int i = 0; i < kCount; i++) { 72 | ASSERT_OK(db_->Delete(WriteOptions(), Key(i))); 73 | } 74 | ASSERT_OK(dbi->TEST_CompactMemTable()); 75 | 76 | // Get initial measurement of the space we will be reading. 77 | const int64_t initial_size = Size(Key(0), Key(n)); 78 | const int64_t initial_other_size = Size(Key(n), Key(kCount)); 79 | 80 | // Read until size drops significantly. 81 | std::string limit_key = Key(n); 82 | for (int read = 0; true; read++) { 83 | ASSERT_LT(read, 100) << "Taking too long to compact"; 84 | Iterator* iter = db_->NewIterator(ReadOptions()); 85 | for (iter->SeekToFirst(); 86 | iter->Valid() && iter->key().ToString() < limit_key; 87 | iter->Next()) { 88 | // Drop data 89 | } 90 | delete iter; 91 | // Wait a little bit to allow any triggered compactions to complete. 92 | Env::Default()->SleepForMicroseconds(1000000); 93 | uint64_t size = Size(Key(0), Key(n)); 94 | fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", 95 | read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0); 96 | if (size <= initial_size/10) { 97 | break; 98 | } 99 | } 100 | 101 | // Verify that the size of the key space not touched by the reads 102 | // is pretty much unchanged. 103 | const int64_t final_other_size = Size(Key(n), Key(kCount)); 104 | ASSERT_LE(final_other_size, initial_other_size + 1048576); 105 | ASSERT_GE(final_other_size, initial_other_size/5 - 1048576); 106 | } 107 | 108 | TEST(AutoCompactTest, ReadAll) { 109 | DoReads(kCount); 110 | } 111 | 112 | #if 0 113 | TEST(AutoCompactTest, ReadHalf) { 114 | DoReads(kCount/2); 115 | } 116 | #endif 117 | 118 | } // namespace leveldb 119 | 120 | int main(int argc, char** argv) { 121 | return leveldb::test::RunAllTests(); 122 | } 123 | -------------------------------------------------------------------------------- /src/db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 6 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 7 | 8 | #include "pebblesdb/status.h" 9 | #include "db/version_set.h" 10 | 11 | namespace leveldb { 12 | 13 | struct Options; 14 | struct FileMetaData; 15 | 16 | class Env; 17 | class Iterator; 18 | class TableCache; 19 | class VersionEdit; 20 | 21 | extern Status BuildLevel0Tables(const std::string& dbname, 22 | Env* env, 23 | const Options& options, 24 | TableCache* table_cache, 25 | Iterator* iter, 26 | std::vector *meta_list, 27 | std::vector *filter_list, 28 | VersionSet* versions_, 29 | std::set* pending_outputs_, 30 | std::vector complete_guards_, 31 | port::Mutex* mutex_, 32 | uint64_t* reserved_file_numbers, 33 | FileLevelFilterBuilder* file_level_filter_builder); 34 | 35 | // Build a Table file from the contents of *iter. The generated file 36 | // will be named according to meta->number. On success, the rest of 37 | // *meta will be filled with metadata about the generated table. 38 | // If no data is present in *iter, meta->file_size will be set to 39 | // zero, and no Table file will be produced. 40 | extern Status BuildTable(const std::string& dbname, 41 | Env* env, 42 | const Options& options, 43 | TableCache* table_cache, 44 | Iterator* iter, 45 | FileMetaData* meta); 46 | 47 | } // namespace leveldb 48 | 49 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 50 | -------------------------------------------------------------------------------- /src/db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 6 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 7 | 8 | #include 9 | #include "pebblesdb/db.h" 10 | #include "db/dbformat.h" 11 | 12 | namespace leveldb { 13 | 14 | class DBImpl; 15 | 16 | // Return a new iterator that converts internal keys (yielded by 17 | // "*internal_iter") that were live at the specified "sequence" number 18 | // into appropriate user keys. 19 | extern Iterator* NewDBIterator( 20 | DBImpl* db, 21 | const Comparator* user_key_comparator, 22 | Iterator* internal_iter, 23 | SequenceNumber sequence, 24 | uint32_t seed); 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 29 | -------------------------------------------------------------------------------- /src/db/dbformat_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/dbformat.h" 6 | #include "util/logging.h" 7 | #include "util/testharness.h" 8 | 9 | namespace leveldb { 10 | 11 | static std::string IKey(const std::string& user_key, 12 | uint64_t seq, 13 | ValueType vt) { 14 | std::string encoded; 15 | AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); 16 | return encoded; 17 | } 18 | 19 | static std::string Shorten(const std::string& s, const std::string& l) { 20 | std::string result = s; 21 | InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); 22 | return result; 23 | } 24 | 25 | static std::string ShortSuccessor(const std::string& s) { 26 | std::string result = s; 27 | InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); 28 | return result; 29 | } 30 | 31 | static void TestKey(const std::string& key, 32 | uint64_t seq, 33 | ValueType vt) { 34 | std::string encoded = IKey(key, seq, vt); 35 | 36 | Slice in(encoded); 37 | ParsedInternalKey decoded("", 0, kTypeValue); 38 | 39 | ASSERT_TRUE(ParseInternalKey(in, &decoded)); 40 | ASSERT_EQ(key, decoded.user_key.ToString()); 41 | ASSERT_EQ(seq, decoded.sequence); 42 | ASSERT_EQ(vt, decoded.type); 43 | 44 | ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); 45 | } 46 | 47 | class FormatTest { }; 48 | 49 | TEST(FormatTest, InternalKey_EncodeDecode) { 50 | const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; 51 | const uint64_t seq[] = { 52 | 1, 2, 3, 53 | (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, 54 | (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, 55 | (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 56 | }; 57 | for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { 58 | for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { 59 | TestKey(keys[k], seq[s], kTypeValue); 60 | TestKey("hello", 1, kTypeDeletion); 61 | } 62 | } 63 | } 64 | 65 | TEST(FormatTest, InternalKeyShortSeparator) { 66 | // When user keys are same 67 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 68 | Shorten(IKey("foo", 100, kTypeValue), 69 | IKey("foo", 99, kTypeValue))); 70 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 71 | Shorten(IKey("foo", 100, kTypeValue), 72 | IKey("foo", 101, kTypeValue))); 73 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 74 | Shorten(IKey("foo", 100, kTypeValue), 75 | IKey("foo", 100, kTypeValue))); 76 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 77 | Shorten(IKey("foo", 100, kTypeValue), 78 | IKey("foo", 100, kTypeDeletion))); 79 | 80 | // When user keys are misordered 81 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 82 | Shorten(IKey("foo", 100, kTypeValue), 83 | IKey("bar", 99, kTypeValue))); 84 | 85 | // When user keys are different, but correctly ordered 86 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 87 | Shorten(IKey("foo", 100, kTypeValue), 88 | IKey("hello", 200, kTypeValue))); 89 | 90 | // When start user key is prefix of limit user key 91 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 92 | Shorten(IKey("foo", 100, kTypeValue), 93 | IKey("foobar", 200, kTypeValue))); 94 | 95 | // When limit user key is prefix of start user key 96 | ASSERT_EQ(IKey("foobar", 100, kTypeValue), 97 | Shorten(IKey("foobar", 100, kTypeValue), 98 | IKey("foo", 200, kTypeValue))); 99 | } 100 | 101 | TEST(FormatTest, InternalKeyShortestSuccessor) { 102 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 103 | ShortSuccessor(IKey("foo", 100, kTypeValue))); 104 | ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), 105 | ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); 106 | } 107 | 108 | } // namespace leveldb 109 | 110 | int main(int argc, char** argv) { 111 | return leveldb::test::RunAllTests(); 112 | } 113 | -------------------------------------------------------------------------------- /src/db/filename.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // File names used by DB code 6 | 7 | #ifndef STORAGE_LEVELDB_DB_FILENAME_H_ 8 | #define STORAGE_LEVELDB_DB_FILENAME_H_ 9 | 10 | #include 11 | #include 12 | #include "pebblesdb/slice.h" 13 | #include "pebblesdb/status.h" 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Env; 19 | 20 | enum FileType { 21 | kLogFile, 22 | kDBLockFile, 23 | kTableFile, 24 | kDescriptorFile, 25 | kCurrentFile, 26 | kTempFile, 27 | kInfoLogFile // Either the current one, or an old one 28 | }; 29 | 30 | // Return the name of the log file with the specified number 31 | // in the db named by "dbname". The result will be prefixed with 32 | // "dbname". 33 | extern std::string LogFileName(const std::string& dbname, uint64_t number); 34 | 35 | // Return the name of the sstable with the specified number 36 | // in the db named by "dbname". The result will be prefixed with 37 | // "dbname". 38 | extern std::string TableFileName(const std::string& dbname, uint64_t number); 39 | 40 | // Return the legacy file name for an sstable with the specified number 41 | // in the db named by "dbname". The result will be prefixed with 42 | // "dbname". 43 | extern std::string LDBTableFileName(const std::string& dbname, uint64_t number); 44 | 45 | // Return the name of the descriptor file for the db named by 46 | // "dbname" and the specified incarnation number. The result will be 47 | // prefixed with "dbname". 48 | extern std::string DescriptorFileName(const std::string& dbname, 49 | uint64_t number); 50 | 51 | // Return the name of the current file. This file contains the name 52 | // of the current manifest file. The result will be prefixed with 53 | // "dbname". 54 | extern std::string CurrentFileName(const std::string& dbname); 55 | 56 | // Return the name of the lock file for the db named by 57 | // "dbname". The result will be prefixed with "dbname". 58 | extern std::string LockFileName(const std::string& dbname); 59 | 60 | // Return the name of a temporary file owned by the db named "dbname". 61 | // The result will be prefixed with "dbname". 62 | extern std::string TempFileName(const std::string& dbname, uint64_t number); 63 | 64 | // Return the name of the info log file for "dbname". 65 | extern std::string InfoLogFileName(const std::string& dbname); 66 | 67 | // Return the name of the old info log file for "dbname". 68 | extern std::string OldInfoLogFileName(const std::string& dbname); 69 | 70 | // If filename is a leveldb file, store the type of the file in *type. 71 | // The number encoded in the filename is stored in *number. If the 72 | // filename was successfully parsed, returns true. Else return false. 73 | extern bool ParseFileName(const std::string& filename, 74 | uint64_t* number, 75 | FileType* type); 76 | 77 | // Make the CURRENT file point to the descriptor file with the 78 | // specified number. 79 | extern Status SetCurrentFile(Env* env, const std::string& dbname, 80 | uint64_t descriptor_number); 81 | 82 | 83 | } // namespace leveldb 84 | 85 | #endif // STORAGE_LEVELDB_DB_FILENAME_H_ 86 | -------------------------------------------------------------------------------- /src/db/filename_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/filename.h" 6 | 7 | #include "db/dbformat.h" 8 | #include "port/port.h" 9 | #include "util/logging.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class FileNameTest { }; 15 | 16 | TEST(FileNameTest, Parse) { 17 | Slice db; 18 | FileType type; 19 | uint64_t number; 20 | 21 | // Successful parses 22 | static struct { 23 | const char* fname; 24 | uint64_t number; 25 | FileType type; 26 | } cases[] = { 27 | { "100.log", 100, kLogFile }, 28 | { "0.log", 0, kLogFile }, 29 | { "0.sst", 0, kTableFile }, 30 | { "0.ldb", 0, kTableFile }, 31 | { "CURRENT", 0, kCurrentFile }, 32 | { "LOCK", 0, kDBLockFile }, 33 | { "MANIFEST-2", 2, kDescriptorFile }, 34 | { "MANIFEST-7", 7, kDescriptorFile }, 35 | { "LOG", 0, kInfoLogFile }, 36 | { "LOG.old", 0, kInfoLogFile }, 37 | { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, 38 | }; 39 | for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { 40 | std::string f = cases[i].fname; 41 | ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; 42 | ASSERT_EQ(cases[i].type, type) << f; 43 | ASSERT_EQ(cases[i].number, number) << f; 44 | } 45 | 46 | // Errors 47 | static const char* errors[] = { 48 | "", 49 | "foo", 50 | "foo-dx-100.log", 51 | ".log", 52 | "", 53 | "manifest", 54 | "CURREN", 55 | "CURRENTX", 56 | "MANIFES", 57 | "MANIFEST", 58 | "MANIFEST-", 59 | "XMANIFEST-3", 60 | "MANIFEST-3x", 61 | "LOC", 62 | "LOCKx", 63 | "LO", 64 | "LOGx", 65 | "18446744073709551616.log", 66 | "184467440737095516150.log", 67 | "100", 68 | "100.", 69 | "100.lop" 70 | }; 71 | for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { 72 | std::string f = errors[i]; 73 | ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; 74 | } 75 | } 76 | 77 | TEST(FileNameTest, Construction) { 78 | uint64_t number; 79 | FileType type; 80 | std::string fname; 81 | 82 | fname = CurrentFileName("foo"); 83 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 84 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 85 | ASSERT_EQ(0, number); 86 | ASSERT_EQ(kCurrentFile, type); 87 | 88 | fname = LockFileName("foo"); 89 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 90 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 91 | ASSERT_EQ(0, number); 92 | ASSERT_EQ(kDBLockFile, type); 93 | 94 | fname = LogFileName("foo", 192); 95 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 96 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 97 | ASSERT_EQ(192, number); 98 | ASSERT_EQ(kLogFile, type); 99 | 100 | fname = TableFileName("bar", 200); 101 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 102 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 103 | ASSERT_EQ(200, number); 104 | ASSERT_EQ(kTableFile, type); 105 | 106 | fname = DescriptorFileName("bar", 100); 107 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 108 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 109 | ASSERT_EQ(100, number); 110 | ASSERT_EQ(kDescriptorFile, type); 111 | 112 | fname = TempFileName("tmp", 999); 113 | ASSERT_EQ("tmp/", std::string(fname.data(), 4)); 114 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 115 | ASSERT_EQ(999, number); 116 | ASSERT_EQ(kTempFile, type); 117 | } 118 | 119 | } // namespace leveldb 120 | 121 | int main(int argc, char** argv) { 122 | return leveldb::test::RunAllTests(); 123 | } 124 | -------------------------------------------------------------------------------- /src/db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Log format information shared by reader and writer. 6 | // See ../doc/log_format.txt for more detail. 7 | 8 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 9 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 10 | 11 | namespace leveldb { 12 | namespace log { 13 | 14 | enum RecordType { 15 | // Zero is reserved for preallocated files 16 | kZeroType = 0, 17 | 18 | kFullType = 1, 19 | 20 | // For fragments 21 | kFirstType = 2, 22 | kMiddleType = 3, 23 | kLastType = 4 24 | }; 25 | static const unsigned kMaxRecordType = kLastType; 26 | 27 | static const unsigned kBlockSize = 32768; 28 | 29 | // Header is checksum (4 bytes), type (1 byte), length (2 bytes). 30 | static const unsigned kHeaderSize = 4 + 1 + 2; 31 | 32 | } // namespace log 33 | } // namespace leveldb 34 | 35 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 36 | -------------------------------------------------------------------------------- /src/db/log_reader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_READER_H_ 7 | 8 | #include 9 | 10 | #include "db/log_format.h" 11 | #include "pebblesdb/slice.h" 12 | #include "pebblesdb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | class SequentialFile; 17 | 18 | namespace log { 19 | 20 | class Reader { 21 | public: 22 | // Interface for reporting errors. 23 | class Reporter { 24 | public: 25 | virtual ~Reporter(); 26 | 27 | // Some corruption was detected. "size" is the approximate number 28 | // of bytes dropped due to the corruption. 29 | virtual void Corruption(size_t bytes, const Status& status) = 0; 30 | }; 31 | 32 | // Create a reader that will return log records from "*file". 33 | // "*file" must remain live while this Reader is in use. 34 | // 35 | // If "reporter" is non-NULL, it is notified whenever some data is 36 | // dropped due to a detected corruption. "*reporter" must remain 37 | // live while this Reader is in use. 38 | // 39 | // If "checksum" is true, verify checksums if available. 40 | // 41 | // The Reader will start reading at the first record located at physical 42 | // position >= initial_offset within the file. 43 | Reader(SequentialFile* file, Reporter* reporter, bool checksum, 44 | uint64_t initial_offset); 45 | 46 | ~Reader(); 47 | 48 | // Read the next record into *record. Returns true if read 49 | // successfully, false if we hit end of the input. May use 50 | // "*scratch" as temporary storage. The contents filled in *record 51 | // will only be valid until the next mutating operation on this 52 | // reader or the next mutation to *scratch. 53 | bool ReadRecord(Slice* record, std::string* scratch); 54 | 55 | // Returns the physical offset of the last record returned by ReadRecord. 56 | // 57 | // Undefined before the first call to ReadRecord. 58 | uint64_t LastRecordOffset(); 59 | 60 | private: 61 | SequentialFile* const file_; 62 | Reporter* const reporter_; 63 | bool const checksum_; 64 | char* const backing_store_; 65 | Slice buffer_; 66 | bool eof_; // Last Read() indicated EOF by returning < kBlockSize 67 | 68 | // Offset of the last record returned by ReadRecord. 69 | uint64_t last_record_offset_; 70 | // Offset of the first location past the end of buffer_. 71 | uint64_t end_of_buffer_offset_; 72 | 73 | // Offset at which to start looking for the first record to return 74 | uint64_t const initial_offset_; 75 | 76 | // Extend record types with the following special values 77 | enum { 78 | kEof = kMaxRecordType + 1, 79 | // Returned whenever we find an invalid physical record. 80 | // Currently there are three situations in which this happens: 81 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 82 | // * The record is a 0-length record (No drop is reported) 83 | // * The record is below constructor's initial_offset (No drop is reported) 84 | kBadRecord = kMaxRecordType + 2 85 | }; 86 | 87 | // Skips all blocks that are completely before "initial_offset_". 88 | // 89 | // Returns true on success. Handles reporting. 90 | bool SkipToInitialBlock(); 91 | 92 | // Return type, or one of the preceding special values 93 | unsigned int ReadPhysicalRecord(Slice* result); 94 | 95 | // Reports dropped bytes to the reporter. 96 | // buffer_ must be updated to remove the dropped bytes prior to invocation. 97 | void ReportCorruption(size_t bytes, const char* reason); 98 | void ReportDrop(size_t bytes, const Status& reason); 99 | 100 | // No copying allowed 101 | Reader(const Reader&); 102 | void operator=(const Reader&); 103 | }; 104 | 105 | } // namespace log 106 | } // namespace leveldb 107 | 108 | #endif // STORAGE_LEVELDB_DB_LOG_READER_H_ 109 | -------------------------------------------------------------------------------- /src/db/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ 7 | 8 | #include 9 | #include "db/log_format.h" 10 | #include "pebblesdb/slice.h" 11 | #include "pebblesdb/status.h" 12 | #include "port/port.h" 13 | 14 | namespace leveldb { 15 | 16 | class ConcurrentWritableFile; 17 | 18 | namespace log { 19 | 20 | class Writer { 21 | public: 22 | // Create a writer that will append data to "*dest". 23 | // "*dest" must be initially empty. 24 | // "*dest" must remain live while this Writer is in use. 25 | explicit Writer(ConcurrentWritableFile* dest); 26 | ~Writer(); 27 | 28 | Status AddRecord(const Slice& slice); 29 | 30 | private: 31 | ConcurrentWritableFile* dest_; 32 | uint64_t offset_; // Current offset in file 33 | 34 | // crc32c values for all supported record types. These are 35 | // pre-computed to reduce the overhead of computing the crc of the 36 | // record type stored in the header. 37 | uint32_t type_crc_[kMaxRecordType + 1]; 38 | 39 | uint64_t ComputeRecordSize(uint64_t start, uint64_t remain); 40 | Status EmitPhysicalRecordAt(RecordType type, const char* ptr, uint64_t offset, size_t length); 41 | 42 | // No copying allowed 43 | Writer(const Writer&); 44 | void operator=(const Writer&); 45 | }; 46 | 47 | } // namespace log 48 | } // namespace leveldb 49 | 50 | #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ 51 | -------------------------------------------------------------------------------- /src/db/memtable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_ 6 | #define STORAGE_LEVELDB_DB_MEMTABLE_H_ 7 | 8 | #include 9 | #include "pebblesdb/db.h" 10 | #include "db/dbformat.h" 11 | #include "db/skiplist.h" 12 | #include "util/arena.h" 13 | #include "util/atomic.h" 14 | 15 | namespace leveldb { 16 | 17 | class InternalKeyComparator; 18 | class Mutex; 19 | class MemTableIterator; 20 | 21 | class MemTable { 22 | public: 23 | // MemTables are reference counted. The initial reference count 24 | // is zero and the caller must call Ref() at least once. 25 | explicit MemTable(const InternalKeyComparator& comparator); 26 | 27 | // Increase reference count. 28 | void Ref() { atomic::increment_64_fullbarrier(&refs_, 1); } 29 | 30 | // Drop reference count. Delete if no more references exist. 31 | void Unref() { 32 | uint64_t ref = atomic::increment_64_fullbarrier(&refs_, -1); 33 | if (ref == 0) { 34 | delete this; 35 | } 36 | } 37 | 38 | // Returns an estimate of the number of bytes of data in use by this 39 | // data structure. 40 | // 41 | // REQUIRES: external synchronization to prevent simultaneous 42 | // operations on the same MemTable. 43 | size_t ApproximateMemoryUsage(); 44 | 45 | // Return an iterator that yields the contents of the memtable. 46 | // 47 | // The caller must ensure that the underlying MemTable remains live 48 | // while the returned iterator is live. The keys returned by this 49 | // iterator are internal keys encoded by AppendInternalKey in the 50 | // db/format.{h,cc} module. 51 | Iterator* NewIterator(); 52 | 53 | // Add an entry into memtable that maps key to value at the 54 | // specified sequence number and with the specified type. 55 | // Typically value will be empty if type==kTypeDeletion. 56 | void Add(SequenceNumber seq, ValueType type, 57 | const Slice& key, 58 | const Slice& value); 59 | 60 | // If memtable contains a value for key, store it in *value and return true. 61 | // If memtable contains a deletion for key, store a NotFound() error 62 | // in *status and return true. 63 | // Else, return false. 64 | bool Get(const LookupKey& key, std::string* value, Status* s); 65 | 66 | int num_entries; 67 | 68 | private: 69 | ~MemTable(); // Private since only Unref() should be used to delete it 70 | 71 | struct KeyComparator { 72 | const InternalKeyComparator comparator; 73 | explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } 74 | int operator()(const char* a, const char* b) const; 75 | }; 76 | struct KeyExtractor { 77 | const InternalKeyComparator comparator; 78 | explicit KeyExtractor(const InternalKeyComparator& c) : comparator(c) { } 79 | uint64_t operator()(const char* k) const; 80 | }; 81 | friend class MemTableIterator; 82 | friend class MemTableBackwardIterator; 83 | 84 | typedef SkipList Table; 85 | 86 | KeyComparator comparator_; 87 | KeyExtractor extractor_; 88 | uint64_t refs_; 89 | Arena arena_; 90 | Table table_; 91 | 92 | // No copying allowed 93 | MemTable(const MemTable&); 94 | void operator=(const MemTable&); 95 | }; 96 | 97 | } // namespace leveldb 98 | 99 | #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_ 100 | -------------------------------------------------------------------------------- /src/db/murmurhash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else// defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); 30 | 31 | void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); 32 | 33 | void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // _MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /src/db/replay_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The HyperLevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_ROLLING_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_DB_ROLLING_ITERATOR_H_ 7 | 8 | #include 9 | #include 10 | #include "pebblesdb/db.h" 11 | #include "pebblesdb/replay_iterator.h" 12 | #include "db/dbformat.h" 13 | #include "db/memtable.h" 14 | 15 | namespace leveldb { 16 | 17 | class DBImpl; 18 | struct ReplayState { 19 | ReplayState(Iterator* i, SequenceNumber s, SequenceNumber l); 20 | ReplayState(MemTable* m, SequenceNumber s); 21 | MemTable* mem_; 22 | Iterator* iter_; 23 | SequenceNumber seq_start_; 24 | SequenceNumber seq_limit_; 25 | }; 26 | 27 | class ReplayIteratorImpl : public ReplayIterator { 28 | public: 29 | // Refs the memtable on its own; caller must hold mutex while creating this 30 | ReplayIteratorImpl(DBImpl* db, port::Mutex* mutex, const Comparator* cmp, 31 | Iterator* iter, MemTable* m, SequenceNumber s); 32 | virtual bool Valid(); 33 | virtual void Next(); 34 | virtual void SkipTo(const Slice& target); 35 | virtual void SkipToLast(); 36 | virtual bool HasValue(); 37 | virtual Slice key() const; 38 | virtual Slice value() const; 39 | virtual Status status() const; 40 | 41 | // extra interface 42 | 43 | // we ref the memtable; caller holds mutex passed into ctor 44 | // REQUIRES: caller must hold mutex passed into ctor 45 | void enqueue(MemTable* m, SequenceNumber s); 46 | 47 | // REQUIRES: caller must hold mutex passed into ctor 48 | void cleanup(); // calls delete this; 49 | 50 | private: 51 | friend class DB; 52 | virtual ~ReplayIteratorImpl(); 53 | bool ParseKey(ParsedInternalKey* ikey); 54 | bool ParseKey(const Slice& k, ParsedInternalKey* ikey); 55 | void Prime(); 56 | 57 | DBImpl* const db_; 58 | port::Mutex* mutex_; 59 | const Comparator* const user_comparator_; 60 | SequenceNumber const start_at_; 61 | bool valid_; 62 | Status status_; 63 | 64 | bool has_current_user_key_; 65 | std::string current_user_key_; 66 | SequenceNumber current_user_sequence_; 67 | 68 | ReplayState rs_; 69 | std::list mems_; 70 | 71 | ReplayIteratorImpl(const ReplayIteratorImpl&); 72 | ReplayIteratorImpl& operator = (const ReplayIteratorImpl&); 73 | }; 74 | 75 | } // namespace leveldb 76 | 77 | #endif // STORAGE_LEVELDB_DB_ROLLING_ITERATOR_H_ 78 | -------------------------------------------------------------------------------- /src/db/snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ 6 | #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ 7 | 8 | #include "pebblesdb/db.h" 9 | 10 | namespace leveldb { 11 | 12 | class SnapshotList; 13 | 14 | // Snapshots are kept in a doubly-linked list in the DB. 15 | // Each SnapshotImpl corresponds to a particular sequence number. 16 | class SnapshotImpl : public Snapshot { 17 | public: 18 | SnapshotImpl() 19 | : number_(), 20 | prev_(NULL), 21 | next_(NULL), 22 | list_(NULL) { 23 | } 24 | SequenceNumber number_; // const after creation 25 | 26 | private: 27 | SnapshotImpl(const SnapshotImpl&); 28 | SnapshotImpl& operator = (const SnapshotImpl&); 29 | friend class SnapshotList; 30 | 31 | // SnapshotImpl is kept in a doubly-linked circular list 32 | SnapshotImpl* prev_; 33 | SnapshotImpl* next_; 34 | 35 | SnapshotList* list_; // just for sanity checks 36 | }; 37 | 38 | class SnapshotList { 39 | public: 40 | SnapshotList() 41 | : list_() { 42 | list_.prev_ = &list_; 43 | list_.next_ = &list_; 44 | } 45 | 46 | bool empty() const { return list_.next_ == &list_; } 47 | SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } 48 | SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } 49 | 50 | const SnapshotImpl* New(SequenceNumber seq) { 51 | SnapshotImpl* s = new SnapshotImpl; 52 | s->number_ = seq; 53 | s->list_ = this; 54 | s->next_ = &list_; 55 | s->prev_ = list_.prev_; 56 | s->prev_->next_ = s; 57 | s->next_->prev_ = s; 58 | return s; 59 | } 60 | 61 | void Delete(const SnapshotImpl* s) { 62 | assert(s->list_ == this); 63 | s->prev_->next_ = s->next_; 64 | s->next_->prev_ = s->prev_; 65 | delete s; 66 | } 67 | 68 | private: 69 | // Dummy head of doubly-linked list of snapshots 70 | SnapshotImpl list_; 71 | }; 72 | 73 | } // namespace leveldb 74 | 75 | #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ 76 | -------------------------------------------------------------------------------- /src/db/table_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Thread-safe (provides internal synchronization) 6 | 7 | #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 8 | #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include "db/dbformat.h" 14 | #include "pebblesdb/cache.h" 15 | #include "pebblesdb/table.h" 16 | #include "port/port.h" 17 | #include "util/timer.h" 18 | #include "db/version_set.h" 19 | //#include 20 | 21 | namespace leveldb { 22 | 23 | class Env; 24 | 25 | class TableCache { 26 | public: 27 | TableCache(const std::string& dbname, const Options* options, int entries); 28 | ~TableCache(); 29 | 30 | // Return an iterator for the specified file number (the corresponding 31 | // file length must be exactly "file_size" bytes). If "tableptr" is 32 | // non-NULL, also sets "*tableptr" to point to the Table object 33 | // underlying the returned iterator, or NULL if no Table object underlies 34 | // the returned iterator. The returned "*tableptr" object is owned by 35 | // the cache and should not be deleted, and is valid for as long as the 36 | // returned iterator is live. 37 | Iterator* NewIterator(const ReadOptions& options, 38 | uint64_t file_number, 39 | uint64_t file_size, 40 | Table** tableptr = NULL); 41 | 42 | // If a seek to internal key "k" in specified file finds an entry, 43 | // call (*handle_result)(arg, found_key, found_value). 44 | Status Get(const ReadOptions& options, 45 | uint64_t file_number, 46 | uint64_t file_size, 47 | const Slice& k, 48 | void* arg, 49 | void (*handle_result)(void*, const Slice&, const Slice&), 50 | Timer* timer); 51 | 52 | // Evict any entry for the specified file number 53 | void Evict(uint64_t file_number); 54 | 55 | void SetFileMetaDataMap(uint64_t file_number, uint64_t file_size, InternalKey smallest, InternalKey largest); 56 | 57 | FileMetaData* GetFileMetaDataForFile(uint64_t file_number) { 58 | if (file_metadata_map.count(file_number) > 0) { 59 | return file_metadata_map[file_number]; 60 | } 61 | return NULL; 62 | } 63 | 64 | void RemoveFileMetaDataMapForFile(uint64_t number) { 65 | if (file_metadata_map.count(number) > 0) { 66 | FileMetaData* f = file_metadata_map[number]; 67 | if (f != NULL) { 68 | delete f; 69 | } 70 | file_metadata_map.erase(number); 71 | } 72 | } 73 | Timer* static_timers_[NUM_SEEK_THREADS]; 74 | 75 | void PrintSeekThreadsStaticTimerAuditIndividual() { 76 | for (int i = 0; i < NUM_SEEK_THREADS; i++) { 77 | printf("-------------------------- Individual static timer information %d --------------------\n", i); 78 | printf("%s\n", static_timers_[i]->DebugString().c_str()); 79 | printf("-------------------------------------------------------------------\n"); 80 | } 81 | } 82 | 83 | void PrintSeekThreadsStaticTimerAuditCumulative() { 84 | Timer* cum_timer = new Timer(); 85 | for (int i = 0; i < NUM_SEEK_THREADS; i++) { 86 | cum_timer->AppendTimerInfo(static_timers_[i]); 87 | } 88 | printf("-------------------------- Cumulative static timer information --------------------\n"); 89 | printf("%s\n", cum_timer->DebugString().c_str()); 90 | printf("-------------------------------------------------------------------\n"); 91 | delete cum_timer; 92 | } 93 | 94 | 95 | private: 96 | TableCache(const TableCache&); 97 | TableCache& operator = (const TableCache&); 98 | Env* const env_; 99 | const std::string dbname_; 100 | const Options* options_; 101 | Cache* cache_; 102 | std::map file_metadata_map; 103 | // std::unordered_map cache_handle_map; 104 | 105 | Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**, Timer* timer); 106 | }; 107 | 108 | } // namespace leveldb 109 | 110 | #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 111 | -------------------------------------------------------------------------------- /src/db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/version_edit.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | static void TestEncodeDecode(const VersionEdit& edit) { 11 | std::string encoded, encoded2; 12 | edit.EncodeTo(&encoded); 13 | VersionEdit parsed; 14 | Status s = parsed.DecodeFrom(encoded); 15 | ASSERT_TRUE(s.ok()) << s.ToString(); 16 | parsed.EncodeTo(&encoded2); 17 | ASSERT_EQ(encoded, encoded2); 18 | } 19 | 20 | class VersionEditTest { }; 21 | 22 | TEST(VersionEditTest, EncodeDecode) { 23 | static const uint64_t kBig = 1ull << 50; 24 | 25 | VersionEdit edit; 26 | for (int i = 0; i < 4; i++) { 27 | TestEncodeDecode(edit); 28 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 29 | InternalKey("foo", kBig + 500 + i, kTypeValue), 30 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 31 | edit.DeleteFile(4, kBig + 700 + i); 32 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 33 | } 34 | 35 | edit.SetComparatorName("foo"); 36 | edit.SetLogNumber(kBig + 100); 37 | edit.SetNextFile(kBig + 200); 38 | edit.SetLastSequence(kBig + 1000); 39 | TestEncodeDecode(edit); 40 | } 41 | 42 | } // namespace leveldb 43 | 44 | int main(int argc, char** argv) { 45 | return leveldb::test::RunAllTests(); 46 | } 47 | -------------------------------------------------------------------------------- /src/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include "pebblesdb/write_batch.h" 9 | 10 | namespace leveldb { 11 | 12 | class MemTable; 13 | class VersionEdit; 14 | class Version; 15 | 16 | // WriteBatchInternal provides static methods for manipulating a 17 | // WriteBatch that we don't want in the public WriteBatch interface. 18 | class WriteBatchInternal { 19 | public: 20 | // Return the number of entries in the batch. 21 | static int Count(const WriteBatch* batch); 22 | 23 | // Set the count for the number of entries in the batch. 24 | static void SetCount(WriteBatch* batch, int n); 25 | 26 | // Return the seqeunce number for the start of this batch. 27 | static SequenceNumber Sequence(const WriteBatch* batch); 28 | 29 | // Store the specified number as the seqeunce number for the start of 30 | // this batch. 31 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 32 | 33 | static Slice Contents(const WriteBatch* batch) { 34 | return Slice(batch->rep_); 35 | } 36 | 37 | static size_t ByteSize(const WriteBatch* batch) { 38 | return batch->rep_.size(); 39 | } 40 | 41 | static void SetContents(WriteBatch* batch, const Slice& contents); 42 | 43 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 44 | 45 | static Status InsertIntoVersion(const WriteBatch* batch, MemTable* memtable, Version* version); 46 | 47 | static Status SetGuards(const WriteBatch* batch, WriteBatch* new_batch); 48 | 49 | static void Append(WriteBatch* dst, const WriteBatch* src); 50 | }; 51 | 52 | } // namespace leveldb 53 | 54 | 55 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 56 | -------------------------------------------------------------------------------- /src/db/write_batch_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #define __STDC_LIMIT_MACROS 6 | 7 | #include "pebblesdb/db.h" 8 | 9 | #include "db/memtable.h" 10 | #include "db/write_batch_internal.h" 11 | #include "pebblesdb/env.h" 12 | #include "util/logging.h" 13 | #include "util/testharness.h" 14 | 15 | namespace leveldb { 16 | 17 | static std::string PrintContents(WriteBatch* b) { 18 | InternalKeyComparator cmp(BytewiseComparator()); 19 | MemTable* mem = new MemTable(cmp); 20 | mem->Ref(); 21 | std::string state; 22 | Status s = WriteBatchInternal::InsertInto(b, mem); 23 | int count = 0; 24 | Iterator* iter = mem->NewIterator(); 25 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 26 | ParsedInternalKey ikey; 27 | ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); 28 | switch (ikey.type) { 29 | case kTypeValue: 30 | state.append("Put("); 31 | state.append(ikey.user_key.ToString()); 32 | state.append(", "); 33 | state.append(iter->value().ToString()); 34 | state.append(")"); 35 | count++; 36 | break; 37 | case kTypeDeletion: 38 | state.append("Delete("); 39 | state.append(ikey.user_key.ToString()); 40 | state.append(")"); 41 | count++; 42 | break; 43 | } 44 | state.append("@"); 45 | state.append(NumberToString(ikey.sequence)); 46 | } 47 | delete iter; 48 | if (!s.ok()) { 49 | state.append("ParseError()"); 50 | } else if (count != WriteBatchInternal::Count(b)) { 51 | state.append("CountMismatch()"); 52 | } 53 | mem->Unref(); 54 | return state; 55 | } 56 | 57 | class WriteBatchTest { }; 58 | 59 | TEST(WriteBatchTest, Empty) { 60 | WriteBatch batch; 61 | ASSERT_EQ("", PrintContents(&batch)); 62 | ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); 63 | } 64 | 65 | TEST(WriteBatchTest, Multiple) { 66 | WriteBatch batch; 67 | batch.Put(Slice("foo"), Slice("bar")); 68 | batch.Delete(Slice("box")); 69 | batch.Put(Slice("baz"), Slice("boo")); 70 | WriteBatchInternal::SetSequence(&batch, 100); 71 | ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); 72 | ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); 73 | ASSERT_EQ("Put(baz, boo)@102" 74 | "Delete(box)@101" 75 | "Put(foo, bar)@100", 76 | PrintContents(&batch)); 77 | } 78 | 79 | TEST(WriteBatchTest, Corruption) { 80 | WriteBatch batch; 81 | batch.Put(Slice("foo"), Slice("bar")); 82 | batch.Delete(Slice("box")); 83 | WriteBatchInternal::SetSequence(&batch, 200); 84 | Slice contents = WriteBatchInternal::Contents(&batch); 85 | WriteBatchInternal::SetContents(&batch, 86 | Slice(contents.data(),contents.size()-1)); 87 | ASSERT_EQ("Put(foo, bar)@200" 88 | "ParseError()", 89 | PrintContents(&batch)); 90 | } 91 | 92 | TEST(WriteBatchTest, Append) { 93 | WriteBatch b1, b2; 94 | WriteBatchInternal::SetSequence(&b1, 200); 95 | WriteBatchInternal::SetSequence(&b2, 300); 96 | WriteBatchInternal::Append(&b1, &b2); 97 | ASSERT_EQ("", 98 | PrintContents(&b1)); 99 | b2.Put("a", "va"); 100 | WriteBatchInternal::Append(&b1, &b2); 101 | ASSERT_EQ("Put(a, va)@200", 102 | PrintContents(&b1)); 103 | b2.Clear(); 104 | b2.Put("b", "vb"); 105 | WriteBatchInternal::Append(&b1, &b2); 106 | ASSERT_EQ("Put(a, va)@200" 107 | "Put(b, vb)@201", 108 | PrintContents(&b1)); 109 | b2.Delete("foo"); 110 | WriteBatchInternal::Append(&b1, &b2); 111 | ASSERT_EQ("Put(a, va)@200" 112 | "Put(b, vb)@202" 113 | "Put(b, vb)@201" 114 | "Delete(foo)@203", 115 | PrintContents(&b1)); 116 | } 117 | 118 | } // namespace leveldb 119 | 120 | int main(int argc, char** argv) { 121 | return leveldb::test::RunAllTests(); 122 | } 123 | -------------------------------------------------------------------------------- /src/demo/README.md: -------------------------------------------------------------------------------- 1 | `installation_test.cc` is a demo on how to use PebblesDB. It also serves as a verification of the successful installation 2 | of PebblesDB. 3 | 4 | How to use it: `g++ installation_test.cc -lpebblesdb; ./a.out` -------------------------------------------------------------------------------- /src/demo/installation_test.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Zeyuan_Hu on 4/16/18. 3 | // 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | int main(){ 10 | leveldb::DB* db; 11 | leveldb::Options options; 12 | // If true, the database will be created if it is missing. 13 | // Default: false 14 | options.create_if_missing = true; 15 | leveldb::Status status = leveldb::DB::Open(options,"/tmp/testdb", &db); 16 | assert(status.ok()); 17 | 18 | //write key1,value1 19 | std::string key="key"; 20 | std::string value = "value"; 21 | 22 | status = db->Put(leveldb::WriteOptions(), key,value); 23 | assert(status.ok()); 24 | 25 | status = db->Get(leveldb::ReadOptions(), key, &value); 26 | assert(status.ok()); 27 | std::cout<Put(leveldb::WriteOptions(),key2,value); 32 | assert(status.ok()); 33 | status = db->Delete(leveldb::WriteOptions(), key); 34 | 35 | assert(status.ok()); 36 | 37 | status = db->Get(leveldb::ReadOptions(),key2, &value); 38 | 39 | assert(status.ok()); 40 | std::cout<Get(leveldb::ReadOptions(),key, &value); 43 | 44 | if(!status.ok()) 45 | std::cerr< 5 | [data block 1] 6 | [data block 2] 7 | ... 8 | [data block N] 9 | [meta block 1] 10 | ... 11 | [meta block K] 12 | [metaindex block] 13 | [index block] 14 | [Footer] (fixed size; starts at file_size - sizeof(Footer)) 15 | 16 | 17 | The file contains internal pointers. Each such pointer is called 18 | a BlockHandle and contains the following information: 19 | offset: varint64 20 | size: varint64 21 | See https://developers.google.com/protocol-buffers/docs/encoding#varints 22 | for an explanation of varint64 format. 23 | 24 | (1) The sequence of key/value pairs in the file are stored in sorted 25 | order and partitioned into a sequence of data blocks. These blocks 26 | come one after another at the beginning of the file. Each data block 27 | is formatted according to the code in block_builder.cc, and then 28 | optionally compressed. 29 | 30 | (2) After the data blocks we store a bunch of meta blocks. The 31 | supported meta block types are described below. More meta block types 32 | may be added in the future. Each meta block is again formatted using 33 | block_builder.cc and then optionally compressed. 34 | 35 | (3) A "metaindex" block. It contains one entry for every other meta 36 | block where the key is the name of the meta block and the value is a 37 | BlockHandle pointing to that meta block. 38 | 39 | (4) An "index" block. This block contains one entry per data block, 40 | where the key is a string >= last key in that data block and before 41 | the first key in the successive data block. The value is the 42 | BlockHandle for the data block. 43 | 44 | (6) At the very end of the file is a fixed length footer that contains 45 | the BlockHandle of the metaindex and index blocks as well as a magic number. 46 | metaindex_handle: char[p]; // Block handle for metaindex 47 | index_handle: char[q]; // Block handle for index 48 | padding: char[40-p-q]; // zeroed bytes to make fixed length 49 | // (40==2*BlockHandle::kMaxEncodedLength) 50 | magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) 51 | 52 | "filter" Meta Block 53 | ------------------- 54 | 55 | If a "FilterPolicy" was specified when the database was opened, a 56 | filter block is stored in each table. The "metaindex" block contains 57 | an entry that maps from "filter." to the BlockHandle for the filter 58 | block where "" is the string returned by the filter policy's 59 | "Name()" method. 60 | 61 | The filter block stores a sequence of filters, where filter i contains 62 | the output of FilterPolicy::CreateFilter() on all keys that are stored 63 | in a block whose file offset falls within the range 64 | 65 | [ i*base ... (i+1)*base-1 ] 66 | 67 | Currently, "base" is 2KB. So for example, if blocks X and Y start in 68 | the range [ 0KB .. 2KB-1 ], all of the keys in X and Y will be 69 | converted to a filter by calling FilterPolicy::CreateFilter(), and the 70 | resulting filter will be stored as the first filter in the filter 71 | block. 72 | 73 | The filter block is formatted as follows: 74 | 75 | [filter 0] 76 | [filter 1] 77 | [filter 2] 78 | ... 79 | [filter N-1] 80 | 81 | [offset of filter 0] : 4 bytes 82 | [offset of filter 1] : 4 bytes 83 | [offset of filter 2] : 4 bytes 84 | ... 85 | [offset of filter N-1] : 4 bytes 86 | 87 | [offset of beginning of offset array] : 4 bytes 88 | lg(base) : 1 byte 89 | 90 | The offset array at the end of the filter block allows efficient 91 | mapping from a data block offset to the corresponding filter. 92 | 93 | "stats" Meta Block 94 | ------------------ 95 | 96 | This meta block contains a bunch of stats. The key is the name 97 | of the statistic. The value contains the statistic. 98 | TODO(postrelease): record following stats. 99 | data size 100 | index size 101 | key size (uncompressed) 102 | value size (uncompressed) 103 | number of entries 104 | number of data blocks 105 | -------------------------------------------------------------------------------- /src/graphs/hyperdex-multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utsaslab/pebblesdb/703bd01bba47c586fc3fd07273452528826cd38e/src/graphs/hyperdex-multi.png -------------------------------------------------------------------------------- /src/graphs/micro-all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utsaslab/pebblesdb/703bd01bba47c586fc3fd07273452528826cd38e/src/graphs/micro-all.png -------------------------------------------------------------------------------- /src/graphs/mongo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utsaslab/pebblesdb/703bd01bba47c586fc3fd07273452528826cd38e/src/graphs/mongo.png -------------------------------------------------------------------------------- /src/graphs/multi-ycsb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/utsaslab/pebblesdb/703bd01bba47c586fc3fd07273452528826cd38e/src/graphs/multi-ycsb.png -------------------------------------------------------------------------------- /src/helpers/memenv/memenv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 6 | #define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Env; 11 | 12 | // Returns a new environment that stores its data in memory and delegates 13 | // all non-file-storage tasks to base_env. The caller must delete the result 14 | // when it is no longer needed. 15 | // *base_env must remain live while the result is in use. 16 | Env* NewMemEnv(Env* base_env); 17 | 18 | } // namespace leveldb 19 | 20 | #endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 21 | -------------------------------------------------------------------------------- /src/include/pebblesdb/cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Cache is an interface that maps keys to values. It has internal 6 | // synchronization and may be safely accessed concurrently from 7 | // multiple threads. It may automatically evict entries to make room 8 | // for new entries. Values have a specified charge against the cache 9 | // capacity. For example, a cache where the values are variable 10 | // length strings, may use the length of the string as the charge for 11 | // the string. 12 | // 13 | // A builtin cache implementation with a least-recently-used eviction 14 | // policy is provided. Clients may use their own implementations if 15 | // they want something more sophisticated (like scan-resistance, a 16 | // custom eviction policy, variable cache sizing, etc.) 17 | 18 | #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ 19 | #define STORAGE_LEVELDB_INCLUDE_CACHE_H_ 20 | 21 | #include 22 | #include "pebblesdb/slice.h" 23 | 24 | namespace leveldb { 25 | 26 | class Cache; 27 | 28 | // Create a new cache with a fixed size capacity. This implementation 29 | // of Cache uses a least-recently-used eviction policy. 30 | extern Cache* NewLRUCache(size_t capacity); 31 | 32 | class Cache { 33 | public: 34 | Cache() : rep_() { } 35 | 36 | // Destroys all existing entries by calling the "deleter" 37 | // function that was passed to the constructor. 38 | virtual ~Cache(); 39 | 40 | // Opaque handle to an entry stored in the cache. 41 | struct Handle { }; 42 | 43 | // Insert a mapping from key->value into the cache and assign it 44 | // the specified charge against the total cache capacity. 45 | // 46 | // Returns a handle that corresponds to the mapping. The caller 47 | // must call this->Release(handle) when the returned mapping is no 48 | // longer needed. 49 | // 50 | // When the inserted entry is no longer needed, the key and 51 | // value will be passed to "deleter". 52 | virtual Handle* Insert(const Slice& key, void* value, size_t charge, 53 | void (*deleter)(const Slice& key, void* value)) = 0; 54 | 55 | // If the cache has no mapping for "key", returns NULL. 56 | // 57 | // Else return a handle that corresponds to the mapping. The caller 58 | // must call this->Release(handle) when the returned mapping is no 59 | // longer needed. 60 | virtual Handle* Lookup(const Slice& key) = 0; 61 | 62 | // Release a mapping returned by a previous Lookup(). 63 | // REQUIRES: handle must not have been released yet. 64 | // REQUIRES: handle must have been returned by a method on *this. 65 | virtual void Release(Handle* handle) = 0; 66 | 67 | // Return the value encapsulated in a handle returned by a 68 | // successful Lookup(). 69 | // REQUIRES: handle must not have been released yet. 70 | // REQUIRES: handle must have been returned by a method on *this. 71 | virtual void* Value(Handle* handle) = 0; 72 | 73 | // If the cache contains entry for key, erase it. Note that the 74 | // underlying entry will be kept around until all existing handles 75 | // to it have been released. 76 | virtual void Erase(const Slice& key) = 0; 77 | 78 | // Return a new numeric id. May be used by multiple clients who are 79 | // sharing the same cache to partition the key space. Typically the 80 | // client will allocate a new id at startup and prepend the id to 81 | // its cache keys. 82 | virtual uint64_t NewId() = 0; 83 | 84 | private: 85 | void LRU_Remove(Handle* e); 86 | void LRU_Append(Handle* e); 87 | void Unref(Handle* e); 88 | 89 | struct Rep; 90 | Rep* rep_; 91 | 92 | // No copying allowed 93 | Cache(const Cache&); 94 | void operator=(const Cache&); 95 | }; 96 | 97 | } // namespace leveldb 98 | 99 | #endif // STORAGE_LEVELDB_UTIL_CACHE_H_ 100 | -------------------------------------------------------------------------------- /src/include/pebblesdb/comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | 13 | class Slice; 14 | 15 | // A Comparator object provides a total order across slices that are 16 | // used as keys in an sstable or a database. A Comparator implementation 17 | // must be thread-safe since leveldb may invoke its methods concurrently 18 | // from multiple threads. 19 | class Comparator { 20 | public: 21 | virtual ~Comparator(); 22 | 23 | // Three-way comparison. Returns value: 24 | // < 0 iff "a" < "b", 25 | // == 0 iff "a" == "b", 26 | // > 0 iff "a" > "b" 27 | virtual int Compare(const Slice& a, const Slice& b) const = 0; 28 | 29 | // The name of the comparator. Used to check for comparator 30 | // mismatches (i.e., a DB created with one comparator is 31 | // accessed using a different comparator. 32 | // 33 | // The client of this package should switch to a new name whenever 34 | // the comparator implementation changes in a way that will cause 35 | // the relative ordering of any two keys to change. 36 | // 37 | // Names starting with "leveldb." are reserved and should not be used 38 | // by any clients of this package. 39 | virtual const char* Name() const = 0; 40 | 41 | // Advanced functions: these are used to reduce the space requirements 42 | // for internal data structures like index blocks. 43 | 44 | // If *start < limit, changes *start to a short string in [start,limit). 45 | // Simple comparator implementations may return with *start unchanged, 46 | // i.e., an implementation of this method that does nothing is correct. 47 | virtual void FindShortestSeparator( 48 | std::string* start, 49 | const Slice& limit) const = 0; 50 | 51 | // Changes *key to a short string >= *key. 52 | // Simple comparator implementations may return with *key unchanged, 53 | // i.e., an implementation of this method that does nothing is correct. 54 | virtual void FindShortSuccessor(std::string* key) const = 0; 55 | 56 | // If unsure, return 0; 57 | virtual uint64_t KeyNum(const Slice& key) const; 58 | }; 59 | 60 | // Return a builtin comparator that uses lexicographic byte-wise 61 | // ordering. The result remains the property of this module and 62 | // must not be deleted. 63 | extern const Comparator* BytewiseComparator(); 64 | 65 | } // namespace leveldb 66 | 67 | #endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 68 | -------------------------------------------------------------------------------- /src/include/pebblesdb/filter_policy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A database can be configured with a custom FilterPolicy object. 6 | // This object is responsible for creating a small filter from a set 7 | // of keys. These filters are stored in leveldb and are consulted 8 | // automatically by leveldb to decide whether or not to read some 9 | // information from disk. In many cases, a filter can cut down the 10 | // number of disk seeks form a handful to a single disk seek per 11 | // DB::Get() call. 12 | // 13 | // Most people will want to use the builtin bloom filter support (see 14 | // NewBloomFilterPolicy() below). 15 | 16 | #ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 17 | #define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 18 | 19 | #include 20 | 21 | namespace leveldb { 22 | 23 | class Slice; 24 | 25 | class FilterPolicy { 26 | public: 27 | virtual ~FilterPolicy(); 28 | 29 | // Return the name of this policy. Note that if the filter encoding 30 | // changes in an incompatible way, the name returned by this method 31 | // must be changed. Otherwise, old incompatible filters may be 32 | // passed to methods of this type. 33 | virtual const char* Name() const = 0; 34 | 35 | // keys[0,n-1] contains a list of keys (potentially with duplicates) 36 | // that are ordered according to the user supplied comparator. 37 | // Append a filter that summarizes keys[0,n-1] to *dst. 38 | // 39 | // Warning: do not change the initial contents of *dst. Instead, 40 | // append the newly constructed filter to *dst. 41 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) 42 | const = 0; 43 | 44 | // "filter" contains the data appended by a preceding call to 45 | // CreateFilter() on this class. This method must return true if 46 | // the key was in the list of keys passed to CreateFilter(). 47 | // This method may return true or false if the key was not on the 48 | // list, but it should aim to return false with a high probability. 49 | virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; 50 | 51 | // accounting the in-memory usage for the filter. 52 | mutable size_t byte_size; 53 | mutable unsigned long filter_count; 54 | }; 55 | 56 | // Return a new filter policy that uses a bloom filter with approximately 57 | // the specified number of bits per key. A good value for bits_per_key 58 | // is 10, which yields a filter with ~ 1% false positive rate. 59 | // 60 | // Callers must delete the result after any database that is using the 61 | // result has been closed. 62 | // 63 | // Note: if you are using a custom comparator that ignores some parts 64 | // of the keys being compared, you must not use NewBloomFilterPolicy() 65 | // and must provide your own FilterPolicy that also ignores the 66 | // corresponding parts of the keys. For example, if the comparator 67 | // ignores trailing spaces, it would be incorrect to use a 68 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 69 | // trailing spaces in keys. 70 | extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); 71 | 72 | } 73 | 74 | #endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 75 | -------------------------------------------------------------------------------- /src/include/pebblesdb/iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // An iterator yields a sequence of key/value pairs from a source. 6 | // The following class defines the interface. Multiple implementations 7 | // are provided by this library. In particular, iterators are provided 8 | // to access the contents of a Table or a DB. 9 | // 10 | // Multiple threads can invoke const methods on an Iterator without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Iterator must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 17 | 18 | #include "pebblesdb/slice.h" 19 | #include "pebblesdb/status.h" 20 | 21 | namespace leveldb { 22 | 23 | class Iterator { 24 | public: 25 | Iterator(); 26 | virtual ~Iterator(); 27 | 28 | // An iterator is either positioned at a key/value pair, or 29 | // not valid. This method returns true iff the iterator is valid. 30 | virtual bool Valid() const = 0; 31 | 32 | // Position at the first key in the source. The iterator is Valid() 33 | // after this call iff the source is not empty. 34 | virtual void SeekToFirst() = 0; 35 | 36 | // Position at the last key in the source. The iterator is 37 | // Valid() after this call iff the source is not empty. 38 | virtual void SeekToLast() = 0; 39 | 40 | // Position at the first key in the source that at or past target 41 | // The iterator is Valid() after this call iff the source contains 42 | // an entry that comes at or past target. 43 | virtual void Seek(const Slice& target) = 0; 44 | 45 | // Moves to the next entry in the source. After this call, Valid() is 46 | // true iff the iterator was not positioned at the last entry in the source. 47 | // REQUIRES: Valid() 48 | virtual void Next() = 0; 49 | 50 | // Moves to the previous entry in the source. After this call, Valid() is 51 | // true iff the iterator was not positioned at the first entry in source. 52 | // REQUIRES: Valid() 53 | virtual void Prev() = 0; 54 | 55 | // Return the key for the current entry. The underlying storage for 56 | // the returned slice is valid only until the next modification of 57 | // the iterator. 58 | // REQUIRES: Valid() 59 | virtual Slice key() const = 0; 60 | 61 | // Return the value for the current entry. The underlying storage for 62 | // the returned slice is valid only until the next modification of 63 | // the iterator. 64 | // REQUIRES: !AtEnd() && !AtStart() 65 | virtual Slice value() const = 0; 66 | 67 | // If an error has occurred, return it. Else return an ok status. 68 | virtual const Status& status() const = 0; 69 | 70 | // Clients are allowed to register function/arg1/arg2 triples that 71 | // will be invoked when this iterator is destroyed. 72 | // 73 | // Note that unlike all of the preceding methods, this method is 74 | // not abstract and therefore clients should not override it. 75 | typedef void (*CleanupFunction)(void* arg1, void* arg2); 76 | void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); 77 | 78 | private: 79 | struct Cleanup { 80 | CleanupFunction function; 81 | void* arg1; 82 | void* arg2; 83 | Cleanup* next; 84 | }; 85 | Cleanup cleanup_; 86 | 87 | // No copying allowed 88 | Iterator(const Iterator&); 89 | void operator=(const Iterator&); 90 | }; 91 | 92 | // Return an empty iterator (yields nothing). 93 | extern Iterator* NewEmptyIterator(); 94 | 95 | // Return an empty iterator with the specified status. 96 | extern Iterator* NewErrorIterator(const Status& status); 97 | 98 | } // namespace leveldb 99 | 100 | #endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 101 | -------------------------------------------------------------------------------- /src/include/pebblesdb/replay_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The HyperLevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_REPLAY_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_REPLAY_ITERATOR_H_ 7 | 8 | #include "pebblesdb/slice.h" 9 | #include "pebblesdb/status.h" 10 | 11 | namespace leveldb { 12 | 13 | class ReplayIterator { 14 | public: 15 | ReplayIterator(); 16 | 17 | // An iterator is either positioned at a deleted key, present key/value pair, 18 | // or not valid. This method returns true iff the iterator is valid. 19 | virtual bool Valid() = 0; 20 | 21 | // Moves to the next entry in the source. After this call, Valid() is 22 | // true iff the iterator was not positioned at the last entry in the source. 23 | // REQUIRES: Valid() 24 | virtual void Next() = 0; 25 | 26 | // Position at the first key in the source that at or past target for this 27 | // pass. Note that this is unlike the Seek call, as the ReplayIterator is 28 | // unsorted. 29 | // The iterator is Valid() after this call iff the source contains 30 | // an entry that comes at or past target. 31 | virtual void SkipTo(const Slice& target) = 0; 32 | virtual void SkipToLast() = 0; 33 | 34 | // Return true if the current entry points to a key-value pair. If this 35 | // returns false, it means the current entry is a deleted entry. 36 | virtual bool HasValue() = 0; 37 | 38 | // Return the key for the current entry. The underlying storage for 39 | // the returned slice is valid only until the next modification of 40 | // the iterator. 41 | // REQUIRES: Valid() 42 | virtual Slice key() const = 0; 43 | 44 | // Return the value for the current entry. The underlying storage for 45 | // the returned slice is valid only until the next modification of 46 | // the iterator. 47 | // REQUIRES: !AtEnd() && !AtStart() 48 | virtual Slice value() const = 0; 49 | 50 | // If an error has occurred, return it. Else return an ok status. 51 | virtual Status status() const = 0; 52 | 53 | protected: 54 | // must be released by giving it back to the DB 55 | virtual ~ReplayIterator(); 56 | friend class DB; 57 | 58 | private: 59 | // No copying allowed 60 | ReplayIterator(const ReplayIterator&); 61 | void operator=(const ReplayIterator&); 62 | }; 63 | 64 | } // namespace leveldb 65 | 66 | #endif // STORAGE_LEVELDB_INCLUDE_REPLAY_ITERATOR_H_ 67 | -------------------------------------------------------------------------------- /src/include/pebblesdb/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Slice is a simple structure containing a pointer into some external 6 | // storage and a size. The user of a Slice must ensure that the slice 7 | // is not used after the corresponding external storage has been 8 | // deallocated. 9 | // 10 | // Multiple threads can invoke const methods on a Slice without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Slice must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_SLICE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace leveldb { 24 | 25 | class Slice { 26 | public: 27 | // Create an empty slice. 28 | Slice() : data_(""), size_(0) { } 29 | 30 | // Create a slice that refers to d[0,n-1]. 31 | Slice(const char* d, size_t n) : data_(d), size_(n) { } 32 | 33 | /* Slice(const Slice& slice) : data_(slice.data_), size_(slice.size_) { } */ 34 | 35 | // Create a slice that refers to the contents of "s" 36 | Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } 37 | 38 | // Create a slice that refers to s[0,strlen(s)-1] 39 | Slice(const char* s) : data_(s), size_(strlen(s)) { } 40 | 41 | // Return a pointer to the beginning of the referenced data 42 | const char* data() const { return data_; } 43 | 44 | // Return the length (in bytes) of the referenced data 45 | size_t size() const { return size_; } 46 | 47 | // Return true iff the length of the referenced data is zero 48 | bool empty() const { return size_ == 0; } 49 | 50 | // Return the ith byte in the referenced data. 51 | // REQUIRES: n < size() 52 | char operator[](size_t n) const { 53 | assert(n < size()); 54 | return data_[n]; 55 | } 56 | 57 | void set_data(const std::string &s) { 58 | data_ = s.data(); 59 | size_ = s.size(); 60 | } 61 | 62 | // Change this slice to refer to an empty array 63 | void clear() { data_ = ""; size_ = 0; } 64 | 65 | // Drop the first "n" bytes from this slice. 66 | void remove_prefix(size_t n) { 67 | assert(n <= size()); 68 | data_ += n; 69 | size_ -= n; 70 | } 71 | 72 | // Return a string that contains the copy of the referenced data. 73 | std::string ToString() const { return std::string(data_, size_); } 74 | 75 | // Three-way comparison. Returns value: 76 | // < 0 iff "*this" < "b", 77 | // == 0 iff "*this" == "b", 78 | // > 0 iff "*this" > "b" 79 | int compare(const Slice& b) const; 80 | 81 | // Return true iff "x" is a prefix of "*this" 82 | bool starts_with(const Slice& x) const { 83 | return ((size_ >= x.size_) && 84 | (memcmp(data_, x.data_, x.size_) == 0)); 85 | } 86 | 87 | private: 88 | const char* data_; 89 | size_t size_; 90 | 91 | // Intentionally copyable 92 | }; 93 | 94 | inline bool operator==(const Slice& x, const Slice& y) { 95 | return ((x.size() == y.size()) && 96 | (memcmp(x.data(), y.data(), x.size()) == 0)); 97 | } 98 | 99 | inline bool operator!=(const Slice& x, const Slice& y) { 100 | return !(x == y); 101 | } 102 | 103 | inline int Slice::compare(const Slice& b) const { 104 | const size_t min_len = (size_ < b.size_) ? size_ : b.size_; 105 | int r = memcmp(data_, b.data_, min_len); 106 | if (r == 0) { 107 | if (size_ < b.size_) r = -1; 108 | else if (size_ > b.size_) r = +1; 109 | } 110 | return r; 111 | } 112 | 113 | } // namespace leveldb 114 | 115 | 116 | #endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ 117 | -------------------------------------------------------------------------------- /src/include/pebblesdb/status.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Status encapsulates the result of an operation. It may indicate success, 6 | // or it may indicate an error with an associated error message. 7 | // 8 | // Multiple threads can invoke const methods on a Status without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same Status must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_STATUS_H_ 15 | 16 | #include 17 | #include "pebblesdb/slice.h" 18 | 19 | namespace leveldb { 20 | 21 | class Status { 22 | public: 23 | // Create a success status. 24 | Status() : state_(NULL) { } 25 | ~Status() { delete[] state_; } 26 | 27 | // Copy the specified status. 28 | Status(const Status& s); 29 | Status& operator=(const Status& s); 30 | 31 | // Return a success status. 32 | static Status OK() { return Status(); } 33 | 34 | // Return error status of an appropriate type. 35 | static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { 36 | return Status(kNotFound, msg, msg2); 37 | } 38 | static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { 39 | return Status(kCorruption, msg, msg2); 40 | } 41 | static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { 42 | return Status(kNotSupported, msg, msg2); 43 | } 44 | static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { 45 | return Status(kInvalidArgument, msg, msg2); 46 | } 47 | static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { 48 | return Status(kIOError, msg, msg2); 49 | } 50 | 51 | // Returns true iff the status indicates success. 52 | bool ok() const { return (state_ == NULL); } 53 | 54 | // Returns true iff the status indicates a NotFound error. 55 | bool IsNotFound() const { return code() == kNotFound; } 56 | 57 | // Returns true iff the status indicates a Corruption error. 58 | bool IsCorruption() const { return code() == kCorruption; } 59 | 60 | // Returns true iff the status indicates an IOError. 61 | bool IsIOError() const { return code() == kIOError; } 62 | 63 | // Return a string representation of this status suitable for printing. 64 | // Returns the string "OK" for success. 65 | std::string ToString() const; 66 | 67 | private: 68 | // OK status has a NULL state_. Otherwise, state_ is a new[] array 69 | // of the following form: 70 | // state_[0..3] == length of message 71 | // state_[4] == code 72 | // state_[5..] == message 73 | const char* state_; 74 | 75 | enum Code { 76 | kOk = 0, 77 | kNotFound = 1, 78 | kCorruption = 2, 79 | kNotSupported = 3, 80 | kInvalidArgument = 4, 81 | kIOError = 5 82 | }; 83 | 84 | Code code() const { 85 | return (state_ == NULL) ? kOk : static_cast(state_[4]); 86 | } 87 | 88 | Status(Code code, const Slice& msg, const Slice& msg2); 89 | static const char* CopyState(const char* s); 90 | }; 91 | 92 | inline Status::Status(const Status& s) : state_(NULL) { 93 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 94 | } 95 | inline Status& Status::operator=(const Status& s) { 96 | // The following condition catches both aliasing (when this == &s), 97 | // and the common case where both s and *this are ok. 98 | if (state_ != s.state_) { 99 | delete[] state_; 100 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 101 | } 102 | return *this; 103 | } 104 | 105 | } // namespace leveldb 106 | 107 | #endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_ 108 | -------------------------------------------------------------------------------- /src/include/pebblesdb/table.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_TABLE_H_ 7 | 8 | #include 9 | #include "pebblesdb/iterator.h" 10 | #include "util/timer.h" 11 | 12 | namespace leveldb { 13 | 14 | class Block; 15 | class BlockHandle; 16 | class Footer; 17 | struct Options; 18 | class RandomAccessFile; 19 | struct ReadOptions; 20 | class TableCache; 21 | 22 | // A Table is a sorted map from strings to strings. Tables are 23 | // immutable and persistent. A Table may be safely accessed from 24 | // multiple threads without external synchronization. 25 | class Table { 26 | public: 27 | // Attempt to open the table that is stored in bytes [0..file_size) 28 | // of "file", and read the metadata entries necessary to allow 29 | // retrieving data from the table. 30 | // 31 | // If successful, returns ok and sets "*table" to the newly opened 32 | // table. The client should delete "*table" when no longer needed. 33 | // If there was an error while initializing the table, sets "*table" 34 | // to NULL and returns a non-ok status. Does not take ownership of 35 | // "*source", but the client must ensure that "source" remains live 36 | // for the duration of the returned table's lifetime. 37 | // 38 | // *file must remain live while this Table is in use. 39 | static Status Open(const Options& options, 40 | RandomAccessFile* file, 41 | uint64_t file_size, 42 | Table** table, 43 | Timer* timer); 44 | 45 | ~Table(); 46 | 47 | // Returns a new iterator over the table contents. 48 | // The result of NewIterator() is initially invalid (caller must 49 | // call one of the Seek methods on the iterator before using it). 50 | Iterator* NewIterator(const ReadOptions&) const; 51 | 52 | // Given a key, return an approximate byte offset in the file where 53 | // the data for that key begins (or would begin if the key were 54 | // present in the file). The returned value is in terms of file 55 | // bytes, and so includes effects like compression of the underlying data. 56 | // E.g., the approximate offset of the last key in the table will 57 | // be close to the file length. 58 | uint64_t ApproximateOffsetOf(const Slice& key) const; 59 | 60 | Timer* static_timers_[NUM_SEEK_THREADS]; 61 | 62 | void SetStaticTimers(Timer* static_timers[]) { 63 | for (int i = 0; i < NUM_SEEK_THREADS; i++) { 64 | static_timers_[i] = static_timers[i]; 65 | } 66 | } 67 | 68 | private: 69 | struct Rep; 70 | Rep* rep_; 71 | 72 | explicit Table(Rep* rep) : rep_(rep) { } 73 | static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); 74 | 75 | // Calls (*handle_result)(arg, ...) with the entry found after a call 76 | // to Seek(key). May not make such a call if filter policy says 77 | // that key is not present. 78 | friend class TableCache; 79 | Status InternalGet( 80 | const ReadOptions&, const Slice& key, 81 | void* arg, 82 | void (*handle_result)(void* arg, const Slice& k, const Slice& v), 83 | Timer* timer); 84 | 85 | 86 | void ReadMeta(const Footer& footer); 87 | void ReadFilter(const Slice& filter_handle_value); 88 | 89 | // No copying allowed 90 | Table(const Table&); 91 | void operator=(const Table&); 92 | }; 93 | 94 | } // namespace leveldb 95 | 96 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_ 97 | -------------------------------------------------------------------------------- /src/include/pebblesdb/table_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // TableBuilder provides the interface used to build a Table 6 | // (an immutable and sorted map from keys to values). 7 | // 8 | // Multiple threads can invoke const methods on a TableBuilder without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same TableBuilder must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 15 | 16 | #include 17 | #include "pebblesdb/options.h" 18 | #include "pebblesdb/status.h" 19 | 20 | namespace leveldb { 21 | 22 | class BlockBuilder; 23 | class BlockHandle; 24 | class WritableFile; 25 | 26 | class TableBuilder { 27 | public: 28 | // Create a builder that will store the contents of the table it is 29 | // building in *file. Does not close the file. It is up to the 30 | // caller to close the file after calling Finish(). 31 | TableBuilder(const Options& options, WritableFile* file); 32 | 33 | // REQUIRES: Either Finish() or Abandon() has been called. 34 | ~TableBuilder(); 35 | 36 | // Change the options used by this builder. Note: only some of the 37 | // option fields can be changed after construction. If a field is 38 | // not allowed to change dynamically and its value in the structure 39 | // passed to the constructor is different from its value in the 40 | // structure passed to this method, this method will return an error 41 | // without changing any fields. 42 | Status ChangeOptions(const Options& options); 43 | 44 | // Add key,value to the table being constructed. 45 | // REQUIRES: key is after any previously added key according to comparator. 46 | // REQUIRES: Finish(), Abandon() have not been called 47 | void Add(const Slice& key, const Slice& value); 48 | 49 | // Advanced operation: flush any buffered key/value pairs to file. 50 | // Can be used to ensure that two adjacent entries never live in 51 | // the same data block. Most clients should not need to use this method. 52 | // REQUIRES: Finish(), Abandon() have not been called 53 | void Flush(); 54 | 55 | // Return non-ok iff some error has been detected. 56 | Status status() const; 57 | 58 | // Finish building the table. Stops using the file passed to the 59 | // constructor after this function returns. 60 | // REQUIRES: Finish(), Abandon() have not been called 61 | Status Finish(); 62 | 63 | // Indicate that the contents of this builder should be abandoned. Stops 64 | // using the file passed to the constructor after this function returns. 65 | // If the caller is not going to call Finish(), it must call Abandon() 66 | // before destroying this builder. 67 | // REQUIRES: Finish(), Abandon() have not been called 68 | void Abandon(); 69 | 70 | // Number of calls to Add() so far. 71 | uint64_t NumEntries() const; 72 | 73 | // Size of the file generated so far. If invoked after a successful 74 | // Finish() call, returns the size of the final generated file. 75 | uint64_t FileSize() const; 76 | 77 | private: 78 | bool ok() const { return status().ok(); } 79 | void WriteBlock(BlockBuilder* block, BlockHandle* handle); 80 | void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle); 81 | 82 | struct Rep; 83 | Rep* rep_; 84 | 85 | // No copying allowed 86 | TableBuilder(const TableBuilder&); 87 | void operator=(const TableBuilder&); 88 | }; 89 | 90 | } // namespace leveldb 91 | 92 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 93 | -------------------------------------------------------------------------------- /src/include/pebblesdb/write_batch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch holds a collection of updates to apply atomically to a DB. 6 | // 7 | // The updates are applied in the order in which they are added 8 | // to the WriteBatch. For example, the value of "key" will be "v3" 9 | // after the following batch is written: 10 | // 11 | // batch.Put("key", "v1"); 12 | // batch.Delete("key"); 13 | // batch.Put("key", "v2"); 14 | // batch.Put("key", "v3"); 15 | // 16 | // Multiple threads can invoke const methods on a WriteBatch without 17 | // external synchronization, but if any of the threads may call a 18 | // non-const method, all threads accessing the same WriteBatch must use 19 | // external synchronization. 20 | 21 | #ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 22 | #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 23 | 24 | #include 25 | #include "pebblesdb/status.h" 26 | 27 | namespace leveldb { 28 | 29 | class Slice; 30 | 31 | class WriteBatch { 32 | public: 33 | WriteBatch(); 34 | ~WriteBatch(); 35 | 36 | // Store the mapping "key->value" in the database. 37 | void Put(const Slice& key, const Slice& value); 38 | 39 | // If the database contains a mapping for "key", erase it. Else do nothing. 40 | void Delete(const Slice& key); 41 | 42 | // Store a Guard in the WriteBatch 43 | void PutGuard(const Slice& key, int level); 44 | 45 | // Clear all updates buffered in this batch. 46 | void Clear(); 47 | 48 | int Count() const; 49 | 50 | // Support for iterating over the contents of a batch. 51 | class Handler { 52 | public: 53 | virtual ~Handler(); 54 | virtual void Put(const Slice& key, const Slice& value) = 0; 55 | virtual void Delete(const Slice& key) = 0; 56 | virtual void HandleGuard(const Slice& key, unsigned level) = 0; 57 | }; 58 | Status Iterate(Handler* handler) const; 59 | 60 | private: 61 | friend class WriteBatchInternal; 62 | 63 | std::string rep_; // See comment in write_batch.cc for the format of rep_ 64 | 65 | // Intentionally copyable 66 | }; 67 | 68 | } // namespace leveldb 69 | 70 | #endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 71 | -------------------------------------------------------------------------------- /src/issues/issue178_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 178: a manual compaction causes deleted data to reappear. 6 | #include 7 | #include 8 | #include 9 | 10 | #include "pebblesdb/db.h" 11 | #include "pebblesdb/write_batch.h" 12 | #include "util/testharness.h" 13 | 14 | namespace { 15 | 16 | const int kNumKeys = 1100000; 17 | 18 | std::string Key1(int i) { 19 | char buf[100]; 20 | snprintf(buf, sizeof(buf), "my_key_%d", i); 21 | return buf; 22 | } 23 | 24 | std::string Key2(int i) { 25 | return Key1(i) + "_xxx"; 26 | } 27 | 28 | class Issue178 { }; 29 | 30 | TEST(Issue178, Test) { 31 | // Get rid of any state from an old run. 32 | std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; 33 | DestroyDB(dbpath, leveldb::Options()); 34 | 35 | // Open database. Disable compression since it affects the creation 36 | // of layers and the code below is trying to test against a very 37 | // specific scenario. 38 | leveldb::DB* db; 39 | leveldb::Options db_options; 40 | db_options.create_if_missing = true; 41 | db_options.compression = leveldb::kNoCompression; 42 | ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); 43 | 44 | // create first key range 45 | leveldb::WriteBatch batch; 46 | for (size_t i = 0; i < kNumKeys; i++) { 47 | batch.Put(Key1(i), "value for range 1 key"); 48 | } 49 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 50 | 51 | // create second key range 52 | batch.Clear(); 53 | for (size_t i = 0; i < kNumKeys; i++) { 54 | batch.Put(Key2(i), "value for range 2 key"); 55 | } 56 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 57 | 58 | // delete second key range 59 | batch.Clear(); 60 | for (size_t i = 0; i < kNumKeys; i++) { 61 | batch.Delete(Key2(i)); 62 | } 63 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 64 | 65 | // compact database 66 | std::string start_key = Key1(0); 67 | std::string end_key = Key1(kNumKeys - 1); 68 | leveldb::Slice least(start_key.data(), start_key.size()); 69 | leveldb::Slice greatest(end_key.data(), end_key.size()); 70 | 71 | // commenting out the line below causes the example to work correctly 72 | db->CompactRange(&least, &greatest); 73 | 74 | // count the keys 75 | leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); 76 | size_t num_keys = 0; 77 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 78 | num_keys++; 79 | } 80 | delete iter; 81 | ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; 82 | 83 | // close database 84 | delete db; 85 | DestroyDB(dbpath, leveldb::Options()); 86 | } 87 | 88 | } // anonymous namespace 89 | 90 | int main(int argc, char** argv) { 91 | return leveldb::test::RunAllTests(); 92 | } 93 | -------------------------------------------------------------------------------- /src/issues/issue200_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 200: when iterator switches direction from backward 6 | // to forward, the current key can be yielded unexpectedly if a new 7 | // mutation has been added just before the current key. 8 | 9 | #include "pebblesdb/db.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class Issue200 { }; 15 | 16 | TEST(Issue200, Test) { 17 | // Get rid of any state from an old run. 18 | std::string dbpath = test::TmpDir() + "/leveldb_issue200_test"; 19 | DestroyDB(dbpath, Options()); 20 | 21 | DB *db; 22 | Options options; 23 | options.create_if_missing = true; 24 | ASSERT_OK(DB::Open(options, dbpath, &db)); 25 | 26 | WriteOptions write_options; 27 | ASSERT_OK(db->Put(write_options, "1", "b")); 28 | ASSERT_OK(db->Put(write_options, "2", "c")); 29 | ASSERT_OK(db->Put(write_options, "3", "d")); 30 | ASSERT_OK(db->Put(write_options, "4", "e")); 31 | ASSERT_OK(db->Put(write_options, "5", "f")); 32 | 33 | ReadOptions read_options; 34 | Iterator *iter = db->NewIterator(read_options); 35 | 36 | // Add an element that should not be reflected in the iterator. 37 | ASSERT_OK(db->Put(write_options, "25", "cd")); 38 | 39 | iter->Seek("5"); 40 | ASSERT_EQ(iter->key().ToString(), "5"); 41 | iter->Prev(); 42 | ASSERT_EQ(iter->key().ToString(), "4"); 43 | iter->Prev(); 44 | ASSERT_EQ(iter->key().ToString(), "3"); 45 | iter->Next(); 46 | ASSERT_EQ(iter->key().ToString(), "4"); 47 | iter->Next(); 48 | ASSERT_EQ(iter->key().ToString(), "5"); 49 | 50 | delete iter; 51 | delete db; 52 | DestroyDB(dbpath, options); 53 | } 54 | 55 | } // namespace leveldb 56 | 57 | int main(int argc, char** argv) { 58 | return leveldb::test::RunAllTests(); 59 | } 60 | -------------------------------------------------------------------------------- /src/libpebblesdb.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@prefix@ 2 | exec_prefix=@exec_prefix@ 3 | libdir=@libdir@ 4 | includedir=@includedir@ 5 | 6 | Name: PebblesDB 7 | Description: A fast and lightweight key-value storage library 8 | Version: @VERSION@ 9 | 10 | Requires: 11 | Libs: -L${libdir} -lpebblesdb 12 | Cflags: -I${includedir} 13 | -------------------------------------------------------------------------------- /src/m4/ax_check_compile_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check whether the given FLAG works with the current language's compiler 12 | # or gives an error. (Warnings, however, are ignored) 13 | # 14 | # ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on 15 | # success/failure. 16 | # 17 | # If EXTRA-FLAGS is defined, it is added to the current language's default 18 | # flags (e.g. CFLAGS) when the check is done. The check is thus made with 19 | # the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to 20 | # force the compiler to issue an error when a bad flag is given. 21 | # 22 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this 23 | # macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. 24 | # 25 | # LICENSE 26 | # 27 | # Copyright (c) 2008 Guido U. Draheim 28 | # Copyright (c) 2011 Maarten Bosmans 29 | # 30 | # This program is free software: you can redistribute it and/or modify it 31 | # under the terms of the GNU General Public License as published by the 32 | # Free Software Foundation, either version 3 of the License, or (at your 33 | # option) any later version. 34 | # 35 | # This program is distributed in the hope that it will be useful, but 36 | # WITHOUT ANY WARRANTY; without even the implied warranty of 37 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 38 | # Public License for more details. 39 | # 40 | # You should have received a copy of the GNU General Public License along 41 | # with this program. If not, see . 42 | # 43 | # As a special exception, the respective Autoconf Macro's copyright owner 44 | # gives unlimited permission to copy, distribute and modify the configure 45 | # scripts that are the output of Autoconf when processing the Macro. You 46 | # need not follow the terms of the GNU General Public License when using 47 | # or distributing such scripts, even though portions of the text of the 48 | # Macro appear in them. The GNU General Public License (GPL) does govern 49 | # all other use of the material that constitutes the Autoconf Macro. 50 | # 51 | # This special exception to the GPL applies to versions of the Autoconf 52 | # Macro released by the Autoconf Archive. When you make and distribute a 53 | # modified version of the Autoconf Macro, you may extend this special 54 | # exception to the GPL to apply to your modified version as well. 55 | 56 | #serial 2 57 | 58 | AC_DEFUN([AX_CHECK_COMPILE_FLAG], 59 | [AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX 60 | AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl 61 | AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ 62 | ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS 63 | _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" 64 | AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], 65 | [AS_VAR_SET(CACHEVAR,[yes])], 66 | [AS_VAR_SET(CACHEVAR,[no])]) 67 | _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) 68 | AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], 69 | [m4_default([$2], :)], 70 | [m4_default([$3], :)]) 71 | AS_VAR_POPDEF([CACHEVAR])dnl 72 | ])dnl AX_CHECK_COMPILE_FLAGS 73 | -------------------------------------------------------------------------------- /src/m4/ax_check_link_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check whether the given FLAG works with the linker or gives an error. 12 | # (Warnings, however, are ignored) 13 | # 14 | # ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on 15 | # success/failure. 16 | # 17 | # If EXTRA-FLAGS is defined, it is added to the linker's default flags 18 | # when the check is done. The check is thus made with the flags: "LDFLAGS 19 | # EXTRA-FLAGS FLAG". This can for example be used to force the linker to 20 | # issue an error when a bad flag is given. 21 | # 22 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this 23 | # macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG. 24 | # 25 | # LICENSE 26 | # 27 | # Copyright (c) 2008 Guido U. Draheim 28 | # Copyright (c) 2011 Maarten Bosmans 29 | # 30 | # This program is free software: you can redistribute it and/or modify it 31 | # under the terms of the GNU General Public License as published by the 32 | # Free Software Foundation, either version 3 of the License, or (at your 33 | # option) any later version. 34 | # 35 | # This program is distributed in the hope that it will be useful, but 36 | # WITHOUT ANY WARRANTY; without even the implied warranty of 37 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 38 | # Public License for more details. 39 | # 40 | # You should have received a copy of the GNU General Public License along 41 | # with this program. If not, see . 42 | # 43 | # As a special exception, the respective Autoconf Macro's copyright owner 44 | # gives unlimited permission to copy, distribute and modify the configure 45 | # scripts that are the output of Autoconf when processing the Macro. You 46 | # need not follow the terms of the GNU General Public License when using 47 | # or distributing such scripts, even though portions of the text of the 48 | # Macro appear in them. The GNU General Public License (GPL) does govern 49 | # all other use of the material that constitutes the Autoconf Macro. 50 | # 51 | # This special exception to the GPL applies to versions of the Autoconf 52 | # Macro released by the Autoconf Archive. When you make and distribute a 53 | # modified version of the Autoconf Macro, you may extend this special 54 | # exception to the GPL to apply to your modified version as well. 55 | 56 | #serial 2 57 | 58 | AC_DEFUN([AX_CHECK_LINK_FLAG], 59 | [AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl 60 | AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [ 61 | ax_check_save_flags=$LDFLAGS 62 | LDFLAGS="$LDFLAGS $4 $1" 63 | AC_LINK_IFELSE([AC_LANG_PROGRAM()], 64 | [AS_VAR_SET(CACHEVAR,[yes])], 65 | [AS_VAR_SET(CACHEVAR,[no])]) 66 | LDFLAGS=$ax_check_save_flags]) 67 | AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], 68 | [m4_default([$2], :)], 69 | [m4_default([$3], :)]) 70 | AS_VAR_POPDEF([CACHEVAR])dnl 71 | ])dnl AX_CHECK_LINK_FLAGS 72 | -------------------------------------------------------------------------------- /src/m4/ax_check_preproc_flag.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CHECK_PREPROC_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check whether the given FLAG works with the current language's 12 | # preprocessor or gives an error. (Warnings, however, are ignored) 13 | # 14 | # ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on 15 | # success/failure. 16 | # 17 | # If EXTRA-FLAGS is defined, it is added to the preprocessor's default 18 | # flags when the check is done. The check is thus made with the flags: 19 | # "CPPFLAGS EXTRA-FLAGS FLAG". This can for example be used to force the 20 | # preprocessor to issue an error when a bad flag is given. 21 | # 22 | # NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this 23 | # macro in sync with AX_CHECK_{COMPILE,LINK}_FLAG. 24 | # 25 | # LICENSE 26 | # 27 | # Copyright (c) 2008 Guido U. Draheim 28 | # Copyright (c) 2011 Maarten Bosmans 29 | # 30 | # This program is free software: you can redistribute it and/or modify it 31 | # under the terms of the GNU General Public License as published by the 32 | # Free Software Foundation, either version 3 of the License, or (at your 33 | # option) any later version. 34 | # 35 | # This program is distributed in the hope that it will be useful, but 36 | # WITHOUT ANY WARRANTY; without even the implied warranty of 37 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 38 | # Public License for more details. 39 | # 40 | # You should have received a copy of the GNU General Public License along 41 | # with this program. If not, see . 42 | # 43 | # As a special exception, the respective Autoconf Macro's copyright owner 44 | # gives unlimited permission to copy, distribute and modify the configure 45 | # scripts that are the output of Autoconf when processing the Macro. You 46 | # need not follow the terms of the GNU General Public License when using 47 | # or distributing such scripts, even though portions of the text of the 48 | # Macro appear in them. The GNU General Public License (GPL) does govern 49 | # all other use of the material that constitutes the Autoconf Macro. 50 | # 51 | # This special exception to the GPL applies to versions of the Autoconf 52 | # Macro released by the Autoconf Archive. When you make and distribute a 53 | # modified version of the Autoconf Macro, you may extend this special 54 | # exception to the GPL to apply to your modified version as well. 55 | 56 | #serial 2 57 | 58 | AC_DEFUN([AX_CHECK_PREPROC_FLAG], 59 | [AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX 60 | AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]cppflags_$4_$1])dnl 61 | AC_CACHE_CHECK([whether _AC_LANG preprocessor accepts $1], CACHEVAR, [ 62 | ax_check_save_flags=$CPPFLAGS 63 | CPPFLAGS="$CPPFLAGS $4 $1" 64 | AC_PREPROC_IFELSE([AC_LANG_PROGRAM()], 65 | [AS_VAR_SET(CACHEVAR,[yes])], 66 | [AS_VAR_SET(CACHEVAR,[no])]) 67 | CPPFLAGS=$ax_check_save_flags]) 68 | AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], 69 | [m4_default([$2], :)], 70 | [m4_default([$3], :)]) 71 | AS_VAR_POPDEF([CACHEVAR])dnl 72 | ])dnl AX_CHECK_PREPROC_FLAGS 73 | -------------------------------------------------------------------------------- /src/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /src/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_posix.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /src/port/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_posix.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | namespace port { 14 | 15 | static void PthreadCall(const char* label, int result) { 16 | if (result != 0) { 17 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 18 | abort(); 19 | } 20 | } 21 | 22 | Mutex::Mutex() : mu_() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 23 | 24 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 25 | 26 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 27 | 28 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 29 | 30 | void CondVar::InitMutex(Mutex* mu) { 31 | mu_ = mu; 32 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 33 | } 34 | CondVar::CondVar(Mutex* mu) 35 | : cv_(), 36 | mu_(mu) { 37 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 38 | } 39 | 40 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 41 | 42 | void CondVar::Wait() { 43 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 44 | } 45 | 46 | void CondVar::Signal() { 47 | PthreadCall("signal", pthread_cond_signal(&cv_)); 48 | } 49 | 50 | void CondVar::SignalAll() { 51 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 52 | } 53 | 54 | void InitOnce(OnceType* once, void (*initializer)()) { 55 | PthreadCall("once", pthread_once(once, initializer)); 56 | } 57 | 58 | } // namespace port 59 | } // namespace leveldb 60 | -------------------------------------------------------------------------------- /src/port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H 6 | 7 | // Some environments provide custom macros to aid in static thread-safety 8 | // analysis. Provide empty definitions of such macros unless they are already 9 | // defined. 10 | 11 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 12 | #define EXCLUSIVE_LOCKS_REQUIRED(...) 13 | #endif 14 | 15 | #ifndef SHARED_LOCKS_REQUIRED 16 | #define SHARED_LOCKS_REQUIRED(...) 17 | #endif 18 | 19 | #ifndef LOCKS_EXCLUDED 20 | #define LOCKS_EXCLUDED(...) 21 | #endif 22 | 23 | #ifndef LOCK_RETURNED 24 | #define LOCK_RETURNED(x) 25 | #endif 26 | 27 | #ifndef LOCKABLE 28 | #define LOCKABLE 29 | #endif 30 | 31 | #ifndef SCOPED_LOCKABLE 32 | #define SCOPED_LOCKABLE 33 | #endif 34 | 35 | #ifndef EXCLUSIVE_LOCK_FUNCTION 36 | #define EXCLUSIVE_LOCK_FUNCTION(...) 37 | #endif 38 | 39 | #ifndef SHARED_LOCK_FUNCTION 40 | #define SHARED_LOCK_FUNCTION(...) 41 | #endif 42 | 43 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 44 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) 45 | #endif 46 | 47 | #ifndef SHARED_TRYLOCK_FUNCTION 48 | #define SHARED_TRYLOCK_FUNCTION(...) 49 | #endif 50 | 51 | #ifndef UNLOCK_FUNCTION 52 | #define UNLOCK_FUNCTION(...) 53 | #endif 54 | 55 | #ifndef NO_THREAD_SAFETY_ANALYSIS 56 | #define NO_THREAD_SAFETY_ANALYSIS 57 | #endif 58 | 59 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H 60 | -------------------------------------------------------------------------------- /src/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /src/reliability.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in `seq 1 10`; 4 | do 5 | echo ----------------------------- Iteration $i ------------------------------- 6 | export file_name='rel_inserted_keys' 7 | echo Starting FillRandom . . 8 | nohup ./db_bench --benchmarks=rel_start & 9 | pid=`pgrep bench` 10 | seconds=`shuf -i 1-20 -n 1` 11 | echo Waiting for $seconds seconds 12 | sleep $seconds 13 | 14 | echo Killing process $pid 15 | kill $pid 16 | echo Process killed. 17 | sleep 5 18 | 19 | echo -------------------------------------------------------------------------- 20 | echo Starting Reliablity Check . . 21 | export file_name='rel_inserted_keys' 22 | ./db_bench --benchmarks=rel_check --use_existing_db=1 23 | echo Reliability check complete. Please check logs for any issues. 24 | echo -------------------------------------------------------------------------- 25 | 26 | done 27 | 28 | -------------------------------------------------------------------------------- /src/scripts/script_background_compaction_time_measure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat $1 | grep 'Time taken to complete BackgroundCompaction' | awk -F 'BackgroundCompaction: ' '{print $2}' | cut -d ' ' -f1 > total_time_background_compaction 4 | cat $1 | grep 'Time taken to pick compaction level:' | awk -F 'level: ' '{print $2}' | cut -d ' ' -f1 > pick_compaction_level 5 | cat $1 | grep 'Time taken to pick compaction:' | awk -F 'compaction: ' '{print $2}' | cut -d ' ' -f1 > pick_compaction 6 | cat $1 | grep 'Time taken to add guards to edit' | awk -F 'edit: ' '{print $2}' | cut -d ' ' -f1 > add_guards 7 | cat $1 | grep 'Time taken to complete DoCompactionWorkGuards' | awk -F 'WorkGuards: ' '{print $2}' | cut -d ' ' -f1 > do_compaction_work_guards 8 | cat $1 | grep 'Time taken to cleanup compaction and delete obsolete files' | awk -F 'files: ' '{print $2}' | cut -d ' ' -f1 > cleanup_compaction 9 | 10 | cat $1 | grep 'Time taken to iterate keys and split them into files' | awk -F 'files: ' '{print $2}' | cut -d ' ' -f1 > iterate_keys_and_split 11 | cat $1 | grep 'collect stats' | awk -F 'stats: ' '{print $2}' | cut -d ' ' -f1 > collect_stats 12 | cat $1 | grep 'Time taken to get the lock' | awk -F 'lock: ' '{print $2}' | cut -d ' ' -f1 > bg_compaction_get_the_lock 13 | cat $1 | grep 'Time taken to install compaction results' | awk -F 'results: ' '{print $2}' | cut -d ' ' -f1 > install_compaction_results 14 | 15 | 16 | echo -n 'total_time_background_compaction: ' 17 | awk '{s+=$1} END {print s}' total_time_background_compaction 18 | echo 19 | 20 | echo -n 'pick_compaction_level: ' 21 | awk '{s+=$1} END {print s}' pick_compaction_level 22 | echo -n 'pick_compaction: ' 23 | awk '{s+=$1} END {print s}' pick_compaction 24 | echo -n 'add_guards: ' 25 | awk '{s+=$1} END {print s}' add_guards 26 | echo -n 'do_compaction_work_guards: ' 27 | awk '{s+=$1} END {print s}' do_compaction_work_guards 28 | echo -n 'cleanup_compaction: ' 29 | awk '{s+=$1} END {print s}' cleanup_compaction 30 | echo 31 | 32 | echo -n 'iterate_keys_and_split: ' 33 | awk '{s+=$1} END {print s}' iterate_keys_and_split 34 | echo -n 'collect_stats: ' 35 | awk '{s+=$1} END {print s}' collect_stats 36 | echo -n 'bg_compaction_get_the_lock: ' 37 | awk '{s+=$1} END {print s}' bg_compaction_get_the_lock 38 | echo -n 'install_compaction_results: ' 39 | awk '{s+=$1} END {print s}' install_compaction_results 40 | -------------------------------------------------------------------------------- /src/scripts/script_memtable_time_measure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat $1 | grep 'Total time taken for memtable compaction' | awk -F 'compaction: ' '{print $2}' | cut -d ' ' -f1 > bmlog_total_time_memtable_compaction 4 | cat $1 | grep 'Time taken for WriteLevel0Table' | awk -F 'Table: ' '{print $2}' | cut -d ' ' -f1 > bmlog_write_level0_table 5 | cat $1 | grep 'add guards to edit' | awk -F 'edit: ' '{print $2}' | cut -d ' ' -f1 > bmlog_add_guards 6 | cat $1 | grep 'add complete guards to edit' | awk -F 'edit: ' '{print $2}' | cut -d ' ' -f1 > bmlog_add_complete_guards 7 | cat $1 | grep 'taken to LogAndApply' | awk -F 'LogAndApply: ' '{print $2}' | cut -d ' ' -f1 > bmlog_log_and_apply 8 | cat $1 | grep 'erase pending outputs' | awk -F 'outputs: ' '{print $2}' | cut -d ' ' -f1 > bmlog_erase_pending_outputs 9 | cat $1 | grep 'Time taken to delete obsolete files' | awk -F 'files: ' '{print $2}' | cut -d ' ' -f1 > bmlog_delete_obsolete_files 10 | 11 | #write level 0 tables 12 | cat $1 | grep 'BuildLevel0Table' | cut -d ' ' -f5 > bmlog_build_level0_table 13 | cat $1 | grep 'get lock after building' | cut -d ' ' -f9 > bmlog_get_lock_after_building 14 | cat $1 | grep 'add level0 files' | cut -d ' ' -f9 > bmlog_add_level0_files_to_edit 15 | 16 | #LogAndApply 17 | cat $1 | grep 'Time taken to save builder' | grep 'mtc' | cut -d ' ' -f11 > bmlog_save_builder 18 | cat $1 | grep 'Time taken to finalize new version' | grep 'mtc' | cut -d ' ' -f9 > bmlog_finalize_version 19 | cat $1 | grep 'Time taken to encode edit' | grep 'mtc' | cut -d ' ' -f8 > bmlog_encode_edit 20 | cat $1 | grep 'Time taken to add record to descriptor log' | grep 'mtc' | cut -d ' ' -f11 > bmlog_add_record_desc_log 21 | cat $1 | grep 'Time taken to sync manifest log write' | grep 'mtc' | cut -d ' ' -f10 > bmlog_sync_manifest_log 22 | cat $1 | grep 'Time taken to append new version' | grep 'mtc' | cut -d ' ' -f9 > bmlog_append_new_version 23 | cat $1 | grep 'Time taken to apply edit to builder'| grep 'mtc' | cut -d ' ' -f10 > bmlog_apply_edit 24 | 25 | echo Overall 26 | echo -n 'total_time_memtable_compaction: ' 27 | awk '{s+=$1} END {print s}' bmlog_total_time_memtable_compaction 28 | echo 29 | 30 | echo First level split 31 | echo -n 'write_level0_table: ' 32 | awk '{s+=$1} END {print s}' bmlog_write_level0_table 33 | echo -n 'add_guards: ' 34 | awk '{s+=$1} END {print s}' bmlog_add_guards 35 | echo -n 'add_complete_guards: ' 36 | awk '{s+=$1} END {print s}' bmlog_add_complete_guards 37 | echo -n 'log_and_apply: ' 38 | awk '{s+=$1} END {print s}' bmlog_log_and_apply 39 | echo -n 'erase_pending_outputs: ' 40 | awk '{s+=$1} END {print s}' bmlog_erase_pending_outputs 41 | echo -n 'delete_obsolete_files: ' 42 | awk '{s+=$1} END {print s}' bmlog_delete_obsolete_files 43 | 44 | echo 45 | echo WriteLevel0Tables 46 | echo -n 'build level0 table: ' 47 | awk '{s+=$1} END {print s}' bmlog_build_level0_table 48 | echo -n 'Get lock after building: ' 49 | awk '{s+=$1} END {print s}' bmlog_get_lock_after_building 50 | echo -n 'add level0 files to edit: ' 51 | awk '{s+=$1} END {print s}' bmlog_add_level0_files_to_edit 52 | 53 | echo 54 | echo LogAndApply 55 | echo -n 'apply edit to version: ' 56 | awk '{s+=$1} END {print s}' bmlog_apply_edit 57 | echo -n 'save builder: ' 58 | awk '{s+=$1} END {print s}' bmlog_save_builder 59 | echo -n 'finalize version: ' 60 | awk '{s+=$1} END {print s}' bmlog_finalize_version 61 | echo -n 'encode edit: ' 62 | awk '{s+=$1} END {print s}' bmlog_encode_edit 63 | echo -n 'add record to desc log: ' 64 | awk '{s+=$1} END {print s}' bmlog_add_record_desc_log 65 | echo -n 'sync manifest log: ' 66 | awk '{s+=$1} END {print s}' bmlog_sync_manifest_log 67 | echo -n 'append new version: ' 68 | awk '{s+=$1} END {print s}' bmlog_append_new_version 69 | 70 | echo 71 | echo -n 'Number of memtable compactions: ' 72 | cat $1 | grep 'Total time taken for memtable compaction' | wc -l 73 | -------------------------------------------------------------------------------- /src/scripts/script_readrandom_time_measure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat $1 | grep 'get from version' | awk -F 'version: ' '{print $2}' | cut -d ' ' -f1 > get_from_version 4 | cat $1 | grep 'ReadRandom' | awk -F 'key: ' '{print $2}' | cut -d ' ' -f1 > read_random_get 5 | cat $1 | grep 'to get mutex' | awk -F 'mutex: ' '{print $2}' | cut -d ' ' -f1 > get_mutex 6 | cat $1 | grep 'to ref mems' | awk -F 'mems: ' '{print $2}' | cut -d ' ' -f1 > ref_mems 7 | cat $1 | grep 'memtable and imm' | awk -F 'imm: ' '{print $2}' | cut -d ' ' -f1 > mem_imm 8 | cat $1 | grep 'lock mutex' | awk -F 'mutex: ' '{print $2}' | cut -d ' ' -f1 > lock_mutex 9 | cat $1 | grep 'finishing unref' | awk -F 'unref: ' '{print $2}' | cut -d ' ' -f1 > unref 10 | cat $1 | grep 'table_cache' | awk -F 'call: ' '{print $2}' | cut -d ' ' -f1 > get_from_table_cache 11 | cat $1 | grep 'list of files to search' | awk -F 'search: ' '{print $2}' | cut -d ' ' -f1 > list_of_files 12 | cat $1 | grep 'clear tmp2 and set g' | awk -F 'set g: ' '{print $2}' | cut -d ' ' -f1 > clear_tmp_set_g 13 | cat $1 | grep 'Get to finish of the search' | awk -F 'search: ' '{print $2}' | cut -d ' ' -f1 > get_from_version_start_to_return 14 | cat $1 | grep 'find the guard' | awk -F 'guard: ' '{print $2}' | cut -d ' ' -f1 > find_guard 15 | cat $1 | grep 'check sentinel files' | cut -d ' ' -f9 > check_sentinel_files 16 | cat $1 | grep 'check guard files' | cut -d ' ' -f9 > check_guard_files 17 | cat $1 | grep 'sort sentinel files' | cut -d ' ' -f9 > sort_sentinel_files 18 | cat $1 | grep 'sort guard files' | cut -d ' ' -f9 > sort_guard_files 19 | cat $1 | grep 'set file_meta_data_map' | cut -d ' ' -f8 > set_file_meta_data_map 20 | 21 | 22 | 23 | echo -n 'read_random_get: ' 24 | awk '{s+=$1} END {print s}' read_random_get 25 | echo 26 | 27 | echo -n 'get_from_version: ' 28 | awk '{s+=$1} END {print s}' get_from_version 29 | echo -n 'get_mutex: ' 30 | awk '{s+=$1} END {print s}' get_mutex 31 | echo -n 'ref_mems: ' 32 | awk '{s+=$1} END {print s}' ref_mems 33 | echo -n 'mem_imm: ' 34 | awk '{s+=$1} END {print s}' mem_imm 35 | echo -n 'lock_mutex: ' 36 | awk '{s+=$1} END {print s}' lock_mutex 37 | echo -n 'unref: ' 38 | awk '{s+=$1} END {print s}' unref 39 | echo 40 | 41 | echo -n 'clear_tmp_set_g: ' 42 | awk '{s+=$1} END {print s}' clear_tmp_set_g 43 | echo -n 'find_guard: ' 44 | awk '{s+=$1} END {print s}' find_guard 45 | echo -n 'find list_of_files: ' 46 | awk '{s+=$1} END {print s}' list_of_files 47 | echo -n 'table_cache_get: ' 48 | awk '{s+=$1} END {print s}' get_from_table_cache 49 | echo 50 | 51 | echo -n 'check_sentinel_files: ' 52 | awk '{s+=$1} END {print s}' check_sentinel_files 53 | echo -n 'sort_sentinel_files: ' 54 | awk '{s+=$1} END {print s}' sort_sentinel_files 55 | echo -n 'check_guard_files: ' 56 | awk '{s+=$1} END {print s}' check_guard_files 57 | echo -n 'sort_guard_files: ' 58 | awk '{s+=$1} END {print s}' sort_guard_files 59 | echo -n 'set file_meta_data_map: ' 60 | awk '{s+=$1} END {print s}' set_file_meta_data_map 61 | echo 62 | 63 | echo -n 'get_from_version_start_to_return: ' 64 | awk '{s+=$1} END {print s}' get_from_version_start_to_return 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/scripts/script_write_time_measure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat $1 | grep 'Total time' | cut -d ' ' -f6 > bmlog_total_write_time 4 | 5 | #cat $1 | grep 'Iteration' | cut -d ' ' -f12 > bmlog_sum_write_times 6 | 7 | cat $1 | grep 'Time taken by SequenceWriteBegin' | cut -d ' ' -f5 > bmlog_seq_write_begin 8 | cat $1 | grep 'Time taken by SequenceWriteEnd' | cut -d ' ' -f5 > bmlog_seq_write_end 9 | cat $1 | grep 'Time taken by log.AddRecord' | cut -d ' ' -f5 > bmlog_log_addrecord 10 | cat $1 | grep 'Time taken by InsertIntoVersion' | cut -d ' ' -f5 > bmlog_insert_into_version 11 | cat $1 | grep 'Time taken to setsequence' | cut -d ' ' -f9 > bmlog_setsequence 12 | cat $1 | grep 'Time taken by SetGuards' | cut -d ' ' -f5 > bmlog_setguards 13 | 14 | #cat $1 | grep 'erase pending outputs' | awk -F 'outputs: ' '{print $2}' | cut -d ' ' -f1 > erase_pending_outputs 15 | #cat $1 | grep 'Time taken to delete obsolete files' | awk -F 'files: ' '{print $2}' | cut -d ' ' -f1 > delete_obsolete_files 16 | 17 | 18 | echo -n 'total_time_write: ' 19 | awk '{s+=$1} END {print s}' bmlog_total_write_time 20 | echo 21 | 22 | #echo -n 'sum_of_individual_write_times: ' 23 | #awk '{s+=$1} END {print s}' bmlog_sum_write_times 24 | #echo 25 | 26 | echo -n 'Time taken by SequenceWriteBegin: ' 27 | awk '{s+=$1} END {print s}' bmlog_seq_write_begin 28 | echo -n 'Time taken by SequenceWriteEnd: ' 29 | awk '{s+=$1} END {print s}' bmlog_seq_write_end 30 | echo -n 'Time taken by SetSequence and create new batch: ' 31 | awk '{s+=$1} END {print s}' bmlog_setsequence 32 | echo -n 'Time taken by Set Guards: ' 33 | awk '{s+=$1} END {print s}' bmlog_setguards 34 | echo -n 'Time taken by log.AddRecord: ' 35 | awk '{s+=$1} END {print s}' bmlog_log_addrecord 36 | echo -n 'Time taken by InsertIntoVersion: ' 37 | awk '{s+=$1} END {print s}' bmlog_insert_into_version 38 | -------------------------------------------------------------------------------- /src/table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 7 | 8 | #include 9 | #include 10 | #include "pebblesdb/iterator.h" 11 | 12 | namespace leveldb { 13 | 14 | struct BlockContents; 15 | class Comparator; 16 | 17 | class Block { 18 | public: 19 | // Initialize the block with the specified contents. 20 | explicit Block(const BlockContents& contents); 21 | 22 | ~Block(); 23 | 24 | size_t size() const { return size_; } 25 | Iterator* NewIterator(const Comparator* comparator); 26 | 27 | private: 28 | uint32_t NumRestarts() const; 29 | 30 | const char* data_; 31 | size_t size_; 32 | uint32_t restart_offset_; // Offset in data_ of restart array 33 | bool owned_; // Block owns data_[] 34 | 35 | // No copying allowed 36 | Block(const Block&); 37 | void operator=(const Block&); 38 | 39 | class Iter; 40 | }; 41 | 42 | } // namespace leveldb 43 | 44 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 45 | -------------------------------------------------------------------------------- /src/table/block_builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // BlockBuilder generates blocks where keys are prefix-compressed: 6 | // 7 | // When we store a key, we drop the prefix shared with the previous 8 | // string. This helps reduce the space requirement significantly. 9 | // Furthermore, once every K keys, we do not apply the prefix 10 | // compression and store the entire key. We call this a "restart 11 | // point". The tail end of the block stores the offsets of all of the 12 | // restart points, and can be used to do a binary search when looking 13 | // for a particular key. Values are stored as-is (without compression) 14 | // immediately following the corresponding key. 15 | // 16 | // An entry for a particular key-value pair has the form: 17 | // shared_bytes: varint32 18 | // unshared_bytes: varint32 19 | // value_length: varint32 20 | // key_delta: char[unshared_bytes] 21 | // value: char[value_length] 22 | // shared_bytes == 0 for restart points. 23 | // 24 | // The trailer of the block has the form: 25 | // restarts: uint32[num_restarts] 26 | // num_restarts: uint32 27 | // restarts[i] contains the offset within the block of the ith restart point. 28 | 29 | #include "table/block_builder.h" 30 | 31 | #include 32 | #include 33 | #include "pebblesdb/comparator.h" 34 | #include "pebblesdb/table_builder.h" 35 | #include "util/coding.h" 36 | 37 | namespace leveldb { 38 | 39 | BlockBuilder::BlockBuilder(const Options* options) 40 | : options_(options), 41 | buffer_(), 42 | restarts_(), 43 | counter_(0), 44 | finished_(false), 45 | last_key_() { 46 | assert(options->block_restart_interval >= 1); 47 | restarts_.push_back(0); // First restart point is at offset 0 48 | } 49 | 50 | void BlockBuilder::Reset() { 51 | buffer_.clear(); 52 | restarts_.clear(); 53 | restarts_.push_back(0); // First restart point is at offset 0 54 | counter_ = 0; 55 | finished_ = false; 56 | last_key_.clear(); 57 | } 58 | 59 | size_t BlockBuilder::CurrentSizeEstimate() const { 60 | return (buffer_.size() + // Raw data buffer 61 | restarts_.size() * sizeof(uint32_t) + // Restart array 62 | sizeof(uint32_t)); // Restart array length 63 | } 64 | 65 | Slice BlockBuilder::Finish() { 66 | // Append restart array 67 | for (size_t i = 0; i < restarts_.size(); i++) { 68 | PutFixed32(&buffer_, restarts_[i]); 69 | } 70 | PutFixed32(&buffer_, restarts_.size()); 71 | finished_ = true; 72 | return buffer_.slice(); 73 | } 74 | 75 | void BlockBuilder::Add(const Slice& key, const Slice& value) { 76 | Slice last_key_piece = last_key_.slice(); 77 | assert(!finished_); 78 | assert(counter_ <= options_->block_restart_interval); 79 | #ifdef STRICT_ASSERT 80 | assert(buffer_.empty() // No values yet? 81 | || options_->comparator->Compare(key, last_key_piece) > 0); 82 | #endif 83 | size_t shared = 0; 84 | if (counter_ < options_->block_restart_interval) { 85 | // See how much sharing to do with previous string 86 | const size_t min_length = std::min(last_key_piece.size(), key.size()); 87 | const char* A = last_key_piece.data(); 88 | const char* B = key.data(); 89 | while ((shared < min_length) && (A[shared] == B[shared])) { 90 | ++shared; 91 | } 92 | } else { 93 | // Restart compression 94 | restarts_.push_back(buffer_.size()); 95 | counter_ = 0; 96 | } 97 | const size_t non_shared = key.size() - shared; 98 | 99 | // Add "" to buffer_ 100 | PutVarint32(&buffer_, shared); 101 | PutVarint32(&buffer_, non_shared); 102 | PutVarint32(&buffer_, value.size()); 103 | 104 | // Add string delta to buffer_ followed by value 105 | buffer_.append(key.data() + shared, non_shared); 106 | buffer_.append(value.data(), value.size()); 107 | 108 | // Update state 109 | last_key_.shrink(shared); 110 | last_key_.append(key.data() + shared, non_shared); 111 | assert(last_key_.slice() == key); 112 | counter_++; 113 | } 114 | 115 | } // namespace leveldb 116 | -------------------------------------------------------------------------------- /src/table/block_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 7 | 8 | #include 9 | 10 | #include 11 | #include "pebblesdb/slice.h" 12 | #include "util/string_builder.h" 13 | 14 | namespace leveldb { 15 | 16 | struct Options; 17 | 18 | class BlockBuilder { 19 | public: 20 | explicit BlockBuilder(const Options* options); 21 | 22 | // Reset the contents as if the BlockBuilder was just constructed. 23 | void Reset(); 24 | 25 | // REQUIRES: Finish() has not been callled since the last call to Reset(). 26 | // REQUIRES: key is larger than any previously added key 27 | void Add(const Slice& key, const Slice& value); 28 | 29 | // Finish building the block and return a slice that refers to the 30 | // block contents. The returned slice will remain valid for the 31 | // lifetime of this builder or until Reset() is called. 32 | Slice Finish(); 33 | 34 | // Returns an estimate of the current (uncompressed) size of the block 35 | // we are building. 36 | size_t CurrentSizeEstimate() const; 37 | 38 | // Return true iff no entries have been added since the last Reset() 39 | bool empty() const { 40 | return buffer_.empty(); 41 | } 42 | 43 | private: 44 | const Options* options_; 45 | StringBuilder buffer_; // Destination buffer 46 | std::vector restarts_; // Restart points 47 | int counter_; // Number of entries emitted since restart 48 | bool finished_; // Has Finish() been called? 49 | StringBuilder last_key_; 50 | 51 | // No copying allowed 52 | BlockBuilder(const BlockBuilder&); 53 | void operator=(const BlockBuilder&); 54 | }; 55 | 56 | } // namespace leveldb 57 | 58 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 59 | -------------------------------------------------------------------------------- /src/table/filter_block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A filter block is stored near the end of a Table file. It contains 6 | // filters (e.g., bloom filters) for all data blocks in the table combined 7 | // into a single filter block. 8 | 9 | #ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 10 | #define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "pebblesdb/slice.h" 17 | #include "util/hash.h" 18 | #include "util/string_builder.h" 19 | 20 | namespace leveldb { 21 | 22 | class FilterPolicy; 23 | 24 | class FileLevelFilterBuilder { 25 | public: 26 | explicit FileLevelFilterBuilder(const FilterPolicy*); 27 | ~FileLevelFilterBuilder(); 28 | 29 | void AddKey(const Slice& key); 30 | void Clear(); 31 | void Destroy(); 32 | std::string* GenerateFilter(); 33 | 34 | private: 35 | 36 | const FilterPolicy* policy_; 37 | StringBuilder keys_; // Flattened key contents 38 | std::vector key_offsets_; // Starting index in keys_ of each key 39 | std::vector tmp_keys_; // policy_->CreateFilter() argument 40 | 41 | // No copying allowed 42 | FileLevelFilterBuilder(const FileLevelFilterBuilder&); 43 | void operator=(const FileLevelFilterBuilder&); 44 | }; 45 | 46 | // A FilterBlockBuilder is used to construct all of the filters for a 47 | // particular Table. It generates a single string which is stored as 48 | // a special block in the Table. 49 | // 50 | // The sequence of calls to FilterBlockBuilder must match the regexp: 51 | // (StartBlock AddKey*)* Finish 52 | class FilterBlockBuilder { 53 | public: 54 | explicit FilterBlockBuilder(const FilterPolicy*); 55 | 56 | void StartBlock(uint64_t block_offset); 57 | void AddKey(const Slice& key); 58 | Slice Finish(); 59 | 60 | private: 61 | void GenerateFilter(); 62 | 63 | const FilterPolicy* policy_; 64 | StringBuilder keys_; // Flattened key contents 65 | std::vector start_; // Starting index in keys_ of each key 66 | std::string result_; // Filter data computed so far 67 | std::vector tmp_keys_; // policy_->CreateFilter() argument 68 | std::vector filter_offsets_; 69 | 70 | // No copying allowed 71 | FilterBlockBuilder(const FilterBlockBuilder&); 72 | void operator=(const FilterBlockBuilder&); 73 | }; 74 | 75 | class FilterBlockReader { 76 | public: 77 | // REQUIRES: "contents" and *policy must stay live while *this is live. 78 | FilterBlockReader(const FilterPolicy* policy, const Slice& contents); 79 | bool KeyMayMatch(uint64_t block_offset, const Slice& key); 80 | 81 | private: 82 | const FilterPolicy* policy_; 83 | const char* data_; // Pointer to filter data (at block-start) 84 | const char* offset_; // Pointer to beginning of offset array (at block-end) 85 | size_t num_; // Number of entries in offset array 86 | size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) 87 | }; 88 | 89 | } 90 | 91 | #endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 92 | -------------------------------------------------------------------------------- /src/table/filter_block_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/filter_block.h" 6 | 7 | #include "pebblesdb/filter_policy.h" 8 | #include "util/coding.h" 9 | #include "util/hash.h" 10 | #include "util/logging.h" 11 | #include "util/testharness.h" 12 | #include "util/testutil.h" 13 | 14 | namespace leveldb { 15 | 16 | // For testing: emit an array with one hash value per key 17 | class TestHashFilter : public FilterPolicy { 18 | public: 19 | virtual const char* Name() const { 20 | return "TestHashFilter"; 21 | } 22 | 23 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { 24 | for (int i = 0; i < n; i++) { 25 | uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); 26 | PutFixed32(dst, h); 27 | } 28 | } 29 | 30 | virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { 31 | uint32_t h = Hash(key.data(), key.size(), 1); 32 | for (size_t i = 0; i + 4 <= filter.size(); i += 4) { 33 | if (h == DecodeFixed32(filter.data() + i)) { 34 | return true; 35 | } 36 | } 37 | return false; 38 | } 39 | }; 40 | 41 | class FilterBlockTest { 42 | public: 43 | TestHashFilter policy_; 44 | }; 45 | 46 | TEST(FilterBlockTest, EmptyBuilder) { 47 | FilterBlockBuilder builder(&policy_); 48 | Slice block = builder.Finish(); 49 | ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); 50 | FilterBlockReader reader(&policy_, block); 51 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 52 | ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); 53 | } 54 | 55 | TEST(FilterBlockTest, SingleChunk) { 56 | FilterBlockBuilder builder(&policy_); 57 | builder.StartBlock(100); 58 | builder.AddKey("foo"); 59 | builder.AddKey("bar"); 60 | builder.AddKey("box"); 61 | builder.StartBlock(200); 62 | builder.AddKey("box"); 63 | builder.StartBlock(300); 64 | builder.AddKey("hello"); 65 | Slice block = builder.Finish(); 66 | FilterBlockReader reader(&policy_, block); 67 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 68 | ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); 69 | ASSERT_TRUE(reader.KeyMayMatch(100, "box")); 70 | ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); 71 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 72 | ASSERT_TRUE(! reader.KeyMayMatch(100, "missing")); 73 | ASSERT_TRUE(! reader.KeyMayMatch(100, "other")); 74 | } 75 | 76 | TEST(FilterBlockTest, MultiChunk) { 77 | FilterBlockBuilder builder(&policy_); 78 | 79 | // First filter 80 | builder.StartBlock(0); 81 | builder.AddKey("foo"); 82 | builder.StartBlock(2000); 83 | builder.AddKey("bar"); 84 | 85 | // Second filter 86 | builder.StartBlock(3100); 87 | builder.AddKey("box"); 88 | 89 | // Third filter is empty 90 | 91 | // Last filter 92 | builder.StartBlock(9000); 93 | builder.AddKey("box"); 94 | builder.AddKey("hello"); 95 | 96 | Slice block = builder.Finish(); 97 | FilterBlockReader reader(&policy_, block); 98 | 99 | // Check first filter 100 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 101 | ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); 102 | ASSERT_TRUE(! reader.KeyMayMatch(0, "box")); 103 | ASSERT_TRUE(! reader.KeyMayMatch(0, "hello")); 104 | 105 | // Check second filter 106 | ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); 107 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo")); 108 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar")); 109 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello")); 110 | 111 | // Check third filter (empty) 112 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo")); 113 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar")); 114 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "box")); 115 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello")); 116 | 117 | // Check last filter 118 | ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); 119 | ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); 120 | ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo")); 121 | ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar")); 122 | } 123 | 124 | } // namespace leveldb 125 | 126 | int main(int argc, char** argv) { 127 | return leveldb::test::RunAllTests(); 128 | } 129 | -------------------------------------------------------------------------------- /src/table/format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ 6 | #define STORAGE_LEVELDB_TABLE_FORMAT_H_ 7 | 8 | #include 9 | #include 10 | #include "pebblesdb/slice.h" 11 | #include "pebblesdb/status.h" 12 | #include "pebblesdb/table_builder.h" 13 | 14 | namespace leveldb { 15 | 16 | class Block; 17 | class RandomAccessFile; 18 | struct ReadOptions; 19 | 20 | // BlockHandle is a pointer to the extent of a file that stores a data 21 | // block or a meta block. 22 | class BlockHandle { 23 | public: 24 | BlockHandle(); 25 | 26 | // The offset of the block in the file. 27 | uint64_t offset() const { return offset_; } 28 | void set_offset(uint64_t _offset) { offset_ = _offset; } 29 | 30 | // The size of the stored block 31 | uint64_t size() const { return size_; } 32 | void set_size(uint64_t _size) { size_ = _size; } 33 | 34 | void EncodeTo(std::string* dst) const; 35 | Status DecodeFrom(Slice* input); 36 | 37 | // Maximum encoding length of a BlockHandle 38 | enum { kMaxEncodedLength = 10 + 10 }; 39 | 40 | private: 41 | uint64_t offset_; 42 | uint64_t size_; 43 | }; 44 | 45 | // Footer encapsulates the fixed information stored at the tail 46 | // end of every table file. 47 | class Footer { 48 | public: 49 | Footer() 50 | : metaindex_handle_(), 51 | index_handle_() { 52 | } 53 | 54 | // The block handle for the metaindex block of the table 55 | const BlockHandle& metaindex_handle() const { return metaindex_handle_; } 56 | void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } 57 | 58 | // The block handle for the index block of the table 59 | const BlockHandle& index_handle() const { 60 | return index_handle_; 61 | } 62 | void set_index_handle(const BlockHandle& h) { 63 | index_handle_ = h; 64 | } 65 | 66 | void EncodeTo(std::string* dst) const; 67 | Status DecodeFrom(Slice* input); 68 | 69 | // Encoded length of a Footer. Note that the serialization of a 70 | // Footer will always occupy exactly this many bytes. It consists 71 | // of two block handles and a magic number. 72 | enum { 73 | kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8 74 | }; 75 | 76 | private: 77 | BlockHandle metaindex_handle_; 78 | BlockHandle index_handle_; 79 | }; 80 | 81 | // kTableMagicNumber was picked by running 82 | // echo http://code.google.com/p/leveldb/ | sha1sum 83 | // and taking the leading 64 bits. 84 | static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; 85 | 86 | // 1-byte type + 32-bit crc 87 | static const size_t kBlockTrailerSize = 5; 88 | 89 | struct BlockContents { 90 | BlockContents() : data(), cachable(), heap_allocated() {} 91 | Slice data; // Actual contents of data 92 | bool cachable; // True iff data can be cached 93 | bool heap_allocated; // True iff caller should delete[] data.data() 94 | }; 95 | 96 | // Read the block identified by "handle" from "file". On failure 97 | // return non-OK. On success fill *result and return OK. 98 | extern Status ReadBlock(RandomAccessFile* file, 99 | const ReadOptions& options, 100 | const BlockHandle& handle, 101 | BlockContents* result); 102 | 103 | // Implementation details follow. Clients should ignore, 104 | 105 | inline BlockHandle::BlockHandle() 106 | : offset_(~static_cast(0)), 107 | size_(~static_cast(0)) { 108 | } 109 | 110 | } // namespace leveldb 111 | 112 | #endif // STORAGE_LEVELDB_TABLE_FORMAT_H_ 113 | -------------------------------------------------------------------------------- /src/table/iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/iterator.h" 6 | 7 | namespace leveldb { 8 | 9 | Iterator::Iterator() 10 | : cleanup_() { 11 | cleanup_.function = NULL; 12 | cleanup_.next = NULL; 13 | } 14 | 15 | Iterator::~Iterator() { 16 | if (cleanup_.function != NULL) { 17 | (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); 18 | for (Cleanup* c = cleanup_.next; c != NULL; ) { 19 | (*c->function)(c->arg1, c->arg2); 20 | Cleanup* next = c->next; 21 | delete c; 22 | c = next; 23 | } 24 | } 25 | } 26 | 27 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 28 | assert(func != NULL); 29 | Cleanup* c; 30 | if (cleanup_.function == NULL) { 31 | c = &cleanup_; 32 | } else { 33 | c = new Cleanup; 34 | c->next = cleanup_.next; 35 | cleanup_.next = c; 36 | } 37 | c->function = func; 38 | c->arg1 = arg1; 39 | c->arg2 = arg2; 40 | } 41 | 42 | namespace { 43 | class EmptyIterator : public Iterator { 44 | public: 45 | EmptyIterator(const Status& s) : status_(s) { } 46 | virtual bool Valid() const { return false; } 47 | virtual void Seek(const Slice& /*target*/) { } 48 | virtual void SeekToFirst() { } 49 | virtual void SeekToLast() { } 50 | virtual void Next() { assert(false); } 51 | virtual void Prev() { assert(false); } 52 | Slice key() const { assert(false); return Slice(); } 53 | Slice value() const { assert(false); return Slice(); } 54 | virtual const Status& status() const { return status_; } 55 | private: 56 | Status status_; 57 | }; 58 | } // namespace 59 | 60 | Iterator* NewEmptyIterator() { 61 | return new EmptyIterator(Status::OK()); 62 | } 63 | 64 | Iterator* NewErrorIterator(const Status& status) { 65 | return new EmptyIterator(status); 66 | } 67 | 68 | } // namespace leveldb 69 | -------------------------------------------------------------------------------- /src/table/iterator_wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 6 | #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | // A internal wrapper class with an interface similar to Iterator that 11 | // caches the valid() and key() results for an underlying iterator. 12 | // This can help avoid virtual function calls and also gives better 13 | // cache locality. 14 | class IteratorWrapper { 15 | public: 16 | IteratorWrapper(): iter_(NULL), valid_(false), key_() { } 17 | explicit IteratorWrapper(Iterator* it): iter_(NULL), valid_(), key_() { 18 | Set(it); 19 | } 20 | ~IteratorWrapper() { delete iter_; } 21 | Iterator* iter() const { return iter_; } 22 | 23 | // Takes ownership of "iter" and will delete it when destroyed, or 24 | // when Set() is invoked again. 25 | void Set(Iterator* it) { 26 | delete iter_; 27 | iter_ = it; 28 | if (iter_ == NULL) { 29 | valid_ = false; 30 | } else { 31 | Update(); 32 | } 33 | } 34 | 35 | 36 | // Iterator interface methods 37 | bool Valid() const { return valid_; } 38 | Slice key() const { assert(Valid()); return key_; } 39 | Slice value() const { assert(Valid()); return iter_->value(); } 40 | // Methods below require iter() != NULL 41 | const Status& status() const { assert(iter_); return iter_->status(); } 42 | void Next() { assert(iter_); iter_->Next(); Update(); } 43 | void Prev() { assert(iter_); iter_->Prev(); Update(); } 44 | void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); } 45 | void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); } 46 | void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); } 47 | void MakeInvalid() { valid_ = false; } 48 | 49 | private: 50 | IteratorWrapper(const IteratorWrapper&); 51 | IteratorWrapper& operator = (const IteratorWrapper&); 52 | void Update() { 53 | valid_ = iter_->Valid(); 54 | if (valid_) { 55 | key_ = iter_->key(); 56 | } 57 | } 58 | 59 | Iterator* iter_; 60 | bool valid_; 61 | Slice key_; 62 | }; 63 | 64 | } // namespace leveldb 65 | 66 | #endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 67 | -------------------------------------------------------------------------------- /src/table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 6 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 7 | 8 | #include "db/version_set.h" 9 | #define MAX_NUM_SEEK_THREADS 1024 10 | 11 | namespace leveldb { 12 | 13 | class Comparator; 14 | class Iterator; 15 | 16 | // Return an iterator that provided the union of the data in 17 | // children[0,n-1]. Takes ownership of the child iterators and 18 | // will delete them when the result iterator is deleted. 19 | // 20 | // The result does no duplicate suppression. I.e., if a particular 21 | // key is present in K child iterators, it will be yielded K times. 22 | // 23 | // REQUIRES: n >= 0 24 | extern Iterator* NewMergingIterator( 25 | const Comparator* comparator, Iterator** children, int n); 26 | 27 | extern Iterator* NewMergingIterator( 28 | const Comparator* comparator, Iterator** children, int n, VersionSet* vset); 29 | 30 | extern Iterator* NewMergingIteratorForFiles( 31 | const Comparator* cmp, Iterator** list, FileMetaData** file_meta_list, int n, const InternalKeyComparator* icmp, VersionSet* vset, unsigned level); 32 | 33 | } // namespace leveldb 34 | 35 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 36 | -------------------------------------------------------------------------------- /src/table/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 7 | 8 | #include "pebblesdb/iterator.h" 9 | 10 | namespace leveldb { 11 | 12 | struct ReadOptions; 13 | 14 | // Return a new two level iterator. A two-level iterator contains an 15 | // index iterator whose values point to a sequence of blocks where 16 | // each block is itself a sequence of key,value pairs. The returned 17 | // two-level iterator yields the concatenation of all key/value pairs 18 | // in the sequence of blocks. Takes ownership of "index_iter" and 19 | // will delete it when no longer needed. 20 | // 21 | // Uses a supplied function to convert an index_iter value into 22 | // an iterator over the contents of the corresponding block. 23 | extern Iterator* NewTwoLevelIterator( 24 | Iterator* index_iter, 25 | Iterator* (*block_function)( 26 | void* arg, 27 | const ReadOptions& options, 28 | const Slice& index_value), 29 | void* arg, 30 | const ReadOptions& options); 31 | 32 | extern Iterator* NewTwoLevelIteratorGuards( 33 | Iterator* index_iter, 34 | Iterator* (*block_function)( 35 | void* arg1, const void* arg2, 36 | void* arg3, unsigned level, 37 | const ReadOptions& options, 38 | const Slice& index_value), 39 | void* arg1, const void* arg2, 40 | void* arg3, 41 | unsigned level, 42 | const ReadOptions& options); 43 | } // namespace leveldb 44 | 45 | #endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 46 | -------------------------------------------------------------------------------- /src/util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "util/atomic.h" 13 | #include "util/mutexlock.h" 14 | 15 | namespace leveldb { 16 | 17 | class Arena { 18 | public: 19 | Arena(); 20 | ~Arena() throw (); 21 | 22 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 23 | char* Allocate(size_t bytes); 24 | 25 | // Allocate memory with the normal alignment guarantees provided by malloc 26 | char* AllocateAligned(size_t bytes); 27 | 28 | // Returns an estimate of the total memory usage of data allocated 29 | // by the arena (including space allocated but not yet used for user 30 | // allocations). 31 | uint64_t MemoryUsage() { return atomic::load_64_nobarrier(&memory_usage_); } 32 | 33 | private: 34 | struct Block; 35 | 36 | Block* NewBlock(size_t bytes); 37 | char* AllocateLarge(size_t bytes); 38 | char* AllocateFinalize(Block* b, size_t bytes); 39 | 40 | const size_t align_; 41 | const size_t page_size_; 42 | uint64_t memory_usage_; 43 | Block* blocks_; 44 | Block* large_; 45 | 46 | // No copying allowed 47 | Arena(const Arena&); 48 | void operator=(const Arena&); 49 | }; 50 | 51 | } // namespace leveldb 52 | 53 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 54 | -------------------------------------------------------------------------------- /src/util/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | #include "util/random.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | class ArenaTest { }; 13 | 14 | TEST(ArenaTest, Empty) { 15 | Arena arena; 16 | } 17 | 18 | TEST(ArenaTest, Simple) { 19 | std::vector > allocated; 20 | Arena arena; 21 | const int N = 100000; 22 | size_t bytes = 0; 23 | Random rnd(301); 24 | for (int i = 0; i < N; i++) { 25 | size_t s; 26 | if (i % (N / 10) == 0) { 27 | s = i; 28 | } else { 29 | s = rnd.OneIn(4000) ? rnd.Uniform(6000) : 30 | (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 31 | } 32 | if (s == 0) { 33 | // Our arena disallows size 0 allocations. 34 | s = 1; 35 | } 36 | char* r; 37 | if (rnd.OneIn(10)) { 38 | r = arena.AllocateAligned(s); 39 | } else { 40 | r = arena.Allocate(s); 41 | } 42 | 43 | for (size_t b = 0; b < s; b++) { 44 | // Fill the "i"th allocation with a known bit pattern 45 | r[b] = i % 256; 46 | } 47 | bytes += s; 48 | allocated.push_back(std::make_pair(s, r)); 49 | ASSERT_GE(arena.MemoryUsage(), bytes); 50 | if (i > N/10) { 51 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 52 | } 53 | } 54 | for (size_t i = 0; i < allocated.size(); i++) { 55 | size_t num_bytes = allocated[i].first; 56 | const char* p = allocated[i].second; 57 | for (size_t b = 0; b < num_bytes; b++) { 58 | // Check the "i"th allocation for the known bit pattern 59 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 60 | } 61 | } 62 | } 63 | 64 | } // namespace leveldb 65 | 66 | int main(int argc, char** argv) { 67 | return leveldb::test::RunAllTests(); 68 | } 69 | -------------------------------------------------------------------------------- /src/util/bloom.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/filter_policy.h" 6 | 7 | #include "pebblesdb/slice.h" 8 | #include "util/hash.h" 9 | #include 10 | 11 | namespace leveldb { 12 | 13 | namespace { 14 | static uint32_t BloomHash(const Slice& key) { 15 | return Hash(key.data(), key.size(), 0xbc9f1d34); 16 | } 17 | 18 | class BloomFilterPolicy : public FilterPolicy { 19 | private: 20 | size_t bits_per_key_; 21 | size_t k_; 22 | 23 | public: 24 | explicit BloomFilterPolicy(int bits_per_key) 25 | : bits_per_key_(bits_per_key), 26 | k_(static_cast(bits_per_key * 0.69)) /* 0.69 =~ ln(2) */ { 27 | // We intentionally round down to reduce probing cost a little bit 28 | if (k_ < 1) k_ = 1; 29 | if (k_ > 30) k_ = 30; 30 | byte_size = 0; 31 | filter_count = 0; 32 | } 33 | 34 | virtual const char* Name() const { 35 | return "leveldb.BuiltinBloomFilter"; 36 | } 37 | 38 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { 39 | // Compute bloom filter size (in both bits and bytes) 40 | filter_count++; 41 | size_t bits = n * bits_per_key_; 42 | 43 | // For small n, we can see a very high false positive rate. Fix it 44 | // by enforcing a minimum bloom filter length. 45 | if (bits < 64) bits = 64; 46 | 47 | size_t bytes = (bits + 7) / 8; 48 | bits = bytes * 8; 49 | // if (n > 200) { 50 | // printf("Bloom create filter called. n: %d bytes: %d\n", n, bytes); 51 | // } 52 | byte_size += bytes; 53 | const size_t init_size = dst->size(); 54 | dst->resize(init_size + bytes, 0); 55 | dst->push_back(static_cast(k_)); // Remember # of probes in filter 56 | // if (n > 200) { 57 | // printf("After resizing - dst pointer: %p address of *dst: %p\n", (void*) dst, (void*) &(*dst)); 58 | // } 59 | char* array = &(*dst)[init_size]; 60 | for (int i = 0; i < n; i++) { 61 | // Use double-hashing to generate a sequence of hash values. 62 | // See analysis in [Kirsch,Mitzenmacher 2006]. 63 | uint32_t h = BloomHash(keys[i]); 64 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 65 | for (size_t j = 0; j < k_; j++) { 66 | const uint32_t bitpos = h % bits; 67 | array[bitpos/8] |= (1 << (bitpos % 8)); 68 | h += delta; 69 | } 70 | } 71 | } 72 | 73 | virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const { 74 | const size_t len = bloom_filter.size(); 75 | if (len < 2) return false; 76 | 77 | const char* array = bloom_filter.data(); 78 | const size_t bits = (len - 1) * 8; 79 | 80 | // Use the encoded k so that we can read filters generated by 81 | // bloom filters created using different parameters. 82 | const size_t k = array[len-1]; 83 | if (k > 30) { 84 | // Reserved for potentially new encodings for short bloom filters. 85 | // Consider it a match. 86 | return true; 87 | } 88 | 89 | uint32_t h = BloomHash(key); 90 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 91 | for (size_t j = 0; j < k; j++) { 92 | const uint32_t bitpos = h % bits; 93 | if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false; 94 | h += delta; 95 | } 96 | return true; 97 | } 98 | }; 99 | } 100 | 101 | const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { 102 | return new BloomFilterPolicy(bits_per_key); 103 | } 104 | 105 | } // namespace leveldb 106 | -------------------------------------------------------------------------------- /src/util/bloom_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/filter_policy.h" 6 | 7 | #include "util/coding.h" 8 | #include "util/logging.h" 9 | #include "util/testharness.h" 10 | #include "util/testutil.h" 11 | 12 | namespace leveldb { 13 | 14 | static const int kVerbose = 1; 15 | 16 | static Slice Key(int i, char* buffer) { 17 | EncodeFixed32(buffer, i); 18 | return Slice(buffer, sizeof(uint32_t)); 19 | } 20 | 21 | class BloomTest { 22 | private: 23 | const FilterPolicy* policy_; 24 | std::string filter_; 25 | std::vector keys_; 26 | 27 | public: 28 | BloomTest() : policy_(NewBloomFilterPolicy(10)) { } 29 | 30 | ~BloomTest() { 31 | delete policy_; 32 | } 33 | 34 | void Reset() { 35 | keys_.clear(); 36 | filter_.clear(); 37 | } 38 | 39 | void Add(const Slice& s) { 40 | keys_.push_back(s.ToString()); 41 | } 42 | 43 | void Build() { 44 | std::vector key_slices; 45 | for (size_t i = 0; i < keys_.size(); i++) { 46 | key_slices.push_back(Slice(keys_[i])); 47 | } 48 | filter_.clear(); 49 | policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_); 50 | keys_.clear(); 51 | if (kVerbose >= 2) DumpFilter(); 52 | } 53 | 54 | size_t FilterSize() const { 55 | return filter_.size(); 56 | } 57 | 58 | void DumpFilter() { 59 | fprintf(stderr, "F("); 60 | for (size_t i = 0; i+1 < filter_.size(); i++) { 61 | const unsigned int c = static_cast(filter_[i]); 62 | for (int j = 0; j < 8; j++) { 63 | fprintf(stderr, "%c", (c & (1 <KeyMayMatch(s, filter_); 74 | } 75 | 76 | double FalsePositiveRate() { 77 | char buffer[sizeof(int)]; 78 | int result = 0; 79 | for (int i = 0; i < 10000; i++) { 80 | if (Matches(Key(i + 1000000000, buffer))) { 81 | result++; 82 | } 83 | } 84 | return result / 10000.0; 85 | } 86 | }; 87 | 88 | TEST(BloomTest, EmptyFilter) { 89 | ASSERT_TRUE(! Matches("hello")); 90 | ASSERT_TRUE(! Matches("world")); 91 | } 92 | 93 | TEST(BloomTest, Small) { 94 | Add("hello"); 95 | Add("world"); 96 | ASSERT_TRUE(Matches("hello")); 97 | ASSERT_TRUE(Matches("world")); 98 | ASSERT_TRUE(! Matches("x")); 99 | ASSERT_TRUE(! Matches("foo")); 100 | } 101 | 102 | static int NextLength(int length) { 103 | if (length < 10) { 104 | length += 1; 105 | } else if (length < 100) { 106 | length += 10; 107 | } else if (length < 1000) { 108 | length += 100; 109 | } else { 110 | length += 1000; 111 | } 112 | return length; 113 | } 114 | 115 | TEST(BloomTest, VaryingLengths) { 116 | char buffer[sizeof(int)]; 117 | 118 | // Count number of filters that significantly exceed the false positive rate 119 | int mediocre_filters = 0; 120 | int good_filters = 0; 121 | 122 | for (int length = 1; length <= 10000; length = NextLength(length)) { 123 | Reset(); 124 | for (int i = 0; i < length; i++) { 125 | Add(Key(i, buffer)); 126 | } 127 | Build(); 128 | 129 | ASSERT_LE(FilterSize(), static_cast((length * 10 / 8) + 40)) 130 | << length; 131 | 132 | // All added keys must match 133 | for (int i = 0; i < length; i++) { 134 | ASSERT_TRUE(Matches(Key(i, buffer))) 135 | << "Length " << length << "; key " << i; 136 | } 137 | 138 | // Check false positive rate 139 | double rate = FalsePositiveRate(); 140 | if (kVerbose >= 1) { 141 | fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", 142 | rate*100.0, length, static_cast(FilterSize())); 143 | } 144 | ASSERT_LE(rate, 0.02); // Must not be over 2% 145 | if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often 146 | else good_filters++; 147 | } 148 | if (kVerbose >= 1) { 149 | fprintf(stderr, "Filters: %d good, %d mediocre\n", 150 | good_filters, mediocre_filters); 151 | } 152 | ASSERT_LE(mediocre_filters, good_filters/5); 153 | } 154 | 155 | // Different bits-per-byte 156 | 157 | } // namespace leveldb 158 | 159 | int main(int argc, char** argv) { 160 | return leveldb::test::RunAllTests(); 161 | } 162 | -------------------------------------------------------------------------------- /src/util/comparator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include "pebblesdb/comparator.h" 8 | #include "pebblesdb/slice.h" 9 | #include "port/port.h" 10 | #include "util/coding.h" 11 | #include "util/logging.h" 12 | 13 | namespace leveldb { 14 | 15 | Comparator::~Comparator() { } 16 | 17 | uint64_t Comparator::KeyNum(const Slice& /*key*/) const { 18 | return 0; 19 | } 20 | 21 | namespace { 22 | class BytewiseComparatorImpl : public Comparator { 23 | public: 24 | BytewiseComparatorImpl() { } 25 | 26 | virtual const char* Name() const { 27 | return "leveldb.BytewiseComparator"; 28 | } 29 | 30 | virtual int Compare(const Slice& a, const Slice& b) const { 31 | return a.compare(b); 32 | } 33 | 34 | virtual void FindShortestSeparator( 35 | std::string* start, 36 | const Slice& limit) const { 37 | // Find length of common prefix 38 | size_t min_length = std::min(start->size(), limit.size()); 39 | size_t diff_index = 0; 40 | while ((diff_index < min_length) && 41 | ((*start)[diff_index] == limit[diff_index])) { 42 | diff_index++; 43 | } 44 | 45 | if (diff_index >= min_length) { 46 | // Do not shorten if one string is a prefix of the other 47 | } else { 48 | uint8_t diff_byte = static_cast((*start)[diff_index]); 49 | if (diff_byte < static_cast(0xff) && 50 | diff_byte + 1 < static_cast(limit[diff_index])) { 51 | (*start)[diff_index]++; 52 | start->resize(diff_index + 1); 53 | assert(Compare(*start, limit) < 0); 54 | } 55 | } 56 | } 57 | 58 | virtual void FindShortSuccessor(std::string* key) const { 59 | // Find first character that can be incremented 60 | size_t n = key->size(); 61 | for (size_t i = 0; i < n; i++) { 62 | const uint8_t byte = (*key)[i]; 63 | if (byte != static_cast(0xff)) { 64 | (*key)[i] = byte + 1; 65 | key->resize(i+1); 66 | return; 67 | } 68 | } 69 | // *key is a run of 0xffs. Leave it alone. 70 | } 71 | 72 | virtual uint64_t KeyNum(const Slice& key) const { 73 | unsigned char buf[sizeof(uint64_t)]; 74 | memset(buf, 0, sizeof(buf)); 75 | memmove(buf, key.data(), std::min(key.size(), sizeof(uint64_t))); 76 | uint64_t number; 77 | number = static_cast(buf[0]) << 56 78 | | static_cast(buf[1]) << 48 79 | | static_cast(buf[2]) << 40 80 | | static_cast(buf[3]) << 32 81 | | static_cast(buf[4]) << 24 82 | | static_cast(buf[5]) << 16 83 | | static_cast(buf[6]) << 8 84 | | static_cast(buf[7]); 85 | return number; 86 | } 87 | }; 88 | } // namespace 89 | 90 | static port::OnceType once = LEVELDB_ONCE_INIT; 91 | static const Comparator* bytewise; 92 | 93 | static void InitModule() { 94 | bytewise = new BytewiseComparatorImpl; 95 | } 96 | 97 | const Comparator* BytewiseComparator() { 98 | port::InitOnce(&once, InitModule); 99 | return bytewise; 100 | } 101 | 102 | } // namespace leveldb 103 | -------------------------------------------------------------------------------- /src/util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ 6 | #define STORAGE_LEVELDB_UTIL_CRC32C_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace crc32c { 13 | 14 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 15 | // crc32c of some string A. Extend() is often used to maintain the 16 | // crc32c of a stream of data. 17 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 18 | 19 | // Return the crc32c of data[0,n-1] 20 | inline uint32_t Value(const char* data, size_t n) { 21 | return Extend(0, data, n); 22 | } 23 | 24 | static const uint32_t kMaskDelta = 0xa282ead8ul; 25 | 26 | // Return a masked representation of crc. 27 | // 28 | // Motivation: it is problematic to compute the CRC of a string that 29 | // contains embedded CRCs. Therefore we recommend that CRCs stored 30 | // somewhere (e.g., in files) should be masked before being stored. 31 | inline uint32_t Mask(uint32_t crc) { 32 | // Rotate right by 15 bits and add a constant. 33 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 34 | } 35 | 36 | // Return the crc whose masked representation is masked_crc. 37 | inline uint32_t Unmask(uint32_t masked_crc) { 38 | uint32_t rot = masked_crc - kMaskDelta; 39 | return ((rot >> 17) | (rot << 15)); 40 | } 41 | 42 | } // namespace crc32c 43 | } // namespace leveldb 44 | 45 | #endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ 46 | -------------------------------------------------------------------------------- /src/util/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/crc32c.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | namespace crc32c { 10 | 11 | class CRC { }; 12 | 13 | TEST(CRC, StandardResults) { 14 | // From rfc3720 section B.4. 15 | char buf[32]; 16 | 17 | memset(buf, 0, sizeof(buf)); 18 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 19 | 20 | memset(buf, 0xff, sizeof(buf)); 21 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 22 | 23 | for (int i = 0; i < 32; i++) { 24 | buf[i] = i; 25 | } 26 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 27 | 28 | for (int i = 0; i < 32; i++) { 29 | buf[i] = 31 - i; 30 | } 31 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 32 | 33 | unsigned char data[48] = { 34 | 0x01, 0xc0, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x00, 0x00, 37 | 0x00, 0x00, 0x00, 0x00, 38 | 0x14, 0x00, 0x00, 0x00, 39 | 0x00, 0x00, 0x04, 0x00, 40 | 0x00, 0x00, 0x00, 0x14, 41 | 0x00, 0x00, 0x00, 0x18, 42 | 0x28, 0x00, 0x00, 0x00, 43 | 0x00, 0x00, 0x00, 0x00, 44 | 0x02, 0x00, 0x00, 0x00, 45 | 0x00, 0x00, 0x00, 0x00, 46 | }; 47 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 48 | } 49 | 50 | TEST(CRC, LargeBuffer) { 51 | std::string tmp("A"); 52 | while (tmp.size() < 4096) { 53 | tmp = tmp + tmp; 54 | } 55 | ASSERT_EQ(0x3c36f666, Value(tmp.data(), 64)); 56 | ASSERT_EQ(0xf6607a92, Value(tmp.data(), 1024)); 57 | ASSERT_EQ(0xa9bc21ef, Value(tmp.data(), 1111)); 58 | ASSERT_EQ(0x88ddb66b, Value(tmp.data(), 2048)); 59 | ASSERT_EQ(0x057251e9, Value(tmp.data(), 4096)); 60 | } 61 | 62 | TEST(CRC, Values) { 63 | ASSERT_NE(Value("a", 1), Value("foo", 3)); 64 | } 65 | 66 | TEST(CRC, Extend) { 67 | ASSERT_EQ(Value("hello world", 11), 68 | Extend(Value("hello ", 6), "world", 5)); 69 | } 70 | 71 | TEST(CRC, Mask) { 72 | uint32_t crc = Value("foo", 3); 73 | ASSERT_NE(crc, Mask(crc)); 74 | ASSERT_NE(crc, Mask(Mask(crc))); 75 | ASSERT_EQ(crc, Unmask(Mask(crc))); 76 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 77 | } 78 | 79 | } // namespace crc32c 80 | } // namespace leveldb 81 | 82 | int main(int argc, char** argv) { 83 | return leveldb::test::RunAllTests(); 84 | } 85 | -------------------------------------------------------------------------------- /src/util/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/env.h" 6 | 7 | namespace leveldb { 8 | 9 | Env::~Env() { 10 | } 11 | 12 | SequentialFile::~SequentialFile() { 13 | } 14 | 15 | RandomAccessFile::~RandomAccessFile() { 16 | } 17 | 18 | WritableFile::~WritableFile() { 19 | } 20 | 21 | ConcurrentWritableFile::~ConcurrentWritableFile() { 22 | } 23 | 24 | Logger::~Logger() { 25 | } 26 | 27 | FileLock::~FileLock() { 28 | } 29 | 30 | void Log(Logger* info_log, const char* format, ...) { 31 | if (info_log != NULL) { 32 | va_list ap; 33 | va_start(ap, format); 34 | info_log->Logv(format, ap); 35 | va_end(ap); 36 | } 37 | } 38 | 39 | static Status DoWriteStringToFile(Env* env, const Slice& data, 40 | const std::string& fname, 41 | bool should_sync) { 42 | WritableFile* file; 43 | Status s = env->NewWritableFile(fname, &file); 44 | if (!s.ok()) { 45 | return s; 46 | } 47 | s = file->Append(data); 48 | if (s.ok() && should_sync) { 49 | s = file->Sync(); 50 | } 51 | if (s.ok()) { 52 | s = file->Close(); 53 | } 54 | delete file; // Will auto-close if we did not close above 55 | if (!s.ok()) { 56 | env->DeleteFile(fname); 57 | } 58 | return s; 59 | } 60 | 61 | Status WriteStringToFile(Env* env, const Slice& data, 62 | const std::string& fname) { 63 | return DoWriteStringToFile(env, data, fname, false); 64 | } 65 | 66 | Status WriteStringToFileSync(Env* env, const Slice& data, 67 | const std::string& fname) { 68 | return DoWriteStringToFile(env, data, fname, true); 69 | } 70 | 71 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { 72 | data->clear(); 73 | SequentialFile* file; 74 | Status s = env->NewSequentialFile(fname, &file); 75 | if (!s.ok()) { 76 | return s; 77 | } 78 | static const int kBufferSize = 8192; 79 | char* space = new char[kBufferSize]; 80 | while (true) { 81 | Slice fragment; 82 | s = file->Read(kBufferSize, &fragment, space); 83 | if (!s.ok()) { 84 | break; 85 | } 86 | data->append(fragment.data(), fragment.size()); 87 | if (fragment.empty()) { 88 | break; 89 | } 90 | } 91 | delete[] space; 92 | delete file; 93 | return s; 94 | } 95 | 96 | EnvWrapper::~EnvWrapper() { 97 | } 98 | 99 | } // namespace leveldb 100 | -------------------------------------------------------------------------------- /src/util/env_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/env.h" 6 | 7 | #include "port/port.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | static const int kDelayMicros = 100000; 13 | 14 | class EnvPosixTest { 15 | private: 16 | port::Mutex mu_; 17 | std::string events_; 18 | 19 | public: 20 | Env* env_; 21 | EnvPosixTest() : env_(Env::Default()) { } 22 | }; 23 | 24 | static void SetBool(void* ptr) { 25 | reinterpret_cast(ptr)->NoBarrier_Store(ptr); 26 | } 27 | 28 | TEST(EnvPosixTest, RunImmediately) { 29 | port::AtomicPointer called (NULL); 30 | env_->Schedule(&SetBool, &called); 31 | Env::Default()->SleepForMicroseconds(kDelayMicros); 32 | ASSERT_TRUE(called.NoBarrier_Load() != NULL); 33 | } 34 | 35 | TEST(EnvPosixTest, RunMany) { 36 | port::AtomicPointer last_id (NULL); 37 | 38 | struct CB { 39 | port::AtomicPointer* last_id_ptr; // Pointer to shared slot 40 | uintptr_t id; // Order# for the execution of this callback 41 | 42 | CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { } 43 | 44 | static void Run(void* v) { 45 | CB* cb = reinterpret_cast(v); 46 | void* cur = cb->last_id_ptr->NoBarrier_Load(); 47 | ASSERT_EQ(cb->id-1, reinterpret_cast(cur)); 48 | cb->last_id_ptr->Release_Store(reinterpret_cast(cb->id)); 49 | } 50 | }; 51 | 52 | // Schedule in different order than start time 53 | CB cb1(&last_id, 1); 54 | CB cb2(&last_id, 2); 55 | CB cb3(&last_id, 3); 56 | CB cb4(&last_id, 4); 57 | env_->Schedule(&CB::Run, &cb1); 58 | env_->Schedule(&CB::Run, &cb2); 59 | env_->Schedule(&CB::Run, &cb3); 60 | env_->Schedule(&CB::Run, &cb4); 61 | 62 | Env::Default()->SleepForMicroseconds(kDelayMicros); 63 | void* cur = last_id.Acquire_Load(); 64 | ASSERT_EQ(4, reinterpret_cast(cur)); 65 | } 66 | 67 | struct State { 68 | port::Mutex mu; 69 | int val; 70 | int num_running; 71 | }; 72 | 73 | static void ThreadBody(void* arg) { 74 | State* s = reinterpret_cast(arg); 75 | s->mu.Lock(); 76 | s->val += 1; 77 | s->num_running -= 1; 78 | s->mu.Unlock(); 79 | } 80 | 81 | TEST(EnvPosixTest, StartThread) { 82 | State state; 83 | state.val = 0; 84 | state.num_running = 3; 85 | for (int i = 0; i < 3; i++) { 86 | env_->StartThread(&ThreadBody, &state); 87 | } 88 | while (true) { 89 | state.mu.Lock(); 90 | int num = state.num_running; 91 | state.mu.Unlock(); 92 | if (num == 0) { 93 | break; 94 | } 95 | Env::Default()->SleepForMicroseconds(kDelayMicros); 96 | } 97 | ASSERT_EQ(state.val, 3); 98 | } 99 | 100 | } // namespace leveldb 101 | 102 | int main(int argc, char** argv) { 103 | return leveldb::test::RunAllTests(); 104 | } 105 | -------------------------------------------------------------------------------- /src/util/filter_policy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/filter_policy.h" 6 | 7 | namespace leveldb { 8 | 9 | FilterPolicy::~FilterPolicy() { } 10 | 11 | } // namespace leveldb 12 | -------------------------------------------------------------------------------- /src/util/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "util/coding.h" 7 | #include "util/hash.h" 8 | 9 | // The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through 10 | // between switch labels. The real definition should be provided externally. 11 | // This one is a fallback version for unsupported compilers. 12 | #ifndef FALLTHROUGH_INTENDED 13 | #define FALLTHROUGH_INTENDED do { } while (0) 14 | #endif 15 | 16 | namespace leveldb { 17 | 18 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 19 | // Similar to murmur hash 20 | const uint32_t m = 0xc6a4a793; 21 | const uint32_t r = 24; 22 | const char* limit = data + n; 23 | uint32_t h = seed ^ (n * m); 24 | 25 | // Pick up four bytes at a time 26 | while (data + 4 <= limit) { 27 | uint32_t w = DecodeFixed32(data); 28 | data += 4; 29 | h += w; 30 | h *= m; 31 | h ^= (h >> 16); 32 | } 33 | 34 | // Pick up remaining bytes 35 | switch (limit - data) { 36 | case 3: 37 | h += data[2] << 16; 38 | FALLTHROUGH_INTENDED; 39 | case 2: 40 | h += data[1] << 8; 41 | FALLTHROUGH_INTENDED; 42 | case 1: 43 | h += data[0]; 44 | h *= m; 45 | h ^= (h >> r); 46 | break; 47 | default: 48 | break; 49 | } 50 | return h; 51 | } 52 | 53 | 54 | } // namespace leveldb 55 | -------------------------------------------------------------------------------- /src/util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Simple hash function used for internal data structures 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 8 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 9 | 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 16 | 17 | } 18 | 19 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 20 | -------------------------------------------------------------------------------- /src/util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 6 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Histogram { 13 | public: 14 | Histogram() 15 | : min_(0), 16 | max_(0), 17 | num_(0), 18 | sum_(0), 19 | sum_squares_(0) { } 20 | ~Histogram() throw () { } 21 | 22 | void Clear(); 23 | void Add(double value); 24 | void Merge(const Histogram& other); 25 | 26 | std::string ToString() const; 27 | 28 | private: 29 | double min_; 30 | double max_; 31 | double num_; 32 | double sum_; 33 | double sum_squares_; 34 | 35 | enum { kNumBuckets = 154 }; 36 | static const double kBucketLimit[kNumBuckets]; 37 | double buckets_[kNumBuckets]; 38 | 39 | double Median() const; 40 | double Percentile(double p) const; 41 | double Average() const; 42 | double StandardDeviation() const; 43 | }; 44 | 45 | } // namespace leveldb 46 | 47 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 48 | -------------------------------------------------------------------------------- /src/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #define __STDC_LIMIT_MACROS 6 | 7 | #include "util/logging.h" 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "pebblesdb/env.h" 14 | #include "pebblesdb/slice.h" 15 | 16 | namespace leveldb { 17 | 18 | void AppendNumberTo(std::string* str, uint64_t num) { 19 | char buf[30]; 20 | snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); 21 | str->append(buf); 22 | } 23 | 24 | void AppendDoubleTo(std::string* str, double num) { 25 | char buf[30]; 26 | snprintf(buf, sizeof(buf), "%f", num); 27 | str->append(buf); 28 | } 29 | 30 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 31 | for (size_t i = 0; i < value.size(); i++) { 32 | char c = value[i]; 33 | if (c >= ' ' && c <= '~') { 34 | str->push_back(c); 35 | } else { 36 | char buf[10]; 37 | snprintf(buf, sizeof(buf), "\\x%02x", 38 | static_cast(c) & 0xff); 39 | str->append(buf); 40 | } 41 | } 42 | } 43 | 44 | std::string NumberToString(uint64_t num) { 45 | std::string r; 46 | AppendNumberTo(&r, num); 47 | return r; 48 | } 49 | 50 | std::string EscapeString(const Slice& value) { 51 | std::string r; 52 | AppendEscapedStringTo(&r, value); 53 | return r; 54 | } 55 | 56 | bool ConsumeChar(Slice* in, char c) { 57 | if (!in->empty() && (*in)[0] == c) { 58 | in->remove_prefix(1); 59 | return true; 60 | } else { 61 | return false; 62 | } 63 | } 64 | 65 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 66 | uint64_t v = 0; 67 | int digits = 0; 68 | while (!in->empty()) { 69 | char c = (*in)[0]; 70 | if (c >= '0' && c <= '9') { 71 | ++digits; 72 | const unsigned delta = (c - '0'); 73 | if (v > UINT64_MAX/10 || 74 | (v == UINT64_MAX/10 && delta > UINT64_MAX%10)) { 75 | // Overflow 76 | return false; 77 | } 78 | v = (v * 10) + delta; 79 | in->remove_prefix(1); 80 | } else { 81 | break; 82 | } 83 | } 84 | *val = v; 85 | return (digits > 0); 86 | } 87 | 88 | } // namespace leveldb 89 | -------------------------------------------------------------------------------- /src/util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Slice; 19 | class WritableFile; 20 | 21 | // Append a human-readable printout of "num" to *str 22 | extern void AppendNumberTo(std::string* str, uint64_t num); 23 | 24 | // Append a human-readable printout of "num" to *str 25 | extern void AppendDoubleTo(std::string* str, double num); 26 | 27 | // Append a human-readable printout of "value" to *str. 28 | // Escapes any non-printable characters found in "value". 29 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 30 | 31 | // Return a human-readable printout of "num" 32 | extern std::string NumberToString(uint64_t num); 33 | 34 | // Return a human-readable version of "value". 35 | // Escapes any non-printable characters found in "value". 36 | extern std::string EscapeString(const Slice& value); 37 | 38 | // If *in starts with "c", advances *in past the first character and 39 | // returns true. Otherwise, returns false. 40 | extern bool ConsumeChar(Slice* in, char c); 41 | 42 | // Parse a human-readable number from "*in" into *value. On success, 43 | // advances "*in" past the consumed number and sets "*val" to the 44 | // numeric value. Otherwise, returns false and leaves *in in an 45 | // unspecified state. 46 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 47 | 48 | } // namespace leveldb 49 | 50 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 51 | -------------------------------------------------------------------------------- /src/util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | #include "port/thread_annotations.h" 10 | 11 | namespace leveldb { 12 | 13 | // Helper class that locks a mutex on construction and unlocks the mutex when 14 | // the destructor of the MutexLock object is invoked. 15 | // 16 | // Typical usage: 17 | // 18 | // void MyClass::MyMethod() { 19 | // MutexLock l(&mu_); // mu_ is an instance variable 20 | // ... some complex code, possibly with multiple return paths ... 21 | // } 22 | 23 | class SCOPED_LOCKABLE MutexLock { 24 | public: 25 | explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) 26 | : mu_(mu) { 27 | this->mu_->Lock(); 28 | } 29 | ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 30 | 31 | private: 32 | port::Mutex *const mu_; 33 | // No copying allowed 34 | MutexLock(const MutexLock&); 35 | void operator=(const MutexLock&); 36 | }; 37 | 38 | } // namespace leveldb 39 | 40 | 41 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 42 | -------------------------------------------------------------------------------- /src/util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/options.h" 6 | 7 | #include "pebblesdb/comparator.h" 8 | #include "pebblesdb/env.h" 9 | 10 | namespace leveldb { 11 | 12 | Options::Options() 13 | : comparator(BytewiseComparator()), 14 | create_if_missing(false), 15 | error_if_exists(false), 16 | paranoid_checks(false), 17 | env(Env::Default()), 18 | info_log(NULL), 19 | write_buffer_size(4<<20), 20 | max_open_files(1000), 21 | block_cache(NULL), 22 | block_size(4096), 23 | block_restart_interval(16), 24 | compression(kNoCompression), 25 | filter_policy(NULL), 26 | manual_garbage_collection(false) { 27 | } 28 | 29 | 30 | } // namespace leveldb 31 | -------------------------------------------------------------------------------- /src/util/posix_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Logger implementation that can be shared by all environments 6 | // where enough posix functionality is available. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 9 | #define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "pebblesdb/env.h" 16 | 17 | #pragma GCC diagnostic push 18 | #pragma GCC diagnostic ignored "-Wmissing-format-attribute" 19 | 20 | namespace leveldb { 21 | 22 | class PosixLogger : public Logger { 23 | private: 24 | PosixLogger(const PosixLogger&); 25 | PosixLogger& operator = (const PosixLogger&); 26 | FILE* file_; 27 | uint64_t (*gettid_)(); // Return the thread id for the current thread 28 | public: 29 | PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { } 30 | virtual ~PosixLogger() { 31 | fclose(file_); 32 | } 33 | virtual void Logv(const char* format, va_list ap) { 34 | const uint64_t thread_id = (*gettid_)(); 35 | 36 | // We try twice: the first time with a fixed-size stack allocated buffer, 37 | // and the second time with a much larger dynamically allocated buffer. 38 | char buffer[500]; 39 | for (int iter = 0; iter < 2; iter++) { 40 | char* base; 41 | int bufsize; 42 | if (iter == 0) { 43 | bufsize = sizeof(buffer); 44 | base = buffer; 45 | } else { 46 | bufsize = 30000; 47 | base = new char[bufsize]; 48 | } 49 | char* p = base; 50 | char* limit = base + bufsize; 51 | 52 | struct timeval now_tv; 53 | gettimeofday(&now_tv, NULL); 54 | const time_t seconds = now_tv.tv_sec; 55 | struct tm t; 56 | localtime_r(&seconds, &t); 57 | p += snprintf(p, limit - p, 58 | "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", 59 | t.tm_year + 1900, 60 | t.tm_mon + 1, 61 | t.tm_mday, 62 | t.tm_hour, 63 | t.tm_min, 64 | t.tm_sec, 65 | static_cast(now_tv.tv_usec), 66 | static_cast(thread_id)); 67 | 68 | // Print the message 69 | if (p < limit) { 70 | va_list backup_ap; 71 | va_copy(backup_ap, ap); 72 | p += vsnprintf(p, limit - p, format, backup_ap); 73 | va_end(backup_ap); 74 | } 75 | 76 | // Truncate to available space if necessary 77 | if (p >= limit) { 78 | if (iter == 0) { 79 | continue; // Try again with larger buffer 80 | } else { 81 | p = limit - 1; 82 | } 83 | } 84 | 85 | // Add newline if necessary 86 | if (p == base || p[-1] != '\n') { 87 | *p++ = '\n'; 88 | } 89 | 90 | assert(p <= limit); 91 | fwrite(base, 1, p - base, file_); 92 | fflush(file_); 93 | if (base != buffer) { 94 | delete[] base; 95 | } 96 | break; 97 | } 98 | } 99 | }; 100 | 101 | } // namespace leveldb 102 | 103 | #pragma GCC diagnostic pop 104 | 105 | #endif // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 106 | -------------------------------------------------------------------------------- /src/util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ 6 | #define STORAGE_LEVELDB_UTIL_RANDOM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | // A very simple random number generator. Not especially good at 13 | // generating truly random bits, but good enough for our needs in this 14 | // package. 15 | class Random { 16 | private: 17 | uint32_t seed_; 18 | public: 19 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { 20 | // Avoid bad seeds. 21 | if (seed_ == 0 || seed_ == 2147483647L) { 22 | seed_ = 1; 23 | } 24 | } 25 | uint32_t Next() { 26 | static const uint32_t M = 2147483647L; // 2^31-1 27 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 28 | // We are computing 29 | // seed_ = (seed_ * A) % M, where M = 2^31-1 30 | // 31 | // seed_ must not be zero or M, or else all subsequent computed values 32 | // will be zero or M respectively. For all other values, seed_ will end 33 | // up cycling through every number in [1,M-1] 34 | uint64_t product = seed_ * A; 35 | 36 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 37 | seed_ = static_cast((product >> 31) + (product & M)); 38 | // The first reduction may overflow by 1 bit, so we may need to 39 | // repeat. mod == M is not possible; using > allows the faster 40 | // sign-bit-based test. 41 | if (seed_ > M) { 42 | seed_ -= M; 43 | } 44 | return seed_; 45 | } 46 | // Returns a uniformly distributed value in the range [0..n-1] 47 | // REQUIRES: n > 0 48 | uint32_t Uniform(int n) { return Next() % n; } 49 | 50 | // Randomly returns true ~"1/n" of the time, and false otherwise. 51 | // REQUIRES: n > 0 52 | bool OneIn(int n) { return (Next() % n) == 0; } 53 | 54 | // Skewed: pick "base" uniformly from range [0,max_log] and then 55 | // return "base" random bits. The effect is to pick a number in the 56 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 57 | uint32_t Skewed(int max_log) { 58 | return Uniform(1 << Uniform(max_log + 1)); 59 | } 60 | }; 61 | 62 | } // namespace leveldb 63 | 64 | #endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ 65 | -------------------------------------------------------------------------------- /src/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "port/port.h" 7 | #include "pebblesdb/status.h" 8 | 9 | namespace leveldb { 10 | 11 | const char* Status::CopyState(const char* state) { 12 | uint32_t size; 13 | memcpy(&size, state, sizeof(size)); 14 | char* result = new char[size + 5]; 15 | memcpy(result, state, size + 5); 16 | return result; 17 | } 18 | 19 | Status::Status(Code c, const Slice& msg, const Slice& msg2) 20 | : state_(NULL) { 21 | assert(c != kOk); 22 | const uint32_t len1 = msg.size(); 23 | const uint32_t len2 = msg2.size(); 24 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 25 | char* result = new char[size + 5]; 26 | memcpy(result, &size, sizeof(size)); 27 | result[4] = static_cast(c); 28 | memcpy(result + 5, msg.data(), len1); 29 | if (len2) { 30 | result[5 + len1] = ':'; 31 | result[6 + len1] = ' '; 32 | memcpy(result + 7 + len1, msg2.data(), len2); 33 | } 34 | state_ = result; 35 | } 36 | 37 | std::string Status::ToString() const { 38 | if (state_ == NULL) { 39 | return "OK"; 40 | } else { 41 | char tmp[30]; 42 | const char* type; 43 | switch (code()) { 44 | case kOk: 45 | type = "OK"; 46 | break; 47 | case kNotFound: 48 | type = "NotFound: "; 49 | break; 50 | case kCorruption: 51 | type = "Corruption: "; 52 | break; 53 | case kNotSupported: 54 | type = "Not implemented: "; 55 | break; 56 | case kInvalidArgument: 57 | type = "Invalid argument: "; 58 | break; 59 | case kIOError: 60 | type = "IO error: "; 61 | break; 62 | default: 63 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 64 | static_cast(code())); 65 | type = tmp; 66 | break; 67 | } 68 | std::string result(type); 69 | uint32_t length; 70 | memcpy(&length, state_, sizeof(length)); 71 | result.append(state_ + 5, length); 72 | return result; 73 | } 74 | } 75 | 76 | } // namespace leveldb 77 | -------------------------------------------------------------------------------- /src/util/string_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "pebblesdb/slice.h" 6 | #include 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_STRING_BUILDER_H_ 9 | #define STORAGE_LEVELDB_UTIL_STRING_BUILDER_H_ 10 | 11 | namespace leveldb { 12 | 13 | struct StringBuilder { 14 | StringBuilder() 15 | : m_buf(static_cast(malloc(8))), 16 | m_cap(8), 17 | m_sz(0) { 18 | m_cap = m_buf ? 8 : 0; 19 | } 20 | ~StringBuilder() throw () { if (m_buf) free(m_buf); } 21 | 22 | void destroy_memory() { 23 | if (m_buf) { 24 | free(m_buf); 25 | m_buf = NULL; 26 | m_sz = 0; 27 | m_cap = 0; 28 | } 29 | } 30 | 31 | void init_buffer(int size) { 32 | if (m_buf) { 33 | free(m_buf); 34 | } 35 | m_buf = static_cast (malloc(size)); 36 | m_cap = m_buf ? size: 0; 37 | m_sz = 0; 38 | } 39 | 40 | void clear() { 41 | m_sz = 0; 42 | // m_buf = static_cast(realloc(m_buf, 8)); 43 | } 44 | size_t size() const { return m_sz; } 45 | bool empty() const { return size() == 0; } 46 | Slice slice() const { return Slice(m_buf, m_sz); } 47 | const char* data() const { return m_buf; } 48 | char* data() { return m_buf; } 49 | void append(const leveldb::Slice& s) { 50 | append(s.data(), s.size()); 51 | } 52 | void shrink(size_t sz) { 53 | assert(sz <= m_sz); 54 | m_sz = sz; 55 | } 56 | void append(const char* buf, size_t sz, int print = 0) { 57 | if (m_sz + sz > m_cap) { 58 | grow_at_least(sz, print); 59 | } 60 | 61 | char* ptr = m_buf + m_sz; 62 | memmove(ptr, buf, sz); 63 | m_sz += sz; 64 | } 65 | 66 | private: 67 | StringBuilder(const StringBuilder&); 68 | StringBuilder& operator = (const StringBuilder&); 69 | void grow_at_least(size_t sz, int print) { 70 | size_t new_cap = m_cap + (m_cap >> 1) + sz; 71 | new_cap = (new_cap + 31) & ~31ULL; 72 | // if (print == 1) { 73 | // printf("Growing m_buf from %d to %d to accomodate key of size %d\n", m_cap, new_cap, sz); 74 | // } 75 | char* new_buf = static_cast(realloc(m_buf, new_cap)); 76 | 77 | if (!new_buf) { 78 | throw std::bad_alloc(); 79 | } 80 | 81 | m_buf = new_buf; 82 | m_cap = new_cap; 83 | } 84 | 85 | char* m_buf; 86 | size_t m_cap; 87 | size_t m_sz; 88 | }; 89 | 90 | } // namespace leveldb 91 | 92 | #endif // STORAGE_LEVELDB_UTIL_STRING_BUILDER_H_ 93 | -------------------------------------------------------------------------------- /src/util/testharness.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testharness.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | namespace { 16 | struct Test { 17 | const char* base; 18 | const char* name; 19 | void (*func)(); 20 | }; 21 | std::vector* tests; 22 | } 23 | 24 | bool RegisterTest(const char* base, const char* name, void (*func)()) { 25 | if (tests == NULL) { 26 | tests = new std::vector; 27 | } 28 | Test t; 29 | t.base = base; 30 | t.name = name; 31 | t.func = func; 32 | tests->push_back(t); 33 | return true; 34 | } 35 | 36 | int RunAllTests() { 37 | const char* matcher = getenv("LEVELDB_TESTS"); 38 | 39 | int num = 0; 40 | if (tests != NULL) { 41 | for (size_t i = 0; i < tests->size(); i++) { 42 | const Test& t = (*tests)[i]; 43 | if (matcher != NULL) { 44 | std::string name = t.base; 45 | name.push_back('.'); 46 | name.append(t.name); 47 | if (strstr(name.c_str(), matcher) == NULL) { 48 | continue; 49 | } 50 | } 51 | fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); 52 | (*t.func)(); 53 | ++num; 54 | } 55 | } 56 | fprintf(stderr, "==== PASSED %d tests\n", num); 57 | return 0; 58 | } 59 | 60 | std::string TmpDir() { 61 | std::string dir; 62 | Status s = Env::Default()->GetTestDirectory(&dir); 63 | ASSERT_TRUE(s.ok()) << s.ToString(); 64 | return dir; 65 | } 66 | 67 | int RandomSeed() { 68 | const char* env = getenv("TEST_RANDOM_SEED"); 69 | int result = (env != NULL ? atoi(env) : 301); 70 | if (result <= 0) { 71 | result = 301; 72 | } 73 | return result; 74 | } 75 | 76 | } // namespace test 77 | } // namespace leveldb 78 | -------------------------------------------------------------------------------- /src/util/testutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testutil.h" 6 | 7 | #include "util/random.h" 8 | 9 | namespace leveldb { 10 | namespace test { 11 | 12 | Slice RandomString(Random* rnd, int len, std::string* dst) { 13 | dst->resize(len); 14 | for (int i = 0; i < len; i++) { 15 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 16 | } 17 | return Slice(*dst); 18 | } 19 | 20 | std::string RandomKey(Random* rnd, int len) { 21 | // Make sure to generate a wide variety of characters so we 22 | // test the boundary conditions for short-key optimizations. 23 | static const char kTestChars[] = { 24 | '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' 25 | }; 26 | std::string result; 27 | for (int i = 0; i < len; i++) { 28 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 29 | } 30 | return result; 31 | } 32 | 33 | 34 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 35 | size_t len, std::string* dst) { 36 | int raw = static_cast(len * compressed_fraction); 37 | if (raw < 1) raw = 1; 38 | std::string raw_data; 39 | RandomString(rnd, raw, &raw_data); 40 | 41 | // Duplicate the random data until we have filled "len" bytes 42 | dst->clear(); 43 | while (dst->size() < len) { 44 | dst->append(raw_data); 45 | } 46 | dst->resize(len); 47 | return Slice(*dst); 48 | } 49 | 50 | } // namespace test 51 | } // namespace leveldb 52 | -------------------------------------------------------------------------------- /src/util/testutil.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 6 | #define STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 7 | 8 | #include "pebblesdb/env.h" 9 | #include "pebblesdb/slice.h" 10 | #include "util/random.h" 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | // Store in *dst a random string of length "len" and return a Slice that 16 | // references the generated data. 17 | extern Slice RandomString(Random* rnd, int len, std::string* dst); 18 | 19 | // Return a random key with the specified length that may contain interesting 20 | // characters (e.g. \x00, \xff, etc.). 21 | extern std::string RandomKey(Random* rnd, int len); 22 | 23 | // Store in *dst a string of length "len" that will compress to 24 | // "N*compressed_fraction" bytes and return a Slice that references 25 | // the generated data. 26 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 27 | size_t len, std::string* dst); 28 | 29 | // A wrapper that allows injection of errors. 30 | class ErrorEnv : public EnvWrapper { 31 | public: 32 | bool writable_file_error_; 33 | int num_writable_file_errors_; 34 | 35 | ErrorEnv() : EnvWrapper(Env::Default()), 36 | writable_file_error_(false), 37 | num_writable_file_errors_(0) { } 38 | 39 | virtual Status NewWritableFile(const std::string& fname, 40 | WritableFile** result) { 41 | if (writable_file_error_) { 42 | ++num_writable_file_errors_; 43 | *result = NULL; 44 | return Status::IOError(fname, "fake error"); 45 | } 46 | return target()->NewWritableFile(fname, result); 47 | } 48 | }; 49 | 50 | } // namespace test 51 | } // namespace leveldb 52 | 53 | #endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 54 | --------------------------------------------------------------------------------