├── .clang-format ├── .github └── workflows │ └── build.yml ├── .gitignore ├── .gitmodules ├── AUTHORS ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── NEWS ├── README.md ├── TODO ├── benchmarks ├── db_bench.cc ├── db_bench_log.cc ├── db_bench_sqlite3.cc └── db_bench_tree_db.cc ├── cmake └── leveldbConfig.cmake.in ├── db ├── autocompact_test.cc ├── builder.cc ├── builder.h ├── c.cc ├── c_test.c ├── corruption_test.cc ├── db_impl.cc ├── db_impl.h ├── db_iter.cc ├── db_iter.h ├── db_test.cc ├── dbformat.cc ├── dbformat.h ├── dbformat_test.cc ├── dumpfile.cc ├── fault_injection_test.cc ├── filename.cc ├── filename.h ├── filename_test.cc ├── leveldbutil.cc ├── log_format.h ├── log_reader.cc ├── log_reader.h ├── log_test.cc ├── log_writer.cc ├── log_writer.h ├── memtable.cc ├── memtable.h ├── recovery_test.cc ├── repair.cc ├── skiplist.h ├── skiplist_test.cc ├── snapshot.h ├── table_cache.cc ├── table_cache.h ├── version_edit.cc ├── version_edit.h ├── version_edit_test.cc ├── version_set.cc ├── version_set.h ├── version_set_test.cc ├── write_batch.cc ├── write_batch_internal.h └── write_batch_test.cc ├── doc ├── benchmark.html ├── impl.md ├── index.md ├── log_format.md └── table_format.md ├── helpers └── memenv │ ├── memenv.cc │ ├── memenv.h │ └── memenv_test.cc ├── include └── leveldb │ ├── c.h │ ├── cache.h │ ├── comparator.h │ ├── db.h │ ├── dumpfile.h │ ├── env.h │ ├── export.h │ ├── filter_policy.h │ ├── iterator.h │ ├── options.h │ ├── slice.h │ ├── status.h │ ├── table.h │ ├── table_builder.h │ └── write_batch.h ├── issues ├── issue178_test.cc ├── issue200_test.cc └── issue320_test.cc ├── port ├── README.md ├── port.h ├── port_config.h.in ├── port_example.h ├── port_stdcxx.h └── thread_annotations.h ├── table ├── block.cc ├── block.h ├── block_builder.cc ├── block_builder.h ├── filter_block.cc ├── filter_block.h ├── filter_block_test.cc ├── format.cc ├── format.h ├── iterator.cc ├── iterator_wrapper.h ├── merger.cc ├── merger.h ├── table.cc ├── table_builder.cc ├── table_test.cc ├── two_level_iterator.cc └── two_level_iterator.h └── util ├── arena.cc ├── arena.h ├── arena_test.cc ├── bloom.cc ├── bloom_test.cc ├── cache.cc ├── cache_test.cc ├── coding.cc ├── coding.h ├── coding_test.cc ├── comparator.cc ├── crc32c.cc ├── crc32c.h ├── crc32c_test.cc ├── env.cc ├── env_posix.cc ├── env_posix_test.cc ├── env_posix_test_helper.h ├── env_test.cc ├── env_windows.cc ├── env_windows_test.cc ├── env_windows_test_helper.h ├── filter_policy.cc ├── hash.cc ├── hash.h ├── hash_test.cc ├── histogram.cc ├── histogram.h ├── logging.cc ├── logging.h ├── logging_test.cc ├── mutexlock.h ├── no_destructor.h ├── no_destructor_test.cc ├── options.cc ├── posix_logger.h ├── random.h ├── status.cc ├── status_test.cc ├── testutil.cc ├── testutil.h └── windows_logger.h /.clang-format: -------------------------------------------------------------------------------- 1 | # Run manually to reformat a file: 2 | # clang-format -i --style=file 3 | # find . -iname '*.cc' -o -iname '*.h' -o -iname '*.h.in' | xargs clang-format -i --style=file 4 | BasedOnStyle: Google 5 | DerivePointerAlignment: false 6 | 7 | # Public headers are in a different location in the internal Google repository. 8 | # Order them so that when imported to the authoritative repository they will be 9 | # in correct alphabetical order. 10 | IncludeCategories: 11 | - Regex: '^(<|"(benchmarks|db|helpers)/)' 12 | Priority: 1 13 | - Regex: '^"(leveldb)/' 14 | Priority: 2 15 | - Regex: '^(<|"(issues|port|table|third_party|util)/)' 16 | Priority: 3 17 | - Regex: '.*' 18 | Priority: 4 19 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The LevelDB Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | name: ci 6 | on: [push, pull_request] 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | build-and-test: 13 | name: >- 14 | CI 15 | ${{ matrix.os }} 16 | ${{ matrix.compiler }} 17 | ${{ matrix.optimized && 'release' || 'debug' }} 18 | runs-on: ${{ matrix.os }} 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | compiler: [clang, gcc, msvc] 23 | os: [ubuntu-latest, macos-latest, windows-latest] 24 | optimized: [true, false] 25 | exclude: 26 | # MSVC only works on Windows. 27 | - os: ubuntu-latest 28 | compiler: msvc 29 | - os: macos-latest 30 | compiler: msvc 31 | # Not testing with GCC on macOS. 32 | - os: macos-latest 33 | compiler: gcc 34 | # Only testing with MSVC on Windows. 35 | - os: windows-latest 36 | compiler: clang 37 | - os: windows-latest 38 | compiler: gcc 39 | include: 40 | - compiler: clang 41 | CC: clang 42 | CXX: clang++ 43 | - compiler: gcc 44 | CC: gcc 45 | CXX: g++ 46 | - compiler: msvc 47 | CC: 48 | CXX: 49 | 50 | env: 51 | CMAKE_BUILD_DIR: ${{ github.workspace }}/build 52 | CMAKE_BUILD_TYPE: ${{ matrix.optimized && 'RelWithDebInfo' || 'Debug' }} 53 | CC: ${{ matrix.CC }} 54 | CXX: ${{ matrix.CXX }} 55 | BINARY_SUFFIX: ${{ startsWith(matrix.os, 'windows') && '.exe' || '' }} 56 | BINARY_PATH: >- 57 | ${{ format( 58 | startsWith(matrix.os, 'windows') && '{0}\build\{1}\' || '{0}/build/', 59 | github.workspace, 60 | matrix.optimized && 'RelWithDebInfo' || 'Debug') }} 61 | 62 | steps: 63 | - uses: actions/checkout@v2 64 | with: 65 | submodules: true 66 | 67 | - name: Install dependencies on Linux 68 | if: ${{ runner.os == 'Linux' }} 69 | # libgoogle-perftools-dev is temporarily removed from the package list 70 | # because it is currently broken on GitHub's Ubuntu 22.04. 71 | run: | 72 | sudo apt-get update 73 | sudo apt-get install libkyotocabinet-dev libsnappy-dev libsqlite3-dev 74 | 75 | - name: Generate build config 76 | run: >- 77 | cmake -S "${{ github.workspace }}" -B "${{ env.CMAKE_BUILD_DIR }}" 78 | -DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} 79 | -DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/install_test/ 80 | 81 | - name: Build 82 | run: >- 83 | cmake --build "${{ env.CMAKE_BUILD_DIR }}" 84 | --config "${{ env.CMAKE_BUILD_TYPE }}" 85 | 86 | - name: Run Tests 87 | working-directory: ${{ github.workspace }}/build 88 | run: ctest -C "${{ env.CMAKE_BUILD_TYPE }}" --verbose 89 | 90 | - name: Run LevelDB Benchmarks 91 | run: ${{ env.BINARY_PATH }}db_bench${{ env.BINARY_SUFFIX }} 92 | 93 | - name: Run SQLite Benchmarks 94 | if: ${{ runner.os != 'Windows' }} 95 | run: ${{ env.BINARY_PATH }}db_bench_sqlite3${{ env.BINARY_SUFFIX }} 96 | 97 | - name: Run Kyoto Cabinet Benchmarks 98 | if: ${{ runner.os == 'Linux' && matrix.compiler == 'clang' }} 99 | run: ${{ env.BINARY_PATH }}db_bench_tree_db${{ env.BINARY_SUFFIX }} 100 | 101 | - name: Test CMake installation 102 | run: cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target install 103 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Editors. 2 | *.sw* 3 | .vscode 4 | .DS_Store 5 | 6 | # Build directory. 7 | build/ 8 | out/ 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/googletest"] 2 | path = third_party/googletest 3 | url = https://github.com/google/googletest.git 4 | [submodule "third_party/benchmark"] 5 | path = third_party/benchmark 6 | url = https://github.com/google/benchmark 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | Johan Bilien 13 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code Reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | See [the README](README.md#contributing-to-the-leveldb-project) for areas 26 | where we are likely to accept external contributions. 27 | 28 | ## Community Guidelines 29 | 30 | This project follows [Google's Open Source Community 31 | Guidelines](https://opensource.google/conduct/). -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | Release 1.2 2011-05-16 2 | ---------------------- 3 | 4 | Fixes for larger databases (tested up to one billion 100-byte entries, 5 | i.e., ~100GB). 6 | 7 | (1) Place hard limit on number of level-0 files. This fixes errors 8 | of the form "too many open files". 9 | 10 | (2) Fixed memtable management. Before the fix, a heavy write burst 11 | could cause unbounded memory usage. 12 | 13 | A fix for a logging bug where the reader would incorrectly complain 14 | about corruption. 15 | 16 | Allow public access to WriteBatch contents so that users can easily 17 | wrap a DB. 18 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | ss 2 | - Stats 3 | 4 | db 5 | - Maybe implement DB::BulkDeleteForRange(start_key, end_key) 6 | that would blow away files whose ranges are entirely contained 7 | within [start_key..end_key]? For Chrome, deletion of obsolete 8 | object stores, etc. can be done in the background anyway, so 9 | probably not that important. 10 | - There have been requests for MultiGet. 11 | 12 | After a range is completely deleted, what gets rid of the 13 | corresponding files if we do no future changes to that range. Make 14 | the conditions for triggering compactions fire in more situations? 15 | -------------------------------------------------------------------------------- /benchmarks/db_bench_log.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "gtest/gtest.h" 10 | #include "benchmark/benchmark.h" 11 | #include "db/version_set.h" 12 | #include "leveldb/comparator.h" 13 | #include "leveldb/db.h" 14 | #include "leveldb/env.h" 15 | #include "leveldb/options.h" 16 | #include "port/port.h" 17 | #include "util/mutexlock.h" 18 | #include "util/testutil.h" 19 | 20 | namespace leveldb { 21 | 22 | namespace { 23 | 24 | std::string MakeKey(unsigned int num) { 25 | char buf[30]; 26 | std::snprintf(buf, sizeof(buf), "%016u", num); 27 | return std::string(buf); 28 | } 29 | 30 | void BM_LogAndApply(benchmark::State& state) { 31 | const int num_base_files = state.range(0); 32 | 33 | std::string dbname = testing::TempDir() + "leveldb_test_benchmark"; 34 | DestroyDB(dbname, Options()); 35 | 36 | DB* db = nullptr; 37 | Options opts; 38 | opts.create_if_missing = true; 39 | Status s = DB::Open(opts, dbname, &db); 40 | ASSERT_LEVELDB_OK(s); 41 | ASSERT_TRUE(db != nullptr); 42 | 43 | delete db; 44 | db = nullptr; 45 | 46 | Env* env = Env::Default(); 47 | 48 | port::Mutex mu; 49 | MutexLock l(&mu); 50 | 51 | InternalKeyComparator cmp(BytewiseComparator()); 52 | Options options; 53 | VersionSet vset(dbname, &options, nullptr, &cmp); 54 | bool save_manifest; 55 | ASSERT_LEVELDB_OK(vset.Recover(&save_manifest)); 56 | VersionEdit vbase; 57 | uint64_t fnum = 1; 58 | for (int i = 0; i < num_base_files; i++) { 59 | InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); 60 | InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); 61 | vbase.AddFile(2, fnum++, 1 /* file size */, start, limit); 62 | } 63 | ASSERT_LEVELDB_OK(vset.LogAndApply(&vbase, &mu)); 64 | 65 | uint64_t start_micros = env->NowMicros(); 66 | 67 | for (auto st : state) { 68 | VersionEdit vedit; 69 | vedit.RemoveFile(2, fnum); 70 | InternalKey start(MakeKey(2 * fnum), 1, kTypeValue); 71 | InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion); 72 | vedit.AddFile(2, fnum++, 1 /* file size */, start, limit); 73 | vset.LogAndApply(&vedit, &mu); 74 | } 75 | 76 | uint64_t stop_micros = env->NowMicros(); 77 | unsigned int us = stop_micros - start_micros; 78 | char buf[16]; 79 | std::snprintf(buf, sizeof(buf), "%d", num_base_files); 80 | std::fprintf(stderr, 81 | "BM_LogAndApply/%-6s %8" PRIu64 82 | " iters : %9u us (%7.0f us / iter)\n", 83 | buf, state.iterations(), us, ((float)us) / state.iterations()); 84 | } 85 | 86 | BENCHMARK(BM_LogAndApply)->Arg(1)->Arg(100)->Arg(10000)->Arg(100000); 87 | 88 | } // namespace 89 | 90 | } // namespace leveldb 91 | 92 | BENCHMARK_MAIN(); 93 | -------------------------------------------------------------------------------- /cmake/leveldbConfig.cmake.in: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The LevelDB Authors. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be 3 | # found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | @PACKAGE_INIT@ 6 | 7 | include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake") 8 | 9 | check_required_components(leveldb) -------------------------------------------------------------------------------- /db/autocompact_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "gtest/gtest.h" 6 | #include "db/db_impl.h" 7 | #include "leveldb/cache.h" 8 | #include "leveldb/db.h" 9 | #include "util/testutil.h" 10 | 11 | namespace leveldb { 12 | 13 | class AutoCompactTest : public testing::Test { 14 | public: 15 | AutoCompactTest() { 16 | dbname_ = testing::TempDir() + "autocompact_test"; 17 | tiny_cache_ = NewLRUCache(100); 18 | options_.block_cache = tiny_cache_; 19 | DestroyDB(dbname_, options_); 20 | options_.create_if_missing = true; 21 | options_.compression = kNoCompression; 22 | EXPECT_LEVELDB_OK(DB::Open(options_, dbname_, &db_)); 23 | } 24 | 25 | ~AutoCompactTest() { 26 | delete db_; 27 | DestroyDB(dbname_, Options()); 28 | delete tiny_cache_; 29 | } 30 | 31 | std::string Key(int i) { 32 | char buf[100]; 33 | std::snprintf(buf, sizeof(buf), "key%06d", i); 34 | return std::string(buf); 35 | } 36 | 37 | uint64_t Size(const Slice& start, const Slice& limit) { 38 | Range r(start, limit); 39 | uint64_t size; 40 | db_->GetApproximateSizes(&r, 1, &size); 41 | return size; 42 | } 43 | 44 | void DoReads(int n); 45 | 46 | private: 47 | std::string dbname_; 48 | Cache* tiny_cache_; 49 | Options options_; 50 | DB* db_; 51 | }; 52 | 53 | static const int kValueSize = 200 * 1024; 54 | static const int kTotalSize = 100 * 1024 * 1024; 55 | static const int kCount = kTotalSize / kValueSize; 56 | 57 | // Read through the first n keys repeatedly and check that they get 58 | // compacted (verified by checking the size of the key space). 59 | void AutoCompactTest::DoReads(int n) { 60 | std::string value(kValueSize, 'x'); 61 | DBImpl* dbi = reinterpret_cast(db_); 62 | 63 | // Fill database 64 | for (int i = 0; i < kCount; i++) { 65 | ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), Key(i), value)); 66 | } 67 | ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); 68 | 69 | // Delete everything 70 | for (int i = 0; i < kCount; i++) { 71 | ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), Key(i))); 72 | } 73 | ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); 74 | 75 | // Get initial measurement of the space we will be reading. 76 | const int64_t initial_size = Size(Key(0), Key(n)); 77 | const int64_t initial_other_size = Size(Key(n), Key(kCount)); 78 | 79 | // Read until size drops significantly. 80 | std::string limit_key = Key(n); 81 | for (int read = 0; true; read++) { 82 | ASSERT_LT(read, 100) << "Taking too long to compact"; 83 | Iterator* iter = db_->NewIterator(ReadOptions()); 84 | for (iter->SeekToFirst(); 85 | iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { 86 | // Drop data 87 | } 88 | delete iter; 89 | // Wait a little bit to allow any triggered compactions to complete. 90 | Env::Default()->SleepForMicroseconds(1000000); 91 | uint64_t size = Size(Key(0), Key(n)); 92 | std::fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, 93 | size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); 94 | if (size <= initial_size / 10) { 95 | break; 96 | } 97 | } 98 | 99 | // Verify that the size of the key space not touched by the reads 100 | // is pretty much unchanged. 101 | const int64_t final_other_size = Size(Key(n), Key(kCount)); 102 | ASSERT_LE(final_other_size, initial_other_size + 1048576); 103 | ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576); 104 | } 105 | 106 | TEST_F(AutoCompactTest, ReadAll) { DoReads(kCount); } 107 | 108 | TEST_F(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); } 109 | 110 | } // namespace leveldb 111 | -------------------------------------------------------------------------------- /db/builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/builder.h" 6 | 7 | #include "db/dbformat.h" 8 | #include "db/filename.h" 9 | #include "db/table_cache.h" 10 | #include "db/version_edit.h" 11 | #include "leveldb/db.h" 12 | #include "leveldb/env.h" 13 | #include "leveldb/iterator.h" 14 | 15 | namespace leveldb { 16 | 17 | Status BuildTable(const std::string& dbname, Env* env, const Options& options, 18 | TableCache* table_cache, Iterator* iter, FileMetaData* meta) { 19 | Status s; 20 | meta->file_size = 0; 21 | iter->SeekToFirst(); 22 | 23 | std::string fname = TableFileName(dbname, meta->number); 24 | if (iter->Valid()) { 25 | WritableFile* file; 26 | s = env->NewWritableFile(fname, &file); 27 | if (!s.ok()) { 28 | return s; 29 | } 30 | 31 | TableBuilder* builder = new TableBuilder(options, file); 32 | meta->smallest.DecodeFrom(iter->key()); 33 | Slice key; 34 | for (; iter->Valid(); iter->Next()) { 35 | key = iter->key(); 36 | builder->Add(key, iter->value()); 37 | } 38 | if (!key.empty()) { 39 | meta->largest.DecodeFrom(key); 40 | } 41 | 42 | // Finish and check for builder errors 43 | s = builder->Finish(); 44 | if (s.ok()) { 45 | meta->file_size = builder->FileSize(); 46 | assert(meta->file_size > 0); 47 | } 48 | delete builder; 49 | 50 | // Finish and check for file errors 51 | if (s.ok()) { 52 | s = file->Sync(); 53 | } 54 | if (s.ok()) { 55 | s = file->Close(); 56 | } 57 | delete file; 58 | file = nullptr; 59 | 60 | if (s.ok()) { 61 | // Verify that the table is usable 62 | Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number, 63 | meta->file_size); 64 | s = it->status(); 65 | delete it; 66 | } 67 | } 68 | 69 | // Check for input iterator errors 70 | if (!iter->status().ok()) { 71 | s = iter->status(); 72 | } 73 | 74 | if (s.ok() && meta->file_size > 0) { 75 | // Keep it 76 | } else { 77 | env->RemoveFile(fname); 78 | } 79 | return s; 80 | } 81 | 82 | } // namespace leveldb 83 | -------------------------------------------------------------------------------- /db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 6 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 7 | 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | 12 | struct Options; 13 | struct FileMetaData; 14 | 15 | class Env; 16 | class Iterator; 17 | class TableCache; 18 | class VersionEdit; 19 | 20 | // Build a Table file from the contents of *iter. The generated file 21 | // will be named according to meta->number. On success, the rest of 22 | // *meta will be filled with metadata about the generated table. 23 | // If no data is present in *iter, meta->file_size will be set to 24 | // zero, and no Table file will be produced. 25 | Status BuildTable(const std::string& dbname, Env* env, const Options& options, 26 | TableCache* table_cache, Iterator* iter, FileMetaData* meta); 27 | 28 | } // namespace leveldb 29 | 30 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 31 | -------------------------------------------------------------------------------- /db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 6 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 7 | 8 | #include 9 | 10 | #include "db/dbformat.h" 11 | #include "leveldb/db.h" 12 | 13 | namespace leveldb { 14 | 15 | class DBImpl; 16 | 17 | // Return a new iterator that converts internal keys (yielded by 18 | // "*internal_iter") that were live at the specified "sequence" number 19 | // into appropriate user keys. 20 | Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, 21 | Iterator* internal_iter, SequenceNumber sequence, 22 | uint32_t seed); 23 | 24 | } // namespace leveldb 25 | 26 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 27 | -------------------------------------------------------------------------------- /db/dbformat_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/dbformat.h" 6 | 7 | #include "gtest/gtest.h" 8 | #include "util/logging.h" 9 | 10 | namespace leveldb { 11 | 12 | static std::string IKey(const std::string& user_key, uint64_t seq, 13 | ValueType vt) { 14 | std::string encoded; 15 | AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); 16 | return encoded; 17 | } 18 | 19 | static std::string Shorten(const std::string& s, const std::string& l) { 20 | std::string result = s; 21 | InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); 22 | return result; 23 | } 24 | 25 | static std::string ShortSuccessor(const std::string& s) { 26 | std::string result = s; 27 | InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); 28 | return result; 29 | } 30 | 31 | static void TestKey(const std::string& key, uint64_t seq, ValueType vt) { 32 | std::string encoded = IKey(key, seq, vt); 33 | 34 | Slice in(encoded); 35 | ParsedInternalKey decoded("", 0, kTypeValue); 36 | 37 | ASSERT_TRUE(ParseInternalKey(in, &decoded)); 38 | ASSERT_EQ(key, decoded.user_key.ToString()); 39 | ASSERT_EQ(seq, decoded.sequence); 40 | ASSERT_EQ(vt, decoded.type); 41 | 42 | ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); 43 | } 44 | 45 | TEST(FormatTest, InternalKey_EncodeDecode) { 46 | const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; 47 | const uint64_t seq[] = {1, 48 | 2, 49 | 3, 50 | (1ull << 8) - 1, 51 | 1ull << 8, 52 | (1ull << 8) + 1, 53 | (1ull << 16) - 1, 54 | 1ull << 16, 55 | (1ull << 16) + 1, 56 | (1ull << 32) - 1, 57 | 1ull << 32, 58 | (1ull << 32) + 1}; 59 | for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { 60 | for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { 61 | TestKey(keys[k], seq[s], kTypeValue); 62 | TestKey("hello", 1, kTypeDeletion); 63 | } 64 | } 65 | } 66 | 67 | TEST(FormatTest, InternalKey_DecodeFromEmpty) { 68 | InternalKey internal_key; 69 | 70 | ASSERT_TRUE(!internal_key.DecodeFrom("")); 71 | } 72 | 73 | TEST(FormatTest, InternalKeyShortSeparator) { 74 | // When user keys are same 75 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 76 | Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue))); 77 | ASSERT_EQ( 78 | IKey("foo", 100, kTypeValue), 79 | Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue))); 80 | ASSERT_EQ( 81 | IKey("foo", 100, kTypeValue), 82 | Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue))); 83 | ASSERT_EQ( 84 | IKey("foo", 100, kTypeValue), 85 | Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion))); 86 | 87 | // When user keys are misordered 88 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 89 | Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue))); 90 | 91 | // When user keys are different, but correctly ordered 92 | ASSERT_EQ( 93 | IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 94 | Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue))); 95 | 96 | // When start user key is prefix of limit user key 97 | ASSERT_EQ( 98 | IKey("foo", 100, kTypeValue), 99 | Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue))); 100 | 101 | // When limit user key is prefix of start user key 102 | ASSERT_EQ( 103 | IKey("foobar", 100, kTypeValue), 104 | Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue))); 105 | } 106 | 107 | TEST(FormatTest, InternalKeyShortestSuccessor) { 108 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 109 | ShortSuccessor(IKey("foo", 100, kTypeValue))); 110 | ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), 111 | ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); 112 | } 113 | 114 | TEST(FormatTest, ParsedInternalKeyDebugString) { 115 | ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); 116 | 117 | ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); 118 | } 119 | 120 | TEST(FormatTest, InternalKeyDebugString) { 121 | InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); 122 | ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); 123 | 124 | InternalKey invalid_key; 125 | ASSERT_EQ("(bad)", invalid_key.DebugString()); 126 | } 127 | 128 | } // namespace leveldb 129 | -------------------------------------------------------------------------------- /db/filename.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/filename.h" 6 | 7 | #include 8 | #include 9 | 10 | #include "db/dbformat.h" 11 | #include "leveldb/env.h" 12 | #include "util/logging.h" 13 | 14 | namespace leveldb { 15 | 16 | // A utility routine: write "data" to the named file and Sync() it. 17 | Status WriteStringToFileSync(Env* env, const Slice& data, 18 | const std::string& fname); 19 | 20 | static std::string MakeFileName(const std::string& dbname, uint64_t number, 21 | const char* suffix) { 22 | char buf[100]; 23 | std::snprintf(buf, sizeof(buf), "/%06llu.%s", 24 | static_cast(number), suffix); 25 | return dbname + buf; 26 | } 27 | 28 | std::string LogFileName(const std::string& dbname, uint64_t number) { 29 | assert(number > 0); 30 | return MakeFileName(dbname, number, "log"); 31 | } 32 | 33 | std::string TableFileName(const std::string& dbname, uint64_t number) { 34 | assert(number > 0); 35 | return MakeFileName(dbname, number, "ldb"); 36 | } 37 | 38 | std::string SSTTableFileName(const std::string& dbname, uint64_t number) { 39 | assert(number > 0); 40 | return MakeFileName(dbname, number, "sst"); 41 | } 42 | 43 | std::string DescriptorFileName(const std::string& dbname, uint64_t number) { 44 | assert(number > 0); 45 | char buf[100]; 46 | std::snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", 47 | static_cast(number)); 48 | return dbname + buf; 49 | } 50 | 51 | std::string CurrentFileName(const std::string& dbname) { 52 | return dbname + "/CURRENT"; 53 | } 54 | 55 | std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; } 56 | 57 | std::string TempFileName(const std::string& dbname, uint64_t number) { 58 | assert(number > 0); 59 | return MakeFileName(dbname, number, "dbtmp"); 60 | } 61 | 62 | std::string InfoLogFileName(const std::string& dbname) { 63 | return dbname + "/LOG"; 64 | } 65 | 66 | // Return the name of the old info log file for "dbname". 67 | std::string OldInfoLogFileName(const std::string& dbname) { 68 | return dbname + "/LOG.old"; 69 | } 70 | 71 | // Owned filenames have the form: 72 | // dbname/CURRENT 73 | // dbname/LOCK 74 | // dbname/LOG 75 | // dbname/LOG.old 76 | // dbname/MANIFEST-[0-9]+ 77 | // dbname/[0-9]+.(log|sst|ldb) 78 | bool ParseFileName(const std::string& filename, uint64_t* number, 79 | FileType* type) { 80 | Slice rest(filename); 81 | if (rest == "CURRENT") { 82 | *number = 0; 83 | *type = kCurrentFile; 84 | } else if (rest == "LOCK") { 85 | *number = 0; 86 | *type = kDBLockFile; 87 | } else if (rest == "LOG" || rest == "LOG.old") { 88 | *number = 0; 89 | *type = kInfoLogFile; 90 | } else if (rest.starts_with("MANIFEST-")) { 91 | rest.remove_prefix(strlen("MANIFEST-")); 92 | uint64_t num; 93 | if (!ConsumeDecimalNumber(&rest, &num)) { 94 | return false; 95 | } 96 | if (!rest.empty()) { 97 | return false; 98 | } 99 | *type = kDescriptorFile; 100 | *number = num; 101 | } else { 102 | // Avoid strtoull() to keep filename format independent of the 103 | // current locale 104 | uint64_t num; 105 | if (!ConsumeDecimalNumber(&rest, &num)) { 106 | return false; 107 | } 108 | Slice suffix = rest; 109 | if (suffix == Slice(".log")) { 110 | *type = kLogFile; 111 | } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) { 112 | *type = kTableFile; 113 | } else if (suffix == Slice(".dbtmp")) { 114 | *type = kTempFile; 115 | } else { 116 | return false; 117 | } 118 | *number = num; 119 | } 120 | return true; 121 | } 122 | 123 | Status SetCurrentFile(Env* env, const std::string& dbname, 124 | uint64_t descriptor_number) { 125 | // Remove leading "dbname/" and add newline to manifest file name 126 | std::string manifest = DescriptorFileName(dbname, descriptor_number); 127 | Slice contents = manifest; 128 | assert(contents.starts_with(dbname + "/")); 129 | contents.remove_prefix(dbname.size() + 1); 130 | std::string tmp = TempFileName(dbname, descriptor_number); 131 | Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp); 132 | if (s.ok()) { 133 | s = env->RenameFile(tmp, CurrentFileName(dbname)); 134 | } 135 | if (!s.ok()) { 136 | env->RemoveFile(tmp); 137 | } 138 | return s; 139 | } 140 | 141 | } // namespace leveldb 142 | -------------------------------------------------------------------------------- /db/filename.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // File names used by DB code 6 | 7 | #ifndef STORAGE_LEVELDB_DB_FILENAME_H_ 8 | #define STORAGE_LEVELDB_DB_FILENAME_H_ 9 | 10 | #include 11 | #include 12 | 13 | #include "leveldb/slice.h" 14 | #include "leveldb/status.h" 15 | #include "port/port.h" 16 | 17 | namespace leveldb { 18 | 19 | class Env; 20 | 21 | enum FileType { 22 | kLogFile, 23 | kDBLockFile, 24 | kTableFile, 25 | kDescriptorFile, 26 | kCurrentFile, 27 | kTempFile, 28 | kInfoLogFile // Either the current one, or an old one 29 | }; 30 | 31 | // Return the name of the log file with the specified number 32 | // in the db named by "dbname". The result will be prefixed with 33 | // "dbname". 34 | std::string LogFileName(const std::string& dbname, uint64_t number); 35 | 36 | // Return the name of the sstable with the specified number 37 | // in the db named by "dbname". The result will be prefixed with 38 | // "dbname". 39 | std::string TableFileName(const std::string& dbname, uint64_t number); 40 | 41 | // Return the legacy file name for an sstable with the specified number 42 | // in the db named by "dbname". The result will be prefixed with 43 | // "dbname". 44 | std::string SSTTableFileName(const std::string& dbname, uint64_t number); 45 | 46 | // Return the name of the descriptor file for the db named by 47 | // "dbname" and the specified incarnation number. The result will be 48 | // prefixed with "dbname". 49 | std::string DescriptorFileName(const std::string& dbname, uint64_t number); 50 | 51 | // Return the name of the current file. This file contains the name 52 | // of the current manifest file. The result will be prefixed with 53 | // "dbname". 54 | std::string CurrentFileName(const std::string& dbname); 55 | 56 | // Return the name of the lock file for the db named by 57 | // "dbname". The result will be prefixed with "dbname". 58 | std::string LockFileName(const std::string& dbname); 59 | 60 | // Return the name of a temporary file owned by the db named "dbname". 61 | // The result will be prefixed with "dbname". 62 | std::string TempFileName(const std::string& dbname, uint64_t number); 63 | 64 | // Return the name of the info log file for "dbname". 65 | std::string InfoLogFileName(const std::string& dbname); 66 | 67 | // Return the name of the old info log file for "dbname". 68 | std::string OldInfoLogFileName(const std::string& dbname); 69 | 70 | // If filename is a leveldb file, store the type of the file in *type. 71 | // The number encoded in the filename is stored in *number. If the 72 | // filename was successfully parsed, returns true. Else return false. 73 | bool ParseFileName(const std::string& filename, uint64_t* number, 74 | FileType* type); 75 | 76 | // Make the CURRENT file point to the descriptor file with the 77 | // specified number. 78 | Status SetCurrentFile(Env* env, const std::string& dbname, 79 | uint64_t descriptor_number); 80 | 81 | } // namespace leveldb 82 | 83 | #endif // STORAGE_LEVELDB_DB_FILENAME_H_ 84 | -------------------------------------------------------------------------------- /db/filename_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/filename.h" 6 | 7 | #include "gtest/gtest.h" 8 | #include "db/dbformat.h" 9 | #include "port/port.h" 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | 14 | TEST(FileNameTest, Parse) { 15 | Slice db; 16 | FileType type; 17 | uint64_t number; 18 | 19 | // Successful parses 20 | static struct { 21 | const char* fname; 22 | uint64_t number; 23 | FileType type; 24 | } cases[] = { 25 | {"100.log", 100, kLogFile}, 26 | {"0.log", 0, kLogFile}, 27 | {"0.sst", 0, kTableFile}, 28 | {"0.ldb", 0, kTableFile}, 29 | {"CURRENT", 0, kCurrentFile}, 30 | {"LOCK", 0, kDBLockFile}, 31 | {"MANIFEST-2", 2, kDescriptorFile}, 32 | {"MANIFEST-7", 7, kDescriptorFile}, 33 | {"LOG", 0, kInfoLogFile}, 34 | {"LOG.old", 0, kInfoLogFile}, 35 | {"18446744073709551615.log", 18446744073709551615ull, kLogFile}, 36 | }; 37 | for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { 38 | std::string f = cases[i].fname; 39 | ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; 40 | ASSERT_EQ(cases[i].type, type) << f; 41 | ASSERT_EQ(cases[i].number, number) << f; 42 | } 43 | 44 | // Errors 45 | static const char* errors[] = {"", 46 | "foo", 47 | "foo-dx-100.log", 48 | ".log", 49 | "", 50 | "manifest", 51 | "CURREN", 52 | "CURRENTX", 53 | "MANIFES", 54 | "MANIFEST", 55 | "MANIFEST-", 56 | "XMANIFEST-3", 57 | "MANIFEST-3x", 58 | "LOC", 59 | "LOCKx", 60 | "LO", 61 | "LOGx", 62 | "18446744073709551616.log", 63 | "184467440737095516150.log", 64 | "100", 65 | "100.", 66 | "100.lop"}; 67 | for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { 68 | std::string f = errors[i]; 69 | ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; 70 | } 71 | } 72 | 73 | TEST(FileNameTest, Construction) { 74 | uint64_t number; 75 | FileType type; 76 | std::string fname; 77 | 78 | fname = CurrentFileName("foo"); 79 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 80 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 81 | ASSERT_EQ(0, number); 82 | ASSERT_EQ(kCurrentFile, type); 83 | 84 | fname = LockFileName("foo"); 85 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 86 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 87 | ASSERT_EQ(0, number); 88 | ASSERT_EQ(kDBLockFile, type); 89 | 90 | fname = LogFileName("foo", 192); 91 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 92 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 93 | ASSERT_EQ(192, number); 94 | ASSERT_EQ(kLogFile, type); 95 | 96 | fname = TableFileName("bar", 200); 97 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 98 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 99 | ASSERT_EQ(200, number); 100 | ASSERT_EQ(kTableFile, type); 101 | 102 | fname = DescriptorFileName("bar", 100); 103 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 104 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 105 | ASSERT_EQ(100, number); 106 | ASSERT_EQ(kDescriptorFile, type); 107 | 108 | fname = TempFileName("tmp", 999); 109 | ASSERT_EQ("tmp/", std::string(fname.data(), 4)); 110 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 111 | ASSERT_EQ(999, number); 112 | ASSERT_EQ(kTempFile, type); 113 | 114 | fname = InfoLogFileName("foo"); 115 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 116 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 117 | ASSERT_EQ(0, number); 118 | ASSERT_EQ(kInfoLogFile, type); 119 | 120 | fname = OldInfoLogFileName("foo"); 121 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 122 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 123 | ASSERT_EQ(0, number); 124 | ASSERT_EQ(kInfoLogFile, type); 125 | } 126 | 127 | } // namespace leveldb 128 | -------------------------------------------------------------------------------- /db/leveldbutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | 7 | #include "leveldb/dumpfile.h" 8 | #include "leveldb/env.h" 9 | #include "leveldb/status.h" 10 | 11 | namespace leveldb { 12 | namespace { 13 | 14 | class StdoutPrinter : public WritableFile { 15 | public: 16 | Status Append(const Slice& data) override { 17 | fwrite(data.data(), 1, data.size(), stdout); 18 | return Status::OK(); 19 | } 20 | Status Close() override { return Status::OK(); } 21 | Status Flush() override { return Status::OK(); } 22 | Status Sync() override { return Status::OK(); } 23 | }; 24 | 25 | bool HandleDumpCommand(Env* env, char** files, int num) { 26 | StdoutPrinter printer; 27 | bool ok = true; 28 | for (int i = 0; i < num; i++) { 29 | Status s = DumpFile(env, files[i], &printer); 30 | if (!s.ok()) { 31 | std::fprintf(stderr, "%s\n", s.ToString().c_str()); 32 | ok = false; 33 | } 34 | } 35 | return ok; 36 | } 37 | 38 | } // namespace 39 | } // namespace leveldb 40 | 41 | static void Usage() { 42 | std::fprintf( 43 | stderr, 44 | "Usage: leveldbutil command...\n" 45 | " dump files... -- dump contents of specified files\n"); 46 | } 47 | 48 | int main(int argc, char** argv) { 49 | leveldb::Env* env = leveldb::Env::Default(); 50 | bool ok = true; 51 | if (argc < 2) { 52 | Usage(); 53 | ok = false; 54 | } else { 55 | std::string command = argv[1]; 56 | if (command == "dump") { 57 | ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2); 58 | } else { 59 | Usage(); 60 | ok = false; 61 | } 62 | } 63 | return (ok ? 0 : 1); 64 | } 65 | -------------------------------------------------------------------------------- /db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Log format information shared by reader and writer. 6 | // See ../doc/log_format.md for more detail. 7 | 8 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 9 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 10 | 11 | namespace leveldb { 12 | namespace log { 13 | 14 | enum RecordType { 15 | // Zero is reserved for preallocated files 16 | kZeroType = 0, 17 | 18 | kFullType = 1, 19 | 20 | // For fragments 21 | kFirstType = 2, 22 | kMiddleType = 3, 23 | kLastType = 4 24 | }; 25 | static const int kMaxRecordType = kLastType; 26 | 27 | static const int kBlockSize = 32768; 28 | 29 | // Header is checksum (4 bytes), length (2 bytes), type (1 byte). 30 | static const int kHeaderSize = 4 + 2 + 1; 31 | 32 | } // namespace log 33 | } // namespace leveldb 34 | 35 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 36 | -------------------------------------------------------------------------------- /db/log_reader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_READER_H_ 7 | 8 | #include 9 | 10 | #include "db/log_format.h" 11 | #include "leveldb/slice.h" 12 | #include "leveldb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | class SequentialFile; 17 | 18 | namespace log { 19 | 20 | class Reader { 21 | public: 22 | // Interface for reporting errors. 23 | class Reporter { 24 | public: 25 | virtual ~Reporter(); 26 | 27 | // Some corruption was detected. "bytes" is the approximate number 28 | // of bytes dropped due to the corruption. 29 | virtual void Corruption(size_t bytes, const Status& status) = 0; 30 | }; 31 | 32 | // Create a reader that will return log records from "*file". 33 | // "*file" must remain live while this Reader is in use. 34 | // 35 | // If "reporter" is non-null, it is notified whenever some data is 36 | // dropped due to a detected corruption. "*reporter" must remain 37 | // live while this Reader is in use. 38 | // 39 | // If "checksum" is true, verify checksums if available. 40 | // 41 | // The Reader will start reading at the first record located at physical 42 | // position >= initial_offset within the file. 43 | Reader(SequentialFile* file, Reporter* reporter, bool checksum, 44 | uint64_t initial_offset); 45 | 46 | Reader(const Reader&) = delete; 47 | Reader& operator=(const Reader&) = delete; 48 | 49 | ~Reader(); 50 | 51 | // Read the next record into *record. Returns true if read 52 | // successfully, false if we hit end of the input. May use 53 | // "*scratch" as temporary storage. The contents filled in *record 54 | // will only be valid until the next mutating operation on this 55 | // reader or the next mutation to *scratch. 56 | bool ReadRecord(Slice* record, std::string* scratch); 57 | 58 | // Returns the physical offset of the last record returned by ReadRecord. 59 | // 60 | // Undefined before the first call to ReadRecord. 61 | uint64_t LastRecordOffset(); 62 | 63 | private: 64 | // Extend record types with the following special values 65 | enum { 66 | kEof = kMaxRecordType + 1, 67 | // Returned whenever we find an invalid physical record. 68 | // Currently there are three situations in which this happens: 69 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 70 | // * The record is a 0-length record (No drop is reported) 71 | // * The record is below constructor's initial_offset (No drop is reported) 72 | kBadRecord = kMaxRecordType + 2 73 | }; 74 | 75 | // Skips all blocks that are completely before "initial_offset_". 76 | // 77 | // Returns true on success. Handles reporting. 78 | bool SkipToInitialBlock(); 79 | 80 | // Return type, or one of the preceding special values 81 | unsigned int ReadPhysicalRecord(Slice* result); 82 | 83 | // Reports dropped bytes to the reporter. 84 | // buffer_ must be updated to remove the dropped bytes prior to invocation. 85 | void ReportCorruption(uint64_t bytes, const char* reason); 86 | void ReportDrop(uint64_t bytes, const Status& reason); 87 | 88 | SequentialFile* const file_; 89 | Reporter* const reporter_; 90 | bool const checksum_; 91 | char* const backing_store_; 92 | Slice buffer_; 93 | bool eof_; // Last Read() indicated EOF by returning < kBlockSize 94 | 95 | // Offset of the last record returned by ReadRecord. 96 | uint64_t last_record_offset_; 97 | // Offset of the first location past the end of buffer_. 98 | uint64_t end_of_buffer_offset_; 99 | 100 | // Offset at which to start looking for the first record to return 101 | uint64_t const initial_offset_; 102 | 103 | // True if we are resynchronizing after a seek (initial_offset_ > 0). In 104 | // particular, a run of kMiddleType and kLastType records can be silently 105 | // skipped in this mode 106 | bool resyncing_; 107 | }; 108 | 109 | } // namespace log 110 | } // namespace leveldb 111 | 112 | #endif // STORAGE_LEVELDB_DB_LOG_READER_H_ 113 | -------------------------------------------------------------------------------- /db/log_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/log_writer.h" 6 | 7 | #include 8 | 9 | #include "leveldb/env.h" 10 | #include "util/coding.h" 11 | #include "util/crc32c.h" 12 | 13 | namespace leveldb { 14 | namespace log { 15 | 16 | static void InitTypeCrc(uint32_t* type_crc) { 17 | for (int i = 0; i <= kMaxRecordType; i++) { 18 | char t = static_cast(i); 19 | type_crc[i] = crc32c::Value(&t, 1); 20 | } 21 | } 22 | 23 | Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) { 24 | InitTypeCrc(type_crc_); 25 | } 26 | 27 | Writer::Writer(WritableFile* dest, uint64_t dest_length) 28 | : dest_(dest), block_offset_(dest_length % kBlockSize) { 29 | InitTypeCrc(type_crc_); 30 | } 31 | 32 | Writer::~Writer() = default; 33 | 34 | Status Writer::AddRecord(const Slice& slice) { 35 | const char* ptr = slice.data(); 36 | size_t left = slice.size(); 37 | 38 | // Fragment the record if necessary and emit it. Note that if slice 39 | // is empty, we still want to iterate once to emit a single 40 | // zero-length record 41 | Status s; 42 | bool begin = true; 43 | do { 44 | const int leftover = kBlockSize - block_offset_; 45 | assert(leftover >= 0); 46 | if (leftover < kHeaderSize) { 47 | // Switch to a new block 48 | if (leftover > 0) { 49 | // Fill the trailer (literal below relies on kHeaderSize being 7) 50 | static_assert(kHeaderSize == 7, ""); 51 | dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); 52 | } 53 | block_offset_ = 0; 54 | } 55 | 56 | // Invariant: we never leave < kHeaderSize bytes in a block. 57 | assert(kBlockSize - block_offset_ - kHeaderSize >= 0); 58 | 59 | const size_t avail = kBlockSize - block_offset_ - kHeaderSize; 60 | const size_t fragment_length = (left < avail) ? left : avail; 61 | 62 | RecordType type; 63 | const bool end = (left == fragment_length); 64 | if (begin && end) { 65 | type = kFullType; 66 | } else if (begin) { 67 | type = kFirstType; 68 | } else if (end) { 69 | type = kLastType; 70 | } else { 71 | type = kMiddleType; 72 | } 73 | 74 | s = EmitPhysicalRecord(type, ptr, fragment_length); 75 | ptr += fragment_length; 76 | left -= fragment_length; 77 | begin = false; 78 | } while (s.ok() && left > 0); 79 | return s; 80 | } 81 | 82 | Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, 83 | size_t length) { 84 | assert(length <= 0xffff); // Must fit in two bytes 85 | assert(block_offset_ + kHeaderSize + length <= kBlockSize); 86 | 87 | // Format the header 88 | char buf[kHeaderSize]; 89 | buf[4] = static_cast(length & 0xff); 90 | buf[5] = static_cast(length >> 8); 91 | buf[6] = static_cast(t); 92 | 93 | // Compute the crc of the record type and the payload. 94 | uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length); 95 | crc = crc32c::Mask(crc); // Adjust for storage 96 | EncodeFixed32(buf, crc); 97 | 98 | // Write the header and the payload 99 | Status s = dest_->Append(Slice(buf, kHeaderSize)); 100 | if (s.ok()) { 101 | s = dest_->Append(Slice(ptr, length)); 102 | if (s.ok()) { 103 | s = dest_->Flush(); 104 | } 105 | } 106 | block_offset_ += kHeaderSize + length; 107 | return s; 108 | } 109 | 110 | } // namespace log 111 | } // namespace leveldb 112 | -------------------------------------------------------------------------------- /db/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ 7 | 8 | #include 9 | 10 | #include "db/log_format.h" 11 | #include "leveldb/slice.h" 12 | #include "leveldb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | class WritableFile; 17 | 18 | namespace log { 19 | 20 | class Writer { 21 | public: 22 | // Create a writer that will append data to "*dest". 23 | // "*dest" must be initially empty. 24 | // "*dest" must remain live while this Writer is in use. 25 | explicit Writer(WritableFile* dest); 26 | 27 | // Create a writer that will append data to "*dest". 28 | // "*dest" must have initial length "dest_length". 29 | // "*dest" must remain live while this Writer is in use. 30 | Writer(WritableFile* dest, uint64_t dest_length); 31 | 32 | Writer(const Writer&) = delete; 33 | Writer& operator=(const Writer&) = delete; 34 | 35 | ~Writer(); 36 | 37 | Status AddRecord(const Slice& slice); 38 | 39 | private: 40 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 41 | 42 | WritableFile* dest_; 43 | int block_offset_; // Current offset in block 44 | 45 | // crc32c values for all supported record types. These are 46 | // pre-computed to reduce the overhead of computing the crc of the 47 | // record type stored in the header. 48 | uint32_t type_crc_[kMaxRecordType + 1]; 49 | }; 50 | 51 | } // namespace log 52 | } // namespace leveldb 53 | 54 | #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ 55 | -------------------------------------------------------------------------------- /db/memtable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_ 6 | #define STORAGE_LEVELDB_DB_MEMTABLE_H_ 7 | 8 | #include 9 | 10 | #include "db/dbformat.h" 11 | #include "db/skiplist.h" 12 | #include "leveldb/db.h" 13 | #include "util/arena.h" 14 | 15 | namespace leveldb { 16 | 17 | class InternalKeyComparator; 18 | class MemTableIterator; 19 | 20 | class MemTable { 21 | public: 22 | // MemTables are reference counted. The initial reference count 23 | // is zero and the caller must call Ref() at least once. 24 | explicit MemTable(const InternalKeyComparator& comparator); 25 | 26 | MemTable(const MemTable&) = delete; 27 | MemTable& operator=(const MemTable&) = delete; 28 | 29 | // Increase reference count. 30 | void Ref() { ++refs_; } 31 | 32 | // Drop reference count. Delete if no more references exist. 33 | void Unref() { 34 | --refs_; 35 | assert(refs_ >= 0); 36 | if (refs_ <= 0) { 37 | delete this; 38 | } 39 | } 40 | 41 | // Returns an estimate of the number of bytes of data in use by this 42 | // data structure. It is safe to call when MemTable is being modified. 43 | size_t ApproximateMemoryUsage(); 44 | 45 | // Return an iterator that yields the contents of the memtable. 46 | // 47 | // The caller must ensure that the underlying MemTable remains live 48 | // while the returned iterator is live. The keys returned by this 49 | // iterator are internal keys encoded by AppendInternalKey in the 50 | // db/format.{h,cc} module. 51 | Iterator* NewIterator(); 52 | 53 | // Add an entry into memtable that maps key to value at the 54 | // specified sequence number and with the specified type. 55 | // Typically value will be empty if type==kTypeDeletion. 56 | void Add(SequenceNumber seq, ValueType type, const Slice& key, 57 | const Slice& value); 58 | 59 | // If memtable contains a value for key, store it in *value and return true. 60 | // If memtable contains a deletion for key, store a NotFound() error 61 | // in *status and return true. 62 | // Else, return false. 63 | bool Get(const LookupKey& key, std::string* value, Status* s); 64 | 65 | private: 66 | friend class MemTableIterator; 67 | friend class MemTableBackwardIterator; 68 | 69 | struct KeyComparator { 70 | const InternalKeyComparator comparator; 71 | explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} 72 | int operator()(const char* a, const char* b) const; 73 | }; 74 | 75 | typedef SkipList Table; 76 | 77 | ~MemTable(); // Private since only Unref() should be used to delete it 78 | 79 | KeyComparator comparator_; 80 | int refs_; 81 | Arena arena_; 82 | Table table_; 83 | }; 84 | 85 | } // namespace leveldb 86 | 87 | #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_ 88 | -------------------------------------------------------------------------------- /db/snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ 6 | #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/db.h" 10 | 11 | namespace leveldb { 12 | 13 | class SnapshotList; 14 | 15 | // Snapshots are kept in a doubly-linked list in the DB. 16 | // Each SnapshotImpl corresponds to a particular sequence number. 17 | class SnapshotImpl : public Snapshot { 18 | public: 19 | SnapshotImpl(SequenceNumber sequence_number) 20 | : sequence_number_(sequence_number) {} 21 | 22 | SequenceNumber sequence_number() const { return sequence_number_; } 23 | 24 | private: 25 | friend class SnapshotList; 26 | 27 | // SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList 28 | // implementation operates on the next/previous fields directly. 29 | SnapshotImpl* prev_; 30 | SnapshotImpl* next_; 31 | 32 | const SequenceNumber sequence_number_; 33 | 34 | #if !defined(NDEBUG) 35 | SnapshotList* list_ = nullptr; 36 | #endif // !defined(NDEBUG) 37 | }; 38 | 39 | class SnapshotList { 40 | public: 41 | SnapshotList() : head_(0) { 42 | head_.prev_ = &head_; 43 | head_.next_ = &head_; 44 | } 45 | 46 | bool empty() const { return head_.next_ == &head_; } 47 | SnapshotImpl* oldest() const { 48 | assert(!empty()); 49 | return head_.next_; 50 | } 51 | SnapshotImpl* newest() const { 52 | assert(!empty()); 53 | return head_.prev_; 54 | } 55 | 56 | // Creates a SnapshotImpl and appends it to the end of the list. 57 | SnapshotImpl* New(SequenceNumber sequence_number) { 58 | assert(empty() || newest()->sequence_number_ <= sequence_number); 59 | 60 | SnapshotImpl* snapshot = new SnapshotImpl(sequence_number); 61 | 62 | #if !defined(NDEBUG) 63 | snapshot->list_ = this; 64 | #endif // !defined(NDEBUG) 65 | snapshot->next_ = &head_; 66 | snapshot->prev_ = head_.prev_; 67 | snapshot->prev_->next_ = snapshot; 68 | snapshot->next_->prev_ = snapshot; 69 | return snapshot; 70 | } 71 | 72 | // Removes a SnapshotImpl from this list. 73 | // 74 | // The snapshot must have been created by calling New() on this list. 75 | // 76 | // The snapshot pointer should not be const, because its memory is 77 | // deallocated. However, that would force us to change DB::ReleaseSnapshot(), 78 | // which is in the API, and currently takes a const Snapshot. 79 | void Delete(const SnapshotImpl* snapshot) { 80 | #if !defined(NDEBUG) 81 | assert(snapshot->list_ == this); 82 | #endif // !defined(NDEBUG) 83 | snapshot->prev_->next_ = snapshot->next_; 84 | snapshot->next_->prev_ = snapshot->prev_; 85 | delete snapshot; 86 | } 87 | 88 | private: 89 | // Dummy head of doubly-linked list of snapshots 90 | SnapshotImpl head_; 91 | }; 92 | 93 | } // namespace leveldb 94 | 95 | #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ 96 | -------------------------------------------------------------------------------- /db/table_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/table_cache.h" 6 | 7 | #include "db/filename.h" 8 | #include "leveldb/env.h" 9 | #include "leveldb/table.h" 10 | #include "util/coding.h" 11 | 12 | namespace leveldb { 13 | 14 | struct TableAndFile { 15 | RandomAccessFile* file; 16 | Table* table; 17 | }; 18 | 19 | static void DeleteEntry(const Slice& key, void* value) { 20 | TableAndFile* tf = reinterpret_cast(value); 21 | delete tf->table; 22 | delete tf->file; 23 | delete tf; 24 | } 25 | 26 | static void UnrefEntry(void* arg1, void* arg2) { 27 | Cache* cache = reinterpret_cast(arg1); 28 | Cache::Handle* h = reinterpret_cast(arg2); 29 | cache->Release(h); 30 | } 31 | 32 | TableCache::TableCache(const std::string& dbname, const Options& options, 33 | int entries) 34 | : env_(options.env), 35 | dbname_(dbname), 36 | options_(options), 37 | cache_(NewLRUCache(entries)) {} 38 | 39 | TableCache::~TableCache() { delete cache_; } 40 | 41 | Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, 42 | Cache::Handle** handle) { 43 | Status s; 44 | char buf[sizeof(file_number)]; 45 | EncodeFixed64(buf, file_number); 46 | Slice key(buf, sizeof(buf)); 47 | *handle = cache_->Lookup(key); 48 | if (*handle == nullptr) { 49 | std::string fname = TableFileName(dbname_, file_number); 50 | RandomAccessFile* file = nullptr; 51 | Table* table = nullptr; 52 | s = env_->NewRandomAccessFile(fname, &file); 53 | if (!s.ok()) { 54 | std::string old_fname = SSTTableFileName(dbname_, file_number); 55 | if (env_->NewRandomAccessFile(old_fname, &file).ok()) { 56 | s = Status::OK(); 57 | } 58 | } 59 | if (s.ok()) { 60 | s = Table::Open(options_, file, file_size, &table); 61 | } 62 | 63 | if (!s.ok()) { 64 | assert(table == nullptr); 65 | delete file; 66 | // We do not cache error results so that if the error is transient, 67 | // or somebody repairs the file, we recover automatically. 68 | } else { 69 | TableAndFile* tf = new TableAndFile; 70 | tf->file = file; 71 | tf->table = table; 72 | *handle = cache_->Insert(key, tf, 1, &DeleteEntry); 73 | } 74 | } 75 | return s; 76 | } 77 | 78 | Iterator* TableCache::NewIterator(const ReadOptions& options, 79 | uint64_t file_number, uint64_t file_size, 80 | Table** tableptr) { 81 | if (tableptr != nullptr) { 82 | *tableptr = nullptr; 83 | } 84 | 85 | Cache::Handle* handle = nullptr; 86 | Status s = FindTable(file_number, file_size, &handle); 87 | if (!s.ok()) { 88 | return NewErrorIterator(s); 89 | } 90 | 91 | Table* table = reinterpret_cast(cache_->Value(handle))->table; 92 | Iterator* result = table->NewIterator(options); 93 | result->RegisterCleanup(&UnrefEntry, cache_, handle); 94 | if (tableptr != nullptr) { 95 | *tableptr = table; 96 | } 97 | return result; 98 | } 99 | 100 | Status TableCache::Get(const ReadOptions& options, uint64_t file_number, 101 | uint64_t file_size, const Slice& k, void* arg, 102 | void (*handle_result)(void*, const Slice&, 103 | const Slice&)) { 104 | Cache::Handle* handle = nullptr; 105 | Status s = FindTable(file_number, file_size, &handle); 106 | if (s.ok()) { 107 | Table* t = reinterpret_cast(cache_->Value(handle))->table; 108 | s = t->InternalGet(options, k, arg, handle_result); 109 | cache_->Release(handle); 110 | } 111 | return s; 112 | } 113 | 114 | void TableCache::Evict(uint64_t file_number) { 115 | char buf[sizeof(file_number)]; 116 | EncodeFixed64(buf, file_number); 117 | cache_->Erase(Slice(buf, sizeof(buf))); 118 | } 119 | 120 | } // namespace leveldb 121 | -------------------------------------------------------------------------------- /db/table_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Thread-safe (provides internal synchronization) 6 | 7 | #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 8 | #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 9 | 10 | #include 11 | #include 12 | 13 | #include "db/dbformat.h" 14 | #include "leveldb/cache.h" 15 | #include "leveldb/table.h" 16 | #include "port/port.h" 17 | 18 | namespace leveldb { 19 | 20 | class Env; 21 | 22 | class TableCache { 23 | public: 24 | TableCache(const std::string& dbname, const Options& options, int entries); 25 | 26 | TableCache(const TableCache&) = delete; 27 | TableCache& operator=(const TableCache&) = delete; 28 | 29 | ~TableCache(); 30 | 31 | // Return an iterator for the specified file number (the corresponding 32 | // file length must be exactly "file_size" bytes). If "tableptr" is 33 | // non-null, also sets "*tableptr" to point to the Table object 34 | // underlying the returned iterator, or to nullptr if no Table object 35 | // underlies the returned iterator. The returned "*tableptr" object is owned 36 | // by the cache and should not be deleted, and is valid for as long as the 37 | // returned iterator is live. 38 | Iterator* NewIterator(const ReadOptions& options, uint64_t file_number, 39 | uint64_t file_size, Table** tableptr = nullptr); 40 | 41 | // If a seek to internal key "k" in specified file finds an entry, 42 | // call (*handle_result)(arg, found_key, found_value). 43 | Status Get(const ReadOptions& options, uint64_t file_number, 44 | uint64_t file_size, const Slice& k, void* arg, 45 | void (*handle_result)(void*, const Slice&, const Slice&)); 46 | 47 | // Evict any entry for the specified file number 48 | void Evict(uint64_t file_number); 49 | 50 | private: 51 | Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); 52 | 53 | Env* const env_; 54 | const std::string dbname_; 55 | const Options& options_; 56 | Cache* cache_; 57 | }; 58 | 59 | } // namespace leveldb 60 | 61 | #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 62 | -------------------------------------------------------------------------------- /db/version_edit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 6 | #define STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "db/dbformat.h" 13 | 14 | namespace leveldb { 15 | 16 | class VersionSet; 17 | 18 | struct FileMetaData { 19 | FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {} 20 | 21 | int refs; 22 | int allowed_seeks; // Seeks allowed until compaction 23 | uint64_t number; 24 | uint64_t file_size; // File size in bytes 25 | InternalKey smallest; // Smallest internal key served by table 26 | InternalKey largest; // Largest internal key served by table 27 | }; 28 | 29 | class VersionEdit { 30 | public: 31 | VersionEdit() { Clear(); } 32 | ~VersionEdit() = default; 33 | 34 | void Clear(); 35 | 36 | void SetComparatorName(const Slice& name) { 37 | has_comparator_ = true; 38 | comparator_ = name.ToString(); 39 | } 40 | void SetLogNumber(uint64_t num) { 41 | has_log_number_ = true; 42 | log_number_ = num; 43 | } 44 | void SetPrevLogNumber(uint64_t num) { 45 | has_prev_log_number_ = true; 46 | prev_log_number_ = num; 47 | } 48 | void SetNextFile(uint64_t num) { 49 | has_next_file_number_ = true; 50 | next_file_number_ = num; 51 | } 52 | void SetLastSequence(SequenceNumber seq) { 53 | has_last_sequence_ = true; 54 | last_sequence_ = seq; 55 | } 56 | void SetCompactPointer(int level, const InternalKey& key) { 57 | compact_pointers_.push_back(std::make_pair(level, key)); 58 | } 59 | 60 | // Add the specified file at the specified number. 61 | // REQUIRES: This version has not been saved (see VersionSet::SaveTo) 62 | // REQUIRES: "smallest" and "largest" are smallest and largest keys in file 63 | void AddFile(int level, uint64_t file, uint64_t file_size, 64 | const InternalKey& smallest, const InternalKey& largest) { 65 | FileMetaData f; 66 | f.number = file; 67 | f.file_size = file_size; 68 | f.smallest = smallest; 69 | f.largest = largest; 70 | new_files_.push_back(std::make_pair(level, f)); 71 | } 72 | 73 | // Delete the specified "file" from the specified "level". 74 | void RemoveFile(int level, uint64_t file) { 75 | deleted_files_.insert(std::make_pair(level, file)); 76 | } 77 | 78 | void EncodeTo(std::string* dst) const; 79 | Status DecodeFrom(const Slice& src); 80 | 81 | std::string DebugString() const; 82 | 83 | private: 84 | friend class VersionSet; 85 | 86 | typedef std::set> DeletedFileSet; 87 | 88 | std::string comparator_; 89 | uint64_t log_number_; 90 | uint64_t prev_log_number_; 91 | uint64_t next_file_number_; 92 | SequenceNumber last_sequence_; 93 | bool has_comparator_; 94 | bool has_log_number_; 95 | bool has_prev_log_number_; 96 | bool has_next_file_number_; 97 | bool has_last_sequence_; 98 | 99 | std::vector> compact_pointers_; 100 | DeletedFileSet deleted_files_; 101 | std::vector> new_files_; 102 | }; 103 | 104 | } // namespace leveldb 105 | 106 | #endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 107 | -------------------------------------------------------------------------------- /db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/version_edit.h" 6 | 7 | #include "gtest/gtest.h" 8 | 9 | namespace leveldb { 10 | 11 | static void TestEncodeDecode(const VersionEdit& edit) { 12 | std::string encoded, encoded2; 13 | edit.EncodeTo(&encoded); 14 | VersionEdit parsed; 15 | Status s = parsed.DecodeFrom(encoded); 16 | ASSERT_TRUE(s.ok()) << s.ToString(); 17 | parsed.EncodeTo(&encoded2); 18 | ASSERT_EQ(encoded, encoded2); 19 | } 20 | 21 | TEST(VersionEditTest, EncodeDecode) { 22 | static const uint64_t kBig = 1ull << 50; 23 | 24 | VersionEdit edit; 25 | for (int i = 0; i < 4; i++) { 26 | TestEncodeDecode(edit); 27 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 28 | InternalKey("foo", kBig + 500 + i, kTypeValue), 29 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 30 | edit.RemoveFile(4, kBig + 700 + i); 31 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 32 | } 33 | 34 | edit.SetComparatorName("foo"); 35 | edit.SetLogNumber(kBig + 100); 36 | edit.SetNextFile(kBig + 200); 37 | edit.SetLastSequence(kBig + 1000); 38 | TestEncodeDecode(edit); 39 | } 40 | 41 | } // namespace leveldb 42 | -------------------------------------------------------------------------------- /db/write_batch.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch::rep_ := 6 | // sequence: fixed64 7 | // count: fixed32 8 | // data: record[count] 9 | // record := 10 | // kTypeValue varstring varstring | 11 | // kTypeDeletion varstring 12 | // varstring := 13 | // len: varint32 14 | // data: uint8[len] 15 | 16 | #include "leveldb/write_batch.h" 17 | 18 | #include "db/dbformat.h" 19 | #include "db/memtable.h" 20 | #include "db/write_batch_internal.h" 21 | #include "leveldb/db.h" 22 | #include "util/coding.h" 23 | 24 | namespace leveldb { 25 | 26 | // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. 27 | static const size_t kHeader = 12; 28 | 29 | WriteBatch::WriteBatch() { Clear(); } 30 | 31 | WriteBatch::~WriteBatch() = default; 32 | 33 | WriteBatch::Handler::~Handler() = default; 34 | 35 | void WriteBatch::Clear() { 36 | rep_.clear(); 37 | rep_.resize(kHeader); 38 | } 39 | 40 | size_t WriteBatch::ApproximateSize() const { return rep_.size(); } 41 | 42 | Status WriteBatch::Iterate(Handler* handler) const { 43 | Slice input(rep_); 44 | if (input.size() < kHeader) { 45 | return Status::Corruption("malformed WriteBatch (too small)"); 46 | } 47 | 48 | input.remove_prefix(kHeader); 49 | Slice key, value; 50 | int found = 0; 51 | while (!input.empty()) { 52 | found++; 53 | char tag = input[0]; 54 | input.remove_prefix(1); 55 | switch (tag) { 56 | case kTypeValue: 57 | if (GetLengthPrefixedSlice(&input, &key) && 58 | GetLengthPrefixedSlice(&input, &value)) { 59 | handler->Put(key, value); 60 | } else { 61 | return Status::Corruption("bad WriteBatch Put"); 62 | } 63 | break; 64 | case kTypeDeletion: 65 | if (GetLengthPrefixedSlice(&input, &key)) { 66 | handler->Delete(key); 67 | } else { 68 | return Status::Corruption("bad WriteBatch Delete"); 69 | } 70 | break; 71 | default: 72 | return Status::Corruption("unknown WriteBatch tag"); 73 | } 74 | } 75 | if (found != WriteBatchInternal::Count(this)) { 76 | return Status::Corruption("WriteBatch has wrong count"); 77 | } else { 78 | return Status::OK(); 79 | } 80 | } 81 | 82 | int WriteBatchInternal::Count(const WriteBatch* b) { 83 | return DecodeFixed32(b->rep_.data() + 8); 84 | } 85 | 86 | void WriteBatchInternal::SetCount(WriteBatch* b, int n) { 87 | EncodeFixed32(&b->rep_[8], n); 88 | } 89 | 90 | SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { 91 | return SequenceNumber(DecodeFixed64(b->rep_.data())); 92 | } 93 | 94 | void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { 95 | EncodeFixed64(&b->rep_[0], seq); 96 | } 97 | 98 | void WriteBatch::Put(const Slice& key, const Slice& value) { 99 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 100 | rep_.push_back(static_cast(kTypeValue)); 101 | PutLengthPrefixedSlice(&rep_, key); 102 | PutLengthPrefixedSlice(&rep_, value); 103 | } 104 | 105 | void WriteBatch::Delete(const Slice& key) { 106 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 107 | rep_.push_back(static_cast(kTypeDeletion)); 108 | PutLengthPrefixedSlice(&rep_, key); 109 | } 110 | 111 | void WriteBatch::Append(const WriteBatch& source) { 112 | WriteBatchInternal::Append(this, &source); 113 | } 114 | 115 | namespace { 116 | class MemTableInserter : public WriteBatch::Handler { 117 | public: 118 | SequenceNumber sequence_; 119 | MemTable* mem_; 120 | 121 | void Put(const Slice& key, const Slice& value) override { 122 | mem_->Add(sequence_, kTypeValue, key, value); 123 | sequence_++; 124 | } 125 | void Delete(const Slice& key) override { 126 | mem_->Add(sequence_, kTypeDeletion, key, Slice()); 127 | sequence_++; 128 | } 129 | }; 130 | } // namespace 131 | 132 | Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) { 133 | MemTableInserter inserter; 134 | inserter.sequence_ = WriteBatchInternal::Sequence(b); 135 | inserter.mem_ = memtable; 136 | return b->Iterate(&inserter); 137 | } 138 | 139 | void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { 140 | assert(contents.size() >= kHeader); 141 | b->rep_.assign(contents.data(), contents.size()); 142 | } 143 | 144 | void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) { 145 | SetCount(dst, Count(dst) + Count(src)); 146 | assert(src->rep_.size() >= kHeader); 147 | dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader); 148 | } 149 | 150 | } // namespace leveldb 151 | -------------------------------------------------------------------------------- /db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/write_batch.h" 10 | 11 | namespace leveldb { 12 | 13 | class MemTable; 14 | 15 | // WriteBatchInternal provides static methods for manipulating a 16 | // WriteBatch that we don't want in the public WriteBatch interface. 17 | class WriteBatchInternal { 18 | public: 19 | // Return the number of entries in the batch. 20 | static int Count(const WriteBatch* batch); 21 | 22 | // Set the count for the number of entries in the batch. 23 | static void SetCount(WriteBatch* batch, int n); 24 | 25 | // Return the sequence number for the start of this batch. 26 | static SequenceNumber Sequence(const WriteBatch* batch); 27 | 28 | // Store the specified number as the sequence number for the start of 29 | // this batch. 30 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 31 | 32 | static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } 33 | 34 | static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } 35 | 36 | static void SetContents(WriteBatch* batch, const Slice& contents); 37 | 38 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 39 | 40 | static void Append(WriteBatch* dst, const WriteBatch* src); 41 | }; 42 | 43 | } // namespace leveldb 44 | 45 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 46 | -------------------------------------------------------------------------------- /db/write_batch_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "gtest/gtest.h" 6 | #include "db/memtable.h" 7 | #include "db/write_batch_internal.h" 8 | #include "leveldb/db.h" 9 | #include "leveldb/env.h" 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | 14 | static std::string PrintContents(WriteBatch* b) { 15 | InternalKeyComparator cmp(BytewiseComparator()); 16 | MemTable* mem = new MemTable(cmp); 17 | mem->Ref(); 18 | std::string state; 19 | Status s = WriteBatchInternal::InsertInto(b, mem); 20 | int count = 0; 21 | Iterator* iter = mem->NewIterator(); 22 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 23 | ParsedInternalKey ikey; 24 | EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey)); 25 | switch (ikey.type) { 26 | case kTypeValue: 27 | state.append("Put("); 28 | state.append(ikey.user_key.ToString()); 29 | state.append(", "); 30 | state.append(iter->value().ToString()); 31 | state.append(")"); 32 | count++; 33 | break; 34 | case kTypeDeletion: 35 | state.append("Delete("); 36 | state.append(ikey.user_key.ToString()); 37 | state.append(")"); 38 | count++; 39 | break; 40 | } 41 | state.append("@"); 42 | state.append(NumberToString(ikey.sequence)); 43 | } 44 | delete iter; 45 | if (!s.ok()) { 46 | state.append("ParseError()"); 47 | } else if (count != WriteBatchInternal::Count(b)) { 48 | state.append("CountMismatch()"); 49 | } 50 | mem->Unref(); 51 | return state; 52 | } 53 | 54 | TEST(WriteBatchTest, Empty) { 55 | WriteBatch batch; 56 | ASSERT_EQ("", PrintContents(&batch)); 57 | ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); 58 | } 59 | 60 | TEST(WriteBatchTest, Multiple) { 61 | WriteBatch batch; 62 | batch.Put(Slice("foo"), Slice("bar")); 63 | batch.Delete(Slice("box")); 64 | batch.Put(Slice("baz"), Slice("boo")); 65 | WriteBatchInternal::SetSequence(&batch, 100); 66 | ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); 67 | ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); 68 | ASSERT_EQ( 69 | "Put(baz, boo)@102" 70 | "Delete(box)@101" 71 | "Put(foo, bar)@100", 72 | PrintContents(&batch)); 73 | } 74 | 75 | TEST(WriteBatchTest, Corruption) { 76 | WriteBatch batch; 77 | batch.Put(Slice("foo"), Slice("bar")); 78 | batch.Delete(Slice("box")); 79 | WriteBatchInternal::SetSequence(&batch, 200); 80 | Slice contents = WriteBatchInternal::Contents(&batch); 81 | WriteBatchInternal::SetContents(&batch, 82 | Slice(contents.data(), contents.size() - 1)); 83 | ASSERT_EQ( 84 | "Put(foo, bar)@200" 85 | "ParseError()", 86 | PrintContents(&batch)); 87 | } 88 | 89 | TEST(WriteBatchTest, Append) { 90 | WriteBatch b1, b2; 91 | WriteBatchInternal::SetSequence(&b1, 200); 92 | WriteBatchInternal::SetSequence(&b2, 300); 93 | b1.Append(b2); 94 | ASSERT_EQ("", PrintContents(&b1)); 95 | b2.Put("a", "va"); 96 | b1.Append(b2); 97 | ASSERT_EQ("Put(a, va)@200", PrintContents(&b1)); 98 | b2.Clear(); 99 | b2.Put("b", "vb"); 100 | b1.Append(b2); 101 | ASSERT_EQ( 102 | "Put(a, va)@200" 103 | "Put(b, vb)@201", 104 | PrintContents(&b1)); 105 | b2.Delete("foo"); 106 | b1.Append(b2); 107 | ASSERT_EQ( 108 | "Put(a, va)@200" 109 | "Put(b, vb)@202" 110 | "Put(b, vb)@201" 111 | "Delete(foo)@203", 112 | PrintContents(&b1)); 113 | } 114 | 115 | TEST(WriteBatchTest, ApproximateSize) { 116 | WriteBatch batch; 117 | size_t empty_size = batch.ApproximateSize(); 118 | 119 | batch.Put(Slice("foo"), Slice("bar")); 120 | size_t one_key_size = batch.ApproximateSize(); 121 | ASSERT_LT(empty_size, one_key_size); 122 | 123 | batch.Put(Slice("baz"), Slice("boo")); 124 | size_t two_keys_size = batch.ApproximateSize(); 125 | ASSERT_LT(one_key_size, two_keys_size); 126 | 127 | batch.Delete(Slice("box")); 128 | size_t post_delete_size = batch.ApproximateSize(); 129 | ASSERT_LT(two_keys_size, post_delete_size); 130 | } 131 | 132 | } // namespace leveldb 133 | -------------------------------------------------------------------------------- /doc/log_format.md: -------------------------------------------------------------------------------- 1 | leveldb Log format 2 | ================== 3 | The log file contents are a sequence of 32KB blocks. The only exception is that 4 | the tail of the file may contain a partial block. 5 | 6 | Each block consists of a sequence of records: 7 | 8 | block := record* trailer? 9 | record := 10 | checksum: uint32 // crc32c of type and data[] ; little-endian 11 | length: uint16 // little-endian 12 | type: uint8 // One of FULL, FIRST, MIDDLE, LAST 13 | data: uint8[length] 14 | 15 | A record never starts within the last six bytes of a block (since it won't fit). 16 | Any leftover bytes here form the trailer, which must consist entirely of zero 17 | bytes and must be skipped by readers. 18 | 19 | Aside: if exactly seven bytes are left in the current block, and a new non-zero 20 | length record is added, the writer must emit a FIRST record (which contains zero 21 | bytes of user data) to fill up the trailing seven bytes of the block and then 22 | emit all of the user data in subsequent blocks. 23 | 24 | More types may be added in the future. Some Readers may skip record types they 25 | do not understand, others may report that some data was skipped. 26 | 27 | FULL == 1 28 | FIRST == 2 29 | MIDDLE == 3 30 | LAST == 4 31 | 32 | The FULL record contains the contents of an entire user record. 33 | 34 | FIRST, MIDDLE, LAST are types used for user records that have been split into 35 | multiple fragments (typically because of block boundaries). FIRST is the type 36 | of the first fragment of a user record, LAST is the type of the last fragment of 37 | a user record, and MIDDLE is the type of all interior fragments of a user 38 | record. 39 | 40 | Example: consider a sequence of user records: 41 | 42 | A: length 1000 43 | B: length 97270 44 | C: length 8000 45 | 46 | **A** will be stored as a FULL record in the first block. 47 | 48 | **B** will be split into three fragments: first fragment occupies the rest of 49 | the first block, second fragment occupies the entirety of the second block, and 50 | the third fragment occupies a prefix of the third block. This will leave six 51 | bytes free in the third block, which will be left empty as the trailer. 52 | 53 | **C** will be stored as a FULL record in the fourth block. 54 | 55 | ---- 56 | 57 | ## Some benefits over the recordio format: 58 | 59 | 1. We do not need any heuristics for resyncing - just go to next block boundary 60 | and scan. If there is a corruption, skip to the next block. As a 61 | side-benefit, we do not get confused when part of the contents of one log 62 | file are embedded as a record inside another log file. 63 | 64 | 2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the 65 | next block boundary and skip records until we hit a FULL or FIRST record. 66 | 67 | 3. We do not need extra buffering for large records. 68 | 69 | ## Some downsides compared to recordio format: 70 | 71 | 1. No packing of tiny records. This could be fixed by adding a new record type, 72 | so it is a shortcoming of the current implementation, not necessarily the 73 | format. 74 | 75 | 2. No compression. Again, this could be fixed by adding new record types. 76 | -------------------------------------------------------------------------------- /doc/table_format.md: -------------------------------------------------------------------------------- 1 | leveldb File format 2 | =================== 3 | 4 | 5 | [data block 1] 6 | [data block 2] 7 | ... 8 | [data block N] 9 | [meta block 1] 10 | ... 11 | [meta block K] 12 | [metaindex block] 13 | [index block] 14 | [Footer] (fixed size; starts at file_size - sizeof(Footer)) 15 | 16 | 17 | The file contains internal pointers. Each such pointer is called 18 | a BlockHandle and contains the following information: 19 | 20 | offset: varint64 21 | size: varint64 22 | 23 | See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints) 24 | for an explanation of varint64 format. 25 | 26 | 1. The sequence of key/value pairs in the file are stored in sorted 27 | order and partitioned into a sequence of data blocks. These blocks 28 | come one after another at the beginning of the file. Each data block 29 | is formatted according to the code in `block_builder.cc`, and then 30 | optionally compressed. 31 | 32 | 2. After the data blocks we store a bunch of meta blocks. The 33 | supported meta block types are described below. More meta block types 34 | may be added in the future. Each meta block is again formatted using 35 | `block_builder.cc` and then optionally compressed. 36 | 37 | 3. A "metaindex" block. It contains one entry for every other meta 38 | block where the key is the name of the meta block and the value is a 39 | BlockHandle pointing to that meta block. 40 | 41 | 4. An "index" block. This block contains one entry per data block, 42 | where the key is a string >= last key in that data block and before 43 | the first key in the successive data block. The value is the 44 | BlockHandle for the data block. 45 | 46 | 5. At the very end of the file is a fixed length footer that contains 47 | the BlockHandle of the metaindex and index blocks as well as a magic number. 48 | 49 | metaindex_handle: char[p]; // Block handle for metaindex 50 | index_handle: char[q]; // Block handle for index 51 | padding: char[40-p-q];// zeroed bytes to make fixed length 52 | // (40==2*BlockHandle::kMaxEncodedLength) 53 | magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) 54 | 55 | ## "filter" Meta Block 56 | 57 | If a `FilterPolicy` was specified when the database was opened, a 58 | filter block is stored in each table. The "metaindex" block contains 59 | an entry that maps from `filter.` to the BlockHandle for the filter 60 | block where `` is the string returned by the filter policy's 61 | `Name()` method. 62 | 63 | The filter block stores a sequence of filters, where filter i contains 64 | the output of `FilterPolicy::CreateFilter()` on all keys that are stored 65 | in a block whose file offset falls within the range 66 | 67 | [ i*base ... (i+1)*base-1 ] 68 | 69 | Currently, "base" is 2KB. So for example, if blocks X and Y start in 70 | the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be 71 | converted to a filter by calling `FilterPolicy::CreateFilter()`, and the 72 | resulting filter will be stored as the first filter in the filter 73 | block. 74 | 75 | The filter block is formatted as follows: 76 | 77 | [filter 0] 78 | [filter 1] 79 | [filter 2] 80 | ... 81 | [filter N-1] 82 | 83 | [offset of filter 0] : 4 bytes 84 | [offset of filter 1] : 4 bytes 85 | [offset of filter 2] : 4 bytes 86 | ... 87 | [offset of filter N-1] : 4 bytes 88 | 89 | [offset of beginning of offset array] : 4 bytes 90 | lg(base) : 1 byte 91 | 92 | The offset array at the end of the filter block allows efficient 93 | mapping from a data block offset to the corresponding filter. 94 | 95 | ## "stats" Meta Block 96 | 97 | This meta block contains a bunch of stats. The key is the name 98 | of the statistic. The value contains the statistic. 99 | 100 | TODO(postrelease): record following stats. 101 | 102 | data size 103 | index size 104 | key size (uncompressed) 105 | value size (uncompressed) 106 | number of entries 107 | number of data blocks 108 | -------------------------------------------------------------------------------- /helpers/memenv/memenv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 6 | #define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 7 | 8 | #include "leveldb/export.h" 9 | 10 | namespace leveldb { 11 | 12 | class Env; 13 | 14 | // Returns a new environment that stores its data in memory and delegates 15 | // all non-file-storage tasks to base_env. The caller must delete the result 16 | // when it is no longer needed. 17 | // *base_env must remain live while the result is in use. 18 | LEVELDB_EXPORT Env* NewMemEnv(Env* base_env); 19 | 20 | } // namespace leveldb 21 | 22 | #endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 23 | -------------------------------------------------------------------------------- /include/leveldb/cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Cache is an interface that maps keys to values. It has internal 6 | // synchronization and may be safely accessed concurrently from 7 | // multiple threads. It may automatically evict entries to make room 8 | // for new entries. Values have a specified charge against the cache 9 | // capacity. For example, a cache where the values are variable 10 | // length strings, may use the length of the string as the charge for 11 | // the string. 12 | // 13 | // A builtin cache implementation with a least-recently-used eviction 14 | // policy is provided. Clients may use their own implementations if 15 | // they want something more sophisticated (like scan-resistance, a 16 | // custom eviction policy, variable cache sizing, etc.) 17 | 18 | #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ 19 | #define STORAGE_LEVELDB_INCLUDE_CACHE_H_ 20 | 21 | #include 22 | 23 | #include "leveldb/export.h" 24 | #include "leveldb/slice.h" 25 | 26 | namespace leveldb { 27 | 28 | class LEVELDB_EXPORT Cache; 29 | 30 | // Create a new cache with a fixed size capacity. This implementation 31 | // of Cache uses a least-recently-used eviction policy. 32 | LEVELDB_EXPORT Cache* NewLRUCache(size_t capacity); 33 | 34 | class LEVELDB_EXPORT Cache { 35 | public: 36 | Cache() = default; 37 | 38 | Cache(const Cache&) = delete; 39 | Cache& operator=(const Cache&) = delete; 40 | 41 | // Destroys all existing entries by calling the "deleter" 42 | // function that was passed to the constructor. 43 | virtual ~Cache(); 44 | 45 | // Opaque handle to an entry stored in the cache. 46 | struct Handle {}; 47 | 48 | // Insert a mapping from key->value into the cache and assign it 49 | // the specified charge against the total cache capacity. 50 | // 51 | // Returns a handle that corresponds to the mapping. The caller 52 | // must call this->Release(handle) when the returned mapping is no 53 | // longer needed. 54 | // 55 | // When the inserted entry is no longer needed, the key and 56 | // value will be passed to "deleter". 57 | virtual Handle* Insert(const Slice& key, void* value, size_t charge, 58 | void (*deleter)(const Slice& key, void* value)) = 0; 59 | 60 | // If the cache has no mapping for "key", returns nullptr. 61 | // 62 | // Else return a handle that corresponds to the mapping. The caller 63 | // must call this->Release(handle) when the returned mapping is no 64 | // longer needed. 65 | virtual Handle* Lookup(const Slice& key) = 0; 66 | 67 | // Release a mapping returned by a previous Lookup(). 68 | // REQUIRES: handle must not have been released yet. 69 | // REQUIRES: handle must have been returned by a method on *this. 70 | virtual void Release(Handle* handle) = 0; 71 | 72 | // Return the value encapsulated in a handle returned by a 73 | // successful Lookup(). 74 | // REQUIRES: handle must not have been released yet. 75 | // REQUIRES: handle must have been returned by a method on *this. 76 | virtual void* Value(Handle* handle) = 0; 77 | 78 | // If the cache contains entry for key, erase it. Note that the 79 | // underlying entry will be kept around until all existing handles 80 | // to it have been released. 81 | virtual void Erase(const Slice& key) = 0; 82 | 83 | // Return a new numeric id. May be used by multiple clients who are 84 | // sharing the same cache to partition the key space. Typically the 85 | // client will allocate a new id at startup and prepend the id to 86 | // its cache keys. 87 | virtual uint64_t NewId() = 0; 88 | 89 | // Remove all cache entries that are not actively in use. Memory-constrained 90 | // applications may wish to call this method to reduce memory usage. 91 | // Default implementation of Prune() does nothing. Subclasses are strongly 92 | // encouraged to override the default implementation. A future release of 93 | // leveldb may change Prune() to a pure abstract method. 94 | virtual void Prune() {} 95 | 96 | // Return an estimate of the combined charges of all elements stored in the 97 | // cache. 98 | virtual size_t TotalCharge() const = 0; 99 | }; 100 | 101 | } // namespace leveldb 102 | 103 | #endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_ 104 | -------------------------------------------------------------------------------- /include/leveldb/comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 7 | 8 | #include 9 | 10 | #include "leveldb/export.h" 11 | 12 | namespace leveldb { 13 | 14 | class Slice; 15 | 16 | // A Comparator object provides a total order across slices that are 17 | // used as keys in an sstable or a database. A Comparator implementation 18 | // must be thread-safe since leveldb may invoke its methods concurrently 19 | // from multiple threads. 20 | class LEVELDB_EXPORT Comparator { 21 | public: 22 | virtual ~Comparator(); 23 | 24 | // Three-way comparison. Returns value: 25 | // < 0 iff "a" < "b", 26 | // == 0 iff "a" == "b", 27 | // > 0 iff "a" > "b" 28 | virtual int Compare(const Slice& a, const Slice& b) const = 0; 29 | 30 | // The name of the comparator. Used to check for comparator 31 | // mismatches (i.e., a DB created with one comparator is 32 | // accessed using a different comparator. 33 | // 34 | // The client of this package should switch to a new name whenever 35 | // the comparator implementation changes in a way that will cause 36 | // the relative ordering of any two keys to change. 37 | // 38 | // Names starting with "leveldb." are reserved and should not be used 39 | // by any clients of this package. 40 | virtual const char* Name() const = 0; 41 | 42 | // Advanced functions: these are used to reduce the space requirements 43 | // for internal data structures like index blocks. 44 | 45 | // If *start < limit, changes *start to a short string in [start,limit). 46 | // Simple comparator implementations may return with *start unchanged, 47 | // i.e., an implementation of this method that does nothing is correct. 48 | virtual void FindShortestSeparator(std::string* start, 49 | const Slice& limit) const = 0; 50 | 51 | // Changes *key to a short string >= *key. 52 | // Simple comparator implementations may return with *key unchanged, 53 | // i.e., an implementation of this method that does nothing is correct. 54 | virtual void FindShortSuccessor(std::string* key) const = 0; 55 | }; 56 | 57 | // Return a builtin comparator that uses lexicographic byte-wise 58 | // ordering. The result remains the property of this module and 59 | // must not be deleted. 60 | LEVELDB_EXPORT const Comparator* BytewiseComparator(); 61 | 62 | } // namespace leveldb 63 | 64 | #endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 65 | -------------------------------------------------------------------------------- /include/leveldb/dumpfile.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 7 | 8 | #include 9 | 10 | #include "leveldb/env.h" 11 | #include "leveldb/export.h" 12 | #include "leveldb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | // Dump the contents of the file named by fname in text format to 17 | // *dst. Makes a sequence of dst->Append() calls; each call is passed 18 | // the newline-terminated text corresponding to a single item found 19 | // in the file. 20 | // 21 | // Returns a non-OK result if fname does not name a leveldb storage 22 | // file, or if the file cannot be read. 23 | LEVELDB_EXPORT Status DumpFile(Env* env, const std::string& fname, 24 | WritableFile* dst); 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 29 | -------------------------------------------------------------------------------- /include/leveldb/export.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 7 | 8 | #if !defined(LEVELDB_EXPORT) 9 | 10 | #if defined(LEVELDB_SHARED_LIBRARY) 11 | #if defined(_WIN32) 12 | 13 | #if defined(LEVELDB_COMPILE_LIBRARY) 14 | #define LEVELDB_EXPORT __declspec(dllexport) 15 | #else 16 | #define LEVELDB_EXPORT __declspec(dllimport) 17 | #endif // defined(LEVELDB_COMPILE_LIBRARY) 18 | 19 | #else // defined(_WIN32) 20 | #if defined(LEVELDB_COMPILE_LIBRARY) 21 | #define LEVELDB_EXPORT __attribute__((visibility("default"))) 22 | #else 23 | #define LEVELDB_EXPORT 24 | #endif 25 | #endif // defined(_WIN32) 26 | 27 | #else // defined(LEVELDB_SHARED_LIBRARY) 28 | #define LEVELDB_EXPORT 29 | #endif 30 | 31 | #endif // !defined(LEVELDB_EXPORT) 32 | 33 | #endif // STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 34 | -------------------------------------------------------------------------------- /include/leveldb/filter_policy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A database can be configured with a custom FilterPolicy object. 6 | // This object is responsible for creating a small filter from a set 7 | // of keys. These filters are stored in leveldb and are consulted 8 | // automatically by leveldb to decide whether or not to read some 9 | // information from disk. In many cases, a filter can cut down the 10 | // number of disk seeks form a handful to a single disk seek per 11 | // DB::Get() call. 12 | // 13 | // Most people will want to use the builtin bloom filter support (see 14 | // NewBloomFilterPolicy() below). 15 | 16 | #ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 17 | #define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 18 | 19 | #include 20 | 21 | #include "leveldb/export.h" 22 | 23 | namespace leveldb { 24 | 25 | class Slice; 26 | 27 | class LEVELDB_EXPORT FilterPolicy { 28 | public: 29 | virtual ~FilterPolicy(); 30 | 31 | // Return the name of this policy. Note that if the filter encoding 32 | // changes in an incompatible way, the name returned by this method 33 | // must be changed. Otherwise, old incompatible filters may be 34 | // passed to methods of this type. 35 | virtual const char* Name() const = 0; 36 | 37 | // keys[0,n-1] contains a list of keys (potentially with duplicates) 38 | // that are ordered according to the user supplied comparator. 39 | // Append a filter that summarizes keys[0,n-1] to *dst. 40 | // 41 | // Warning: do not change the initial contents of *dst. Instead, 42 | // append the newly constructed filter to *dst. 43 | virtual void CreateFilter(const Slice* keys, int n, 44 | std::string* dst) const = 0; 45 | 46 | // "filter" contains the data appended by a preceding call to 47 | // CreateFilter() on this class. This method must return true if 48 | // the key was in the list of keys passed to CreateFilter(). 49 | // This method may return true or false if the key was not on the 50 | // list, but it should aim to return false with a high probability. 51 | virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; 52 | }; 53 | 54 | // Return a new filter policy that uses a bloom filter with approximately 55 | // the specified number of bits per key. A good value for bits_per_key 56 | // is 10, which yields a filter with ~ 1% false positive rate. 57 | // 58 | // Callers must delete the result after any database that is using the 59 | // result has been closed. 60 | // 61 | // Note: if you are using a custom comparator that ignores some parts 62 | // of the keys being compared, you must not use NewBloomFilterPolicy() 63 | // and must provide your own FilterPolicy that also ignores the 64 | // corresponding parts of the keys. For example, if the comparator 65 | // ignores trailing spaces, it would be incorrect to use a 66 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 67 | // trailing spaces in keys. 68 | LEVELDB_EXPORT const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); 69 | 70 | } // namespace leveldb 71 | 72 | #endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 73 | -------------------------------------------------------------------------------- /include/leveldb/iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // An iterator yields a sequence of key/value pairs from a source. 6 | // The following class defines the interface. Multiple implementations 7 | // are provided by this library. In particular, iterators are provided 8 | // to access the contents of a Table or a DB. 9 | // 10 | // Multiple threads can invoke const methods on an Iterator without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Iterator must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 17 | 18 | #include "leveldb/export.h" 19 | #include "leveldb/slice.h" 20 | #include "leveldb/status.h" 21 | 22 | namespace leveldb { 23 | 24 | class LEVELDB_EXPORT Iterator { 25 | public: 26 | Iterator(); 27 | 28 | Iterator(const Iterator&) = delete; 29 | Iterator& operator=(const Iterator&) = delete; 30 | 31 | virtual ~Iterator(); 32 | 33 | // An iterator is either positioned at a key/value pair, or 34 | // not valid. This method returns true iff the iterator is valid. 35 | virtual bool Valid() const = 0; 36 | 37 | // Position at the first key in the source. The iterator is Valid() 38 | // after this call iff the source is not empty. 39 | virtual void SeekToFirst() = 0; 40 | 41 | // Position at the last key in the source. The iterator is 42 | // Valid() after this call iff the source is not empty. 43 | virtual void SeekToLast() = 0; 44 | 45 | // Position at the first key in the source that is at or past target. 46 | // The iterator is Valid() after this call iff the source contains 47 | // an entry that comes at or past target. 48 | virtual void Seek(const Slice& target) = 0; 49 | 50 | // Moves to the next entry in the source. After this call, Valid() is 51 | // true iff the iterator was not positioned at the last entry in the source. 52 | // REQUIRES: Valid() 53 | virtual void Next() = 0; 54 | 55 | // Moves to the previous entry in the source. After this call, Valid() is 56 | // true iff the iterator was not positioned at the first entry in source. 57 | // REQUIRES: Valid() 58 | virtual void Prev() = 0; 59 | 60 | // Return the key for the current entry. The underlying storage for 61 | // the returned slice is valid only until the next modification of 62 | // the iterator. 63 | // REQUIRES: Valid() 64 | virtual Slice key() const = 0; 65 | 66 | // Return the value for the current entry. The underlying storage for 67 | // the returned slice is valid only until the next modification of 68 | // the iterator. 69 | // REQUIRES: Valid() 70 | virtual Slice value() const = 0; 71 | 72 | // If an error has occurred, return it. Else return an ok status. 73 | virtual Status status() const = 0; 74 | 75 | // Clients are allowed to register function/arg1/arg2 triples that 76 | // will be invoked when this iterator is destroyed. 77 | // 78 | // Note that unlike all of the preceding methods, this method is 79 | // not abstract and therefore clients should not override it. 80 | using CleanupFunction = void (*)(void* arg1, void* arg2); 81 | void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); 82 | 83 | private: 84 | // Cleanup functions are stored in a single-linked list. 85 | // The list's head node is inlined in the iterator. 86 | struct CleanupNode { 87 | // True if the node is not used. Only head nodes might be unused. 88 | bool IsEmpty() const { return function == nullptr; } 89 | // Invokes the cleanup function. 90 | void Run() { 91 | assert(function != nullptr); 92 | (*function)(arg1, arg2); 93 | } 94 | 95 | // The head node is used if the function pointer is not null. 96 | CleanupFunction function; 97 | void* arg1; 98 | void* arg2; 99 | CleanupNode* next; 100 | }; 101 | CleanupNode cleanup_head_; 102 | }; 103 | 104 | // Return an empty iterator (yields nothing). 105 | LEVELDB_EXPORT Iterator* NewEmptyIterator(); 106 | 107 | // Return an empty iterator with the specified status. 108 | LEVELDB_EXPORT Iterator* NewErrorIterator(const Status& status); 109 | 110 | } // namespace leveldb 111 | 112 | #endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 113 | -------------------------------------------------------------------------------- /include/leveldb/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Slice is a simple structure containing a pointer into some external 6 | // storage and a size. The user of a Slice must ensure that the slice 7 | // is not used after the corresponding external storage has been 8 | // deallocated. 9 | // 10 | // Multiple threads can invoke const methods on a Slice without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Slice must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_SLICE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "leveldb/export.h" 24 | 25 | namespace leveldb { 26 | 27 | class LEVELDB_EXPORT Slice { 28 | public: 29 | // Create an empty slice. 30 | Slice() : data_(""), size_(0) {} 31 | 32 | // Create a slice that refers to d[0,n-1]. 33 | Slice(const char* d, size_t n) : data_(d), size_(n) {} 34 | 35 | // Create a slice that refers to the contents of "s" 36 | Slice(const std::string& s) : data_(s.data()), size_(s.size()) {} 37 | 38 | // Create a slice that refers to s[0,strlen(s)-1] 39 | Slice(const char* s) : data_(s), size_(strlen(s)) {} 40 | 41 | // Intentionally copyable. 42 | Slice(const Slice&) = default; 43 | Slice& operator=(const Slice&) = default; 44 | 45 | // Return a pointer to the beginning of the referenced data 46 | const char* data() const { return data_; } 47 | 48 | // Return the length (in bytes) of the referenced data 49 | size_t size() const { return size_; } 50 | 51 | // Return true iff the length of the referenced data is zero 52 | bool empty() const { return size_ == 0; } 53 | 54 | const char* begin() const { return data(); } 55 | const char* end() const { return data() + size(); } 56 | 57 | // Return the ith byte in the referenced data. 58 | // REQUIRES: n < size() 59 | char operator[](size_t n) const { 60 | assert(n < size()); 61 | return data_[n]; 62 | } 63 | 64 | // Change this slice to refer to an empty array 65 | void clear() { 66 | data_ = ""; 67 | size_ = 0; 68 | } 69 | 70 | // Drop the first "n" bytes from this slice. 71 | void remove_prefix(size_t n) { 72 | assert(n <= size()); 73 | data_ += n; 74 | size_ -= n; 75 | } 76 | 77 | // Return a string that contains the copy of the referenced data. 78 | std::string ToString() const { return std::string(data_, size_); } 79 | 80 | // Three-way comparison. Returns value: 81 | // < 0 iff "*this" < "b", 82 | // == 0 iff "*this" == "b", 83 | // > 0 iff "*this" > "b" 84 | int compare(const Slice& b) const; 85 | 86 | // Return true iff "x" is a prefix of "*this" 87 | bool starts_with(const Slice& x) const { 88 | return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0)); 89 | } 90 | 91 | private: 92 | const char* data_; 93 | size_t size_; 94 | }; 95 | 96 | inline bool operator==(const Slice& x, const Slice& y) { 97 | return ((x.size() == y.size()) && 98 | (memcmp(x.data(), y.data(), x.size()) == 0)); 99 | } 100 | 101 | inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); } 102 | 103 | inline int Slice::compare(const Slice& b) const { 104 | const size_t min_len = (size_ < b.size_) ? size_ : b.size_; 105 | int r = memcmp(data_, b.data_, min_len); 106 | if (r == 0) { 107 | if (size_ < b.size_) 108 | r = -1; 109 | else if (size_ > b.size_) 110 | r = +1; 111 | } 112 | return r; 113 | } 114 | 115 | } // namespace leveldb 116 | 117 | #endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ 118 | -------------------------------------------------------------------------------- /include/leveldb/status.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Status encapsulates the result of an operation. It may indicate success, 6 | // or it may indicate an error with an associated error message. 7 | // 8 | // Multiple threads can invoke const methods on a Status without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same Status must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_STATUS_H_ 15 | 16 | #include 17 | #include 18 | 19 | #include "leveldb/export.h" 20 | #include "leveldb/slice.h" 21 | 22 | namespace leveldb { 23 | 24 | class LEVELDB_EXPORT Status { 25 | public: 26 | // Create a success status. 27 | Status() noexcept : state_(nullptr) {} 28 | ~Status() { delete[] state_; } 29 | 30 | Status(const Status& rhs); 31 | Status& operator=(const Status& rhs); 32 | 33 | Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } 34 | Status& operator=(Status&& rhs) noexcept; 35 | 36 | // Return a success status. 37 | static Status OK() { return Status(); } 38 | 39 | // Return error status of an appropriate type. 40 | static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { 41 | return Status(kNotFound, msg, msg2); 42 | } 43 | static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { 44 | return Status(kCorruption, msg, msg2); 45 | } 46 | static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { 47 | return Status(kNotSupported, msg, msg2); 48 | } 49 | static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { 50 | return Status(kInvalidArgument, msg, msg2); 51 | } 52 | static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { 53 | return Status(kIOError, msg, msg2); 54 | } 55 | 56 | // Returns true iff the status indicates success. 57 | bool ok() const { return (state_ == nullptr); } 58 | 59 | // Returns true iff the status indicates a NotFound error. 60 | bool IsNotFound() const { return code() == kNotFound; } 61 | 62 | // Returns true iff the status indicates a Corruption error. 63 | bool IsCorruption() const { return code() == kCorruption; } 64 | 65 | // Returns true iff the status indicates an IOError. 66 | bool IsIOError() const { return code() == kIOError; } 67 | 68 | // Returns true iff the status indicates a NotSupportedError. 69 | bool IsNotSupportedError() const { return code() == kNotSupported; } 70 | 71 | // Returns true iff the status indicates an InvalidArgument. 72 | bool IsInvalidArgument() const { return code() == kInvalidArgument; } 73 | 74 | // Return a string representation of this status suitable for printing. 75 | // Returns the string "OK" for success. 76 | std::string ToString() const; 77 | 78 | private: 79 | enum Code { 80 | kOk = 0, 81 | kNotFound = 1, 82 | kCorruption = 2, 83 | kNotSupported = 3, 84 | kInvalidArgument = 4, 85 | kIOError = 5 86 | }; 87 | 88 | Code code() const { 89 | return (state_ == nullptr) ? kOk : static_cast(state_[4]); 90 | } 91 | 92 | Status(Code code, const Slice& msg, const Slice& msg2); 93 | static const char* CopyState(const char* s); 94 | 95 | // OK status has a null state_. Otherwise, state_ is a new[] array 96 | // of the following form: 97 | // state_[0..3] == length of message 98 | // state_[4] == code 99 | // state_[5..] == message 100 | const char* state_; 101 | }; 102 | 103 | inline Status::Status(const Status& rhs) { 104 | state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); 105 | } 106 | inline Status& Status::operator=(const Status& rhs) { 107 | // The following condition catches both aliasing (when this == &rhs), 108 | // and the common case where both rhs and *this are ok. 109 | if (state_ != rhs.state_) { 110 | delete[] state_; 111 | state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); 112 | } 113 | return *this; 114 | } 115 | inline Status& Status::operator=(Status&& rhs) noexcept { 116 | std::swap(state_, rhs.state_); 117 | return *this; 118 | } 119 | 120 | } // namespace leveldb 121 | 122 | #endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_ 123 | -------------------------------------------------------------------------------- /include/leveldb/table.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_TABLE_H_ 7 | 8 | #include 9 | 10 | #include "leveldb/export.h" 11 | #include "leveldb/iterator.h" 12 | 13 | namespace leveldb { 14 | 15 | class Block; 16 | class BlockHandle; 17 | class Footer; 18 | struct Options; 19 | class RandomAccessFile; 20 | struct ReadOptions; 21 | class TableCache; 22 | 23 | // A Table is a sorted map from strings to strings. Tables are 24 | // immutable and persistent. A Table may be safely accessed from 25 | // multiple threads without external synchronization. 26 | class LEVELDB_EXPORT Table { 27 | public: 28 | // Attempt to open the table that is stored in bytes [0..file_size) 29 | // of "file", and read the metadata entries necessary to allow 30 | // retrieving data from the table. 31 | // 32 | // If successful, returns ok and sets "*table" to the newly opened 33 | // table. The client should delete "*table" when no longer needed. 34 | // If there was an error while initializing the table, sets "*table" 35 | // to nullptr and returns a non-ok status. Does not take ownership of 36 | // "*source", but the client must ensure that "source" remains live 37 | // for the duration of the returned table's lifetime. 38 | // 39 | // *file must remain live while this Table is in use. 40 | static Status Open(const Options& options, RandomAccessFile* file, 41 | uint64_t file_size, Table** table); 42 | 43 | Table(const Table&) = delete; 44 | Table& operator=(const Table&) = delete; 45 | 46 | ~Table(); 47 | 48 | // Returns a new iterator over the table contents. 49 | // The result of NewIterator() is initially invalid (caller must 50 | // call one of the Seek methods on the iterator before using it). 51 | Iterator* NewIterator(const ReadOptions&) const; 52 | 53 | // Given a key, return an approximate byte offset in the file where 54 | // the data for that key begins (or would begin if the key were 55 | // present in the file). The returned value is in terms of file 56 | // bytes, and so includes effects like compression of the underlying data. 57 | // E.g., the approximate offset of the last key in the table will 58 | // be close to the file length. 59 | uint64_t ApproximateOffsetOf(const Slice& key) const; 60 | 61 | private: 62 | friend class TableCache; 63 | struct Rep; 64 | 65 | static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); 66 | 67 | explicit Table(Rep* rep) : rep_(rep) {} 68 | 69 | // Calls (*handle_result)(arg, ...) with the entry found after a call 70 | // to Seek(key). May not make such a call if filter policy says 71 | // that key is not present. 72 | Status InternalGet(const ReadOptions&, const Slice& key, void* arg, 73 | void (*handle_result)(void* arg, const Slice& k, 74 | const Slice& v)); 75 | 76 | void ReadMeta(const Footer& footer); 77 | void ReadFilter(const Slice& filter_handle_value); 78 | 79 | Rep* const rep_; 80 | }; 81 | 82 | } // namespace leveldb 83 | 84 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_ 85 | -------------------------------------------------------------------------------- /include/leveldb/table_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // TableBuilder provides the interface used to build a Table 6 | // (an immutable and sorted map from keys to values). 7 | // 8 | // Multiple threads can invoke const methods on a TableBuilder without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same TableBuilder must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 15 | 16 | #include 17 | 18 | #include "leveldb/export.h" 19 | #include "leveldb/options.h" 20 | #include "leveldb/status.h" 21 | 22 | namespace leveldb { 23 | 24 | class BlockBuilder; 25 | class BlockHandle; 26 | class WritableFile; 27 | 28 | class LEVELDB_EXPORT TableBuilder { 29 | public: 30 | // Create a builder that will store the contents of the table it is 31 | // building in *file. Does not close the file. It is up to the 32 | // caller to close the file after calling Finish(). 33 | TableBuilder(const Options& options, WritableFile* file); 34 | 35 | TableBuilder(const TableBuilder&) = delete; 36 | TableBuilder& operator=(const TableBuilder&) = delete; 37 | 38 | // REQUIRES: Either Finish() or Abandon() has been called. 39 | ~TableBuilder(); 40 | 41 | // Change the options used by this builder. Note: only some of the 42 | // option fields can be changed after construction. If a field is 43 | // not allowed to change dynamically and its value in the structure 44 | // passed to the constructor is different from its value in the 45 | // structure passed to this method, this method will return an error 46 | // without changing any fields. 47 | Status ChangeOptions(const Options& options); 48 | 49 | // Add key,value to the table being constructed. 50 | // REQUIRES: key is after any previously added key according to comparator. 51 | // REQUIRES: Finish(), Abandon() have not been called 52 | void Add(const Slice& key, const Slice& value); 53 | 54 | // Advanced operation: flush any buffered key/value pairs to file. 55 | // Can be used to ensure that two adjacent entries never live in 56 | // the same data block. Most clients should not need to use this method. 57 | // REQUIRES: Finish(), Abandon() have not been called 58 | void Flush(); 59 | 60 | // Return non-ok iff some error has been detected. 61 | Status status() const; 62 | 63 | // Finish building the table. Stops using the file passed to the 64 | // constructor after this function returns. 65 | // REQUIRES: Finish(), Abandon() have not been called 66 | Status Finish(); 67 | 68 | // Indicate that the contents of this builder should be abandoned. Stops 69 | // using the file passed to the constructor after this function returns. 70 | // If the caller is not going to call Finish(), it must call Abandon() 71 | // before destroying this builder. 72 | // REQUIRES: Finish(), Abandon() have not been called 73 | void Abandon(); 74 | 75 | // Number of calls to Add() so far. 76 | uint64_t NumEntries() const; 77 | 78 | // Size of the file generated so far. If invoked after a successful 79 | // Finish() call, returns the size of the final generated file. 80 | uint64_t FileSize() const; 81 | 82 | private: 83 | bool ok() const { return status().ok(); } 84 | void WriteBlock(BlockBuilder* block, BlockHandle* handle); 85 | void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle); 86 | 87 | struct Rep; 88 | Rep* rep_; 89 | }; 90 | 91 | } // namespace leveldb 92 | 93 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 94 | -------------------------------------------------------------------------------- /include/leveldb/write_batch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch holds a collection of updates to apply atomically to a DB. 6 | // 7 | // The updates are applied in the order in which they are added 8 | // to the WriteBatch. For example, the value of "key" will be "v3" 9 | // after the following batch is written: 10 | // 11 | // batch.Put("key", "v1"); 12 | // batch.Delete("key"); 13 | // batch.Put("key", "v2"); 14 | // batch.Put("key", "v3"); 15 | // 16 | // Multiple threads can invoke const methods on a WriteBatch without 17 | // external synchronization, but if any of the threads may call a 18 | // non-const method, all threads accessing the same WriteBatch must use 19 | // external synchronization. 20 | 21 | #ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 22 | #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 23 | 24 | #include 25 | 26 | #include "leveldb/export.h" 27 | #include "leveldb/status.h" 28 | 29 | namespace leveldb { 30 | 31 | class Slice; 32 | 33 | class LEVELDB_EXPORT WriteBatch { 34 | public: 35 | class LEVELDB_EXPORT Handler { 36 | public: 37 | virtual ~Handler(); 38 | virtual void Put(const Slice& key, const Slice& value) = 0; 39 | virtual void Delete(const Slice& key) = 0; 40 | }; 41 | 42 | WriteBatch(); 43 | 44 | // Intentionally copyable. 45 | WriteBatch(const WriteBatch&) = default; 46 | WriteBatch& operator=(const WriteBatch&) = default; 47 | 48 | ~WriteBatch(); 49 | 50 | // Store the mapping "key->value" in the database. 51 | void Put(const Slice& key, const Slice& value); 52 | 53 | // If the database contains a mapping for "key", erase it. Else do nothing. 54 | void Delete(const Slice& key); 55 | 56 | // Clear all updates buffered in this batch. 57 | void Clear(); 58 | 59 | // The size of the database changes caused by this batch. 60 | // 61 | // This number is tied to implementation details, and may change across 62 | // releases. It is intended for LevelDB usage metrics. 63 | size_t ApproximateSize() const; 64 | 65 | // Copies the operations in "source" to this batch. 66 | // 67 | // This runs in O(source size) time. However, the constant factor is better 68 | // than calling Iterate() over the source batch with a Handler that replicates 69 | // the operations into this batch. 70 | void Append(const WriteBatch& source); 71 | 72 | // Support for iterating over the contents of a batch. 73 | Status Iterate(Handler* handler) const; 74 | 75 | private: 76 | friend class WriteBatchInternal; 77 | 78 | std::string rep_; // See comment in write_batch.cc for the format of rep_ 79 | }; 80 | 81 | } // namespace leveldb 82 | 83 | #endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 84 | -------------------------------------------------------------------------------- /issues/issue178_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 178: a manual compaction causes deleted data to reappear. 6 | #include 7 | #include 8 | #include 9 | 10 | #include "gtest/gtest.h" 11 | #include "leveldb/db.h" 12 | #include "leveldb/write_batch.h" 13 | #include "util/testutil.h" 14 | 15 | namespace { 16 | 17 | const int kNumKeys = 1100000; 18 | 19 | std::string Key1(int i) { 20 | char buf[100]; 21 | std::snprintf(buf, sizeof(buf), "my_key_%d", i); 22 | return buf; 23 | } 24 | 25 | std::string Key2(int i) { return Key1(i) + "_xxx"; } 26 | 27 | TEST(Issue178, Test) { 28 | // Get rid of any state from an old run. 29 | std::string dbpath = testing::TempDir() + "leveldb_cbug_test"; 30 | DestroyDB(dbpath, leveldb::Options()); 31 | 32 | // Open database. Disable compression since it affects the creation 33 | // of layers and the code below is trying to test against a very 34 | // specific scenario. 35 | leveldb::DB* db; 36 | leveldb::Options db_options; 37 | db_options.create_if_missing = true; 38 | db_options.compression = leveldb::kNoCompression; 39 | ASSERT_LEVELDB_OK(leveldb::DB::Open(db_options, dbpath, &db)); 40 | 41 | // create first key range 42 | leveldb::WriteBatch batch; 43 | for (size_t i = 0; i < kNumKeys; i++) { 44 | batch.Put(Key1(i), "value for range 1 key"); 45 | } 46 | ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); 47 | 48 | // create second key range 49 | batch.Clear(); 50 | for (size_t i = 0; i < kNumKeys; i++) { 51 | batch.Put(Key2(i), "value for range 2 key"); 52 | } 53 | ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); 54 | 55 | // delete second key range 56 | batch.Clear(); 57 | for (size_t i = 0; i < kNumKeys; i++) { 58 | batch.Delete(Key2(i)); 59 | } 60 | ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch)); 61 | 62 | // compact database 63 | std::string start_key = Key1(0); 64 | std::string end_key = Key1(kNumKeys - 1); 65 | leveldb::Slice least(start_key.data(), start_key.size()); 66 | leveldb::Slice greatest(end_key.data(), end_key.size()); 67 | 68 | // commenting out the line below causes the example to work correctly 69 | db->CompactRange(&least, &greatest); 70 | 71 | // count the keys 72 | leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); 73 | size_t num_keys = 0; 74 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 75 | num_keys++; 76 | } 77 | delete iter; 78 | ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; 79 | 80 | // close database 81 | delete db; 82 | DestroyDB(dbpath, leveldb::Options()); 83 | } 84 | 85 | } // anonymous namespace 86 | -------------------------------------------------------------------------------- /issues/issue200_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 200: when iterator switches direction from backward 6 | // to forward, the current key can be yielded unexpectedly if a new 7 | // mutation has been added just before the current key. 8 | 9 | #include "gtest/gtest.h" 10 | #include "leveldb/db.h" 11 | #include "util/testutil.h" 12 | 13 | namespace leveldb { 14 | 15 | TEST(Issue200, Test) { 16 | // Get rid of any state from an old run. 17 | std::string dbpath = testing::TempDir() + "leveldb_issue200_test"; 18 | DestroyDB(dbpath, Options()); 19 | 20 | DB* db; 21 | Options options; 22 | options.create_if_missing = true; 23 | ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db)); 24 | 25 | WriteOptions write_options; 26 | ASSERT_LEVELDB_OK(db->Put(write_options, "1", "b")); 27 | ASSERT_LEVELDB_OK(db->Put(write_options, "2", "c")); 28 | ASSERT_LEVELDB_OK(db->Put(write_options, "3", "d")); 29 | ASSERT_LEVELDB_OK(db->Put(write_options, "4", "e")); 30 | ASSERT_LEVELDB_OK(db->Put(write_options, "5", "f")); 31 | 32 | ReadOptions read_options; 33 | Iterator* iter = db->NewIterator(read_options); 34 | 35 | // Add an element that should not be reflected in the iterator. 36 | ASSERT_LEVELDB_OK(db->Put(write_options, "25", "cd")); 37 | 38 | iter->Seek("5"); 39 | ASSERT_EQ(iter->key().ToString(), "5"); 40 | iter->Prev(); 41 | ASSERT_EQ(iter->key().ToString(), "4"); 42 | iter->Prev(); 43 | ASSERT_EQ(iter->key().ToString(), "3"); 44 | iter->Next(); 45 | ASSERT_EQ(iter->key().ToString(), "4"); 46 | iter->Next(); 47 | ASSERT_EQ(iter->key().ToString(), "5"); 48 | 49 | delete iter; 50 | delete db; 51 | DestroyDB(dbpath, options); 52 | } 53 | 54 | } // namespace leveldb 55 | -------------------------------------------------------------------------------- /issues/issue320_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "gtest/gtest.h" 13 | #include "leveldb/db.h" 14 | #include "leveldb/write_batch.h" 15 | #include "util/testutil.h" 16 | 17 | namespace leveldb { 18 | 19 | namespace { 20 | 21 | // Creates a random number in the range of [0, max). 22 | int GenerateRandomNumber(int max) { return std::rand() % max; } 23 | 24 | std::string CreateRandomString(int32_t index) { 25 | static const size_t len = 1024; 26 | char bytes[len]; 27 | size_t i = 0; 28 | while (i < 8) { 29 | bytes[i] = 'a' + ((index >> (4 * i)) & 0xf); 30 | ++i; 31 | } 32 | while (i < sizeof(bytes)) { 33 | bytes[i] = 'a' + GenerateRandomNumber(26); 34 | ++i; 35 | } 36 | return std::string(bytes, sizeof(bytes)); 37 | } 38 | 39 | } // namespace 40 | 41 | TEST(Issue320, Test) { 42 | std::srand(0); 43 | 44 | bool delete_before_put = false; 45 | bool keep_snapshots = true; 46 | 47 | std::vector>> test_map( 48 | 10000); 49 | std::vector snapshots(100, nullptr); 50 | 51 | DB* db; 52 | Options options; 53 | options.create_if_missing = true; 54 | 55 | std::string dbpath = testing::TempDir() + "leveldb_issue320_test"; 56 | ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db)); 57 | 58 | uint32_t target_size = 10000; 59 | uint32_t num_items = 0; 60 | uint32_t count = 0; 61 | std::string key; 62 | std::string value, old_value; 63 | 64 | WriteOptions writeOptions; 65 | ReadOptions readOptions; 66 | while (count < 200000) { 67 | if ((++count % 1000) == 0) { 68 | std::cout << "count: " << count << std::endl; 69 | } 70 | 71 | int index = GenerateRandomNumber(test_map.size()); 72 | WriteBatch batch; 73 | 74 | if (test_map[index] == nullptr) { 75 | num_items++; 76 | test_map[index].reset(new std::pair( 77 | CreateRandomString(index), CreateRandomString(index))); 78 | batch.Put(test_map[index]->first, test_map[index]->second); 79 | } else { 80 | ASSERT_LEVELDB_OK( 81 | db->Get(readOptions, test_map[index]->first, &old_value)); 82 | if (old_value != test_map[index]->second) { 83 | std::cout << "ERROR incorrect value returned by Get" << std::endl; 84 | std::cout << " count=" << count << std::endl; 85 | std::cout << " old value=" << old_value << std::endl; 86 | std::cout << " test_map[index]->second=" << test_map[index]->second 87 | << std::endl; 88 | std::cout << " test_map[index]->first=" << test_map[index]->first 89 | << std::endl; 90 | std::cout << " index=" << index << std::endl; 91 | ASSERT_EQ(old_value, test_map[index]->second); 92 | } 93 | 94 | if (num_items >= target_size && GenerateRandomNumber(100) > 30) { 95 | batch.Delete(test_map[index]->first); 96 | test_map[index] = nullptr; 97 | --num_items; 98 | } else { 99 | test_map[index]->second = CreateRandomString(index); 100 | if (delete_before_put) batch.Delete(test_map[index]->first); 101 | batch.Put(test_map[index]->first, test_map[index]->second); 102 | } 103 | } 104 | 105 | ASSERT_LEVELDB_OK(db->Write(writeOptions, &batch)); 106 | 107 | if (keep_snapshots && GenerateRandomNumber(10) == 0) { 108 | int i = GenerateRandomNumber(snapshots.size()); 109 | if (snapshots[i] != nullptr) { 110 | db->ReleaseSnapshot(snapshots[i]); 111 | } 112 | snapshots[i] = db->GetSnapshot(); 113 | } 114 | } 115 | 116 | for (Snapshot const* snapshot : snapshots) { 117 | if (snapshot) { 118 | db->ReleaseSnapshot(snapshot); 119 | } 120 | } 121 | 122 | delete db; 123 | DestroyDB(dbpath, options); 124 | } 125 | 126 | } // namespace leveldb 127 | -------------------------------------------------------------------------------- /port/README.md: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_stdcxx.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) || defined(LEVELDB_PLATFORM_WINDOWS) 14 | #include "port/port_stdcxx.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | #include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /port/port_config.h.in: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ 7 | 8 | // Define to 1 if you have a definition for fdatasync() in . 9 | #if !defined(HAVE_FDATASYNC) 10 | #cmakedefine01 HAVE_FDATASYNC 11 | #endif // !defined(HAVE_FDATASYNC) 12 | 13 | // Define to 1 if you have a definition for F_FULLFSYNC in . 14 | #if !defined(HAVE_FULLFSYNC) 15 | #cmakedefine01 HAVE_FULLFSYNC 16 | #endif // !defined(HAVE_FULLFSYNC) 17 | 18 | // Define to 1 if you have a definition for O_CLOEXEC in . 19 | #if !defined(HAVE_O_CLOEXEC) 20 | #cmakedefine01 HAVE_O_CLOEXEC 21 | #endif // !defined(HAVE_O_CLOEXEC) 22 | 23 | // Define to 1 if you have Google CRC32C. 24 | #if !defined(HAVE_CRC32C) 25 | #cmakedefine01 HAVE_CRC32C 26 | #endif // !defined(HAVE_CRC32C) 27 | 28 | // Define to 1 if you have Google Snappy. 29 | #if !defined(HAVE_SNAPPY) 30 | #cmakedefine01 HAVE_SNAPPY 31 | #endif // !defined(HAVE_SNAPPY) 32 | 33 | // Define to 1 if you have Zstd. 34 | #if !defined(HAVE_Zstd) 35 | #cmakedefine01 HAVE_ZSTD 36 | #endif // !defined(HAVE_ZSTD) 37 | 38 | #endif // STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ 39 | -------------------------------------------------------------------------------- /port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 6 | #define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 7 | 8 | // Use Clang's thread safety analysis annotations when available. In other 9 | // environments, the macros receive empty definitions. 10 | // Usage documentation: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html 11 | 12 | #if !defined(THREAD_ANNOTATION_ATTRIBUTE__) 13 | 14 | #if defined(__clang__) 15 | 16 | #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) 17 | #else 18 | #define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op 19 | #endif 20 | 21 | #endif // !defined(THREAD_ANNOTATION_ATTRIBUTE__) 22 | 23 | #ifndef GUARDED_BY 24 | #define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) 25 | #endif 26 | 27 | #ifndef PT_GUARDED_BY 28 | #define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) 29 | #endif 30 | 31 | #ifndef ACQUIRED_AFTER 32 | #define ACQUIRED_AFTER(...) \ 33 | THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) 34 | #endif 35 | 36 | #ifndef ACQUIRED_BEFORE 37 | #define ACQUIRED_BEFORE(...) \ 38 | THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) 39 | #endif 40 | 41 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 42 | #define EXCLUSIVE_LOCKS_REQUIRED(...) \ 43 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) 44 | #endif 45 | 46 | #ifndef SHARED_LOCKS_REQUIRED 47 | #define SHARED_LOCKS_REQUIRED(...) \ 48 | THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) 49 | #endif 50 | 51 | #ifndef LOCKS_EXCLUDED 52 | #define LOCKS_EXCLUDED(...) \ 53 | THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) 54 | #endif 55 | 56 | #ifndef LOCK_RETURNED 57 | #define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) 58 | #endif 59 | 60 | #ifndef LOCKABLE 61 | #define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable) 62 | #endif 63 | 64 | #ifndef SCOPED_LOCKABLE 65 | #define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) 66 | #endif 67 | 68 | #ifndef EXCLUSIVE_LOCK_FUNCTION 69 | #define EXCLUSIVE_LOCK_FUNCTION(...) \ 70 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) 71 | #endif 72 | 73 | #ifndef SHARED_LOCK_FUNCTION 74 | #define SHARED_LOCK_FUNCTION(...) \ 75 | THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) 76 | #endif 77 | 78 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 79 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ 80 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) 81 | #endif 82 | 83 | #ifndef SHARED_TRYLOCK_FUNCTION 84 | #define SHARED_TRYLOCK_FUNCTION(...) \ 85 | THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) 86 | #endif 87 | 88 | #ifndef UNLOCK_FUNCTION 89 | #define UNLOCK_FUNCTION(...) \ 90 | THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) 91 | #endif 92 | 93 | #ifndef NO_THREAD_SAFETY_ANALYSIS 94 | #define NO_THREAD_SAFETY_ANALYSIS \ 95 | THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) 96 | #endif 97 | 98 | #ifndef ASSERT_EXCLUSIVE_LOCK 99 | #define ASSERT_EXCLUSIVE_LOCK(...) \ 100 | THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) 101 | #endif 102 | 103 | #ifndef ASSERT_SHARED_LOCK 104 | #define ASSERT_SHARED_LOCK(...) \ 105 | THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) 106 | #endif 107 | 108 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 109 | -------------------------------------------------------------------------------- /table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "leveldb/iterator.h" 12 | 13 | namespace leveldb { 14 | 15 | struct BlockContents; 16 | class Comparator; 17 | 18 | class Block { 19 | public: 20 | // Initialize the block with the specified contents. 21 | explicit Block(const BlockContents& contents); 22 | 23 | Block(const Block&) = delete; 24 | Block& operator=(const Block&) = delete; 25 | 26 | ~Block(); 27 | 28 | size_t size() const { return size_; } 29 | Iterator* NewIterator(const Comparator* comparator); 30 | 31 | private: 32 | class Iter; 33 | 34 | uint32_t NumRestarts() const; 35 | 36 | const char* data_; 37 | size_t size_; 38 | uint32_t restart_offset_; // Offset in data_ of restart array 39 | bool owned_; // Block owns data_[] 40 | }; 41 | 42 | } // namespace leveldb 43 | 44 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 45 | -------------------------------------------------------------------------------- /table/block_builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // BlockBuilder generates blocks where keys are prefix-compressed: 6 | // 7 | // When we store a key, we drop the prefix shared with the previous 8 | // string. This helps reduce the space requirement significantly. 9 | // Furthermore, once every K keys, we do not apply the prefix 10 | // compression and store the entire key. We call this a "restart 11 | // point". The tail end of the block stores the offsets of all of the 12 | // restart points, and can be used to do a binary search when looking 13 | // for a particular key. Values are stored as-is (without compression) 14 | // immediately following the corresponding key. 15 | // 16 | // An entry for a particular key-value pair has the form: 17 | // shared_bytes: varint32 18 | // unshared_bytes: varint32 19 | // value_length: varint32 20 | // key_delta: char[unshared_bytes] 21 | // value: char[value_length] 22 | // shared_bytes == 0 for restart points. 23 | // 24 | // The trailer of the block has the form: 25 | // restarts: uint32[num_restarts] 26 | // num_restarts: uint32 27 | // restarts[i] contains the offset within the block of the ith restart point. 28 | 29 | #include "table/block_builder.h" 30 | 31 | #include 32 | #include 33 | 34 | #include "leveldb/comparator.h" 35 | #include "leveldb/options.h" 36 | #include "util/coding.h" 37 | 38 | namespace leveldb { 39 | 40 | BlockBuilder::BlockBuilder(const Options* options) 41 | : options_(options), restarts_(), counter_(0), finished_(false) { 42 | assert(options->block_restart_interval >= 1); 43 | restarts_.push_back(0); // First restart point is at offset 0 44 | } 45 | 46 | void BlockBuilder::Reset() { 47 | buffer_.clear(); 48 | restarts_.clear(); 49 | restarts_.push_back(0); // First restart point is at offset 0 50 | counter_ = 0; 51 | finished_ = false; 52 | last_key_.clear(); 53 | } 54 | 55 | size_t BlockBuilder::CurrentSizeEstimate() const { 56 | return (buffer_.size() + // Raw data buffer 57 | restarts_.size() * sizeof(uint32_t) + // Restart array 58 | sizeof(uint32_t)); // Restart array length 59 | } 60 | 61 | Slice BlockBuilder::Finish() { 62 | // Append restart array 63 | for (size_t i = 0; i < restarts_.size(); i++) { 64 | PutFixed32(&buffer_, restarts_[i]); 65 | } 66 | PutFixed32(&buffer_, restarts_.size()); 67 | finished_ = true; 68 | return Slice(buffer_); 69 | } 70 | 71 | void BlockBuilder::Add(const Slice& key, const Slice& value) { 72 | Slice last_key_piece(last_key_); 73 | assert(!finished_); 74 | assert(counter_ <= options_->block_restart_interval); 75 | assert(buffer_.empty() // No values yet? 76 | || options_->comparator->Compare(key, last_key_piece) > 0); 77 | size_t shared = 0; 78 | if (counter_ < options_->block_restart_interval) { 79 | // See how much sharing to do with previous string 80 | const size_t min_length = std::min(last_key_piece.size(), key.size()); 81 | while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { 82 | shared++; 83 | } 84 | } else { 85 | // Restart compression 86 | restarts_.push_back(buffer_.size()); 87 | counter_ = 0; 88 | } 89 | const size_t non_shared = key.size() - shared; 90 | 91 | // Add "" to buffer_ 92 | PutVarint32(&buffer_, shared); 93 | PutVarint32(&buffer_, non_shared); 94 | PutVarint32(&buffer_, value.size()); 95 | 96 | // Add string delta to buffer_ followed by value 97 | buffer_.append(key.data() + shared, non_shared); 98 | buffer_.append(value.data(), value.size()); 99 | 100 | // Update state 101 | last_key_.resize(shared); 102 | last_key_.append(key.data() + shared, non_shared); 103 | assert(Slice(last_key_) == key); 104 | counter_++; 105 | } 106 | 107 | } // namespace leveldb 108 | -------------------------------------------------------------------------------- /table/block_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "leveldb/slice.h" 12 | 13 | namespace leveldb { 14 | 15 | struct Options; 16 | 17 | class BlockBuilder { 18 | public: 19 | explicit BlockBuilder(const Options* options); 20 | 21 | BlockBuilder(const BlockBuilder&) = delete; 22 | BlockBuilder& operator=(const BlockBuilder&) = delete; 23 | 24 | // Reset the contents as if the BlockBuilder was just constructed. 25 | void Reset(); 26 | 27 | // REQUIRES: Finish() has not been called since the last call to Reset(). 28 | // REQUIRES: key is larger than any previously added key 29 | void Add(const Slice& key, const Slice& value); 30 | 31 | // Finish building the block and return a slice that refers to the 32 | // block contents. The returned slice will remain valid for the 33 | // lifetime of this builder or until Reset() is called. 34 | Slice Finish(); 35 | 36 | // Returns an estimate of the current (uncompressed) size of the block 37 | // we are building. 38 | size_t CurrentSizeEstimate() const; 39 | 40 | // Return true iff no entries have been added since the last Reset() 41 | bool empty() const { return buffer_.empty(); } 42 | 43 | private: 44 | const Options* options_; 45 | std::string buffer_; // Destination buffer 46 | std::vector restarts_; // Restart points 47 | int counter_; // Number of entries emitted since restart 48 | bool finished_; // Has Finish() been called? 49 | std::string last_key_; 50 | }; 51 | 52 | } // namespace leveldb 53 | 54 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 55 | -------------------------------------------------------------------------------- /table/filter_block.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/filter_block.h" 6 | 7 | #include "leveldb/filter_policy.h" 8 | #include "util/coding.h" 9 | 10 | namespace leveldb { 11 | 12 | // See doc/table_format.md for an explanation of the filter block format. 13 | 14 | // Generate new filter every 2KB of data 15 | static const size_t kFilterBaseLg = 11; 16 | static const size_t kFilterBase = 1 << kFilterBaseLg; 17 | 18 | FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) 19 | : policy_(policy) {} 20 | 21 | void FilterBlockBuilder::StartBlock(uint64_t block_offset) { 22 | uint64_t filter_index = (block_offset / kFilterBase); 23 | assert(filter_index >= filter_offsets_.size()); 24 | while (filter_index > filter_offsets_.size()) { 25 | GenerateFilter(); 26 | } 27 | } 28 | 29 | void FilterBlockBuilder::AddKey(const Slice& key) { 30 | Slice k = key; 31 | start_.push_back(keys_.size()); 32 | keys_.append(k.data(), k.size()); 33 | } 34 | 35 | Slice FilterBlockBuilder::Finish() { 36 | if (!start_.empty()) { 37 | GenerateFilter(); 38 | } 39 | 40 | // Append array of per-filter offsets 41 | const uint32_t array_offset = result_.size(); 42 | for (size_t i = 0; i < filter_offsets_.size(); i++) { 43 | PutFixed32(&result_, filter_offsets_[i]); 44 | } 45 | 46 | PutFixed32(&result_, array_offset); 47 | result_.push_back(kFilterBaseLg); // Save encoding parameter in result 48 | return Slice(result_); 49 | } 50 | 51 | void FilterBlockBuilder::GenerateFilter() { 52 | const size_t num_keys = start_.size(); 53 | if (num_keys == 0) { 54 | // Fast path if there are no keys for this filter 55 | filter_offsets_.push_back(result_.size()); 56 | return; 57 | } 58 | 59 | // Make list of keys from flattened key structure 60 | start_.push_back(keys_.size()); // Simplify length computation 61 | tmp_keys_.resize(num_keys); 62 | for (size_t i = 0; i < num_keys; i++) { 63 | const char* base = keys_.data() + start_[i]; 64 | size_t length = start_[i + 1] - start_[i]; 65 | tmp_keys_[i] = Slice(base, length); 66 | } 67 | 68 | // Generate filter for current set of keys and append to result_. 69 | filter_offsets_.push_back(result_.size()); 70 | policy_->CreateFilter(&tmp_keys_[0], static_cast(num_keys), &result_); 71 | 72 | tmp_keys_.clear(); 73 | keys_.clear(); 74 | start_.clear(); 75 | } 76 | 77 | FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, 78 | const Slice& contents) 79 | : policy_(policy), data_(nullptr), offset_(nullptr), num_(0), base_lg_(0) { 80 | size_t n = contents.size(); 81 | if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array 82 | base_lg_ = contents[n - 1]; 83 | uint32_t last_word = DecodeFixed32(contents.data() + n - 5); 84 | if (last_word > n - 5) return; 85 | data_ = contents.data(); 86 | offset_ = data_ + last_word; 87 | num_ = (n - 5 - last_word) / 4; 88 | } 89 | 90 | bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { 91 | uint64_t index = block_offset >> base_lg_; 92 | if (index < num_) { 93 | uint32_t start = DecodeFixed32(offset_ + index * 4); 94 | uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4); 95 | if (start <= limit && limit <= static_cast(offset_ - data_)) { 96 | Slice filter = Slice(data_ + start, limit - start); 97 | return policy_->KeyMayMatch(key, filter); 98 | } else if (start == limit) { 99 | // Empty filters do not match any keys 100 | return false; 101 | } 102 | } 103 | return true; // Errors are treated as potential matches 104 | } 105 | 106 | } // namespace leveldb 107 | -------------------------------------------------------------------------------- /table/filter_block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A filter block is stored near the end of a Table file. It contains 6 | // filters (e.g., bloom filters) for all data blocks in the table combined 7 | // into a single filter block. 8 | 9 | #ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 10 | #define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "leveldb/slice.h" 18 | #include "util/hash.h" 19 | 20 | namespace leveldb { 21 | 22 | class FilterPolicy; 23 | 24 | // A FilterBlockBuilder is used to construct all of the filters for a 25 | // particular Table. It generates a single string which is stored as 26 | // a special block in the Table. 27 | // 28 | // The sequence of calls to FilterBlockBuilder must match the regexp: 29 | // (StartBlock AddKey*)* Finish 30 | class FilterBlockBuilder { 31 | public: 32 | explicit FilterBlockBuilder(const FilterPolicy*); 33 | 34 | FilterBlockBuilder(const FilterBlockBuilder&) = delete; 35 | FilterBlockBuilder& operator=(const FilterBlockBuilder&) = delete; 36 | 37 | void StartBlock(uint64_t block_offset); 38 | void AddKey(const Slice& key); 39 | Slice Finish(); 40 | 41 | private: 42 | void GenerateFilter(); 43 | 44 | const FilterPolicy* policy_; 45 | std::string keys_; // Flattened key contents 46 | std::vector start_; // Starting index in keys_ of each key 47 | std::string result_; // Filter data computed so far 48 | std::vector tmp_keys_; // policy_->CreateFilter() argument 49 | std::vector filter_offsets_; 50 | }; 51 | 52 | class FilterBlockReader { 53 | public: 54 | // REQUIRES: "contents" and *policy must stay live while *this is live. 55 | FilterBlockReader(const FilterPolicy* policy, const Slice& contents); 56 | bool KeyMayMatch(uint64_t block_offset, const Slice& key); 57 | 58 | private: 59 | const FilterPolicy* policy_; 60 | const char* data_; // Pointer to filter data (at block-start) 61 | const char* offset_; // Pointer to beginning of offset array (at block-end) 62 | size_t num_; // Number of entries in offset array 63 | size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) 64 | }; 65 | 66 | } // namespace leveldb 67 | 68 | #endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 69 | -------------------------------------------------------------------------------- /table/filter_block_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/filter_block.h" 6 | 7 | #include "gtest/gtest.h" 8 | #include "leveldb/filter_policy.h" 9 | #include "util/coding.h" 10 | #include "util/hash.h" 11 | #include "util/logging.h" 12 | #include "util/testutil.h" 13 | 14 | namespace leveldb { 15 | 16 | // For testing: emit an array with one hash value per key 17 | class TestHashFilter : public FilterPolicy { 18 | public: 19 | const char* Name() const override { return "TestHashFilter"; } 20 | 21 | void CreateFilter(const Slice* keys, int n, std::string* dst) const override { 22 | for (int i = 0; i < n; i++) { 23 | uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); 24 | PutFixed32(dst, h); 25 | } 26 | } 27 | 28 | bool KeyMayMatch(const Slice& key, const Slice& filter) const override { 29 | uint32_t h = Hash(key.data(), key.size(), 1); 30 | for (size_t i = 0; i + 4 <= filter.size(); i += 4) { 31 | if (h == DecodeFixed32(filter.data() + i)) { 32 | return true; 33 | } 34 | } 35 | return false; 36 | } 37 | }; 38 | 39 | class FilterBlockTest : public testing::Test { 40 | public: 41 | TestHashFilter policy_; 42 | }; 43 | 44 | TEST_F(FilterBlockTest, EmptyBuilder) { 45 | FilterBlockBuilder builder(&policy_); 46 | Slice block = builder.Finish(); 47 | ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); 48 | FilterBlockReader reader(&policy_, block); 49 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 50 | ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); 51 | } 52 | 53 | TEST_F(FilterBlockTest, SingleChunk) { 54 | FilterBlockBuilder builder(&policy_); 55 | builder.StartBlock(100); 56 | builder.AddKey("foo"); 57 | builder.AddKey("bar"); 58 | builder.AddKey("box"); 59 | builder.StartBlock(200); 60 | builder.AddKey("box"); 61 | builder.StartBlock(300); 62 | builder.AddKey("hello"); 63 | Slice block = builder.Finish(); 64 | FilterBlockReader reader(&policy_, block); 65 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 66 | ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); 67 | ASSERT_TRUE(reader.KeyMayMatch(100, "box")); 68 | ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); 69 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 70 | ASSERT_TRUE(!reader.KeyMayMatch(100, "missing")); 71 | ASSERT_TRUE(!reader.KeyMayMatch(100, "other")); 72 | } 73 | 74 | TEST_F(FilterBlockTest, MultiChunk) { 75 | FilterBlockBuilder builder(&policy_); 76 | 77 | // First filter 78 | builder.StartBlock(0); 79 | builder.AddKey("foo"); 80 | builder.StartBlock(2000); 81 | builder.AddKey("bar"); 82 | 83 | // Second filter 84 | builder.StartBlock(3100); 85 | builder.AddKey("box"); 86 | 87 | // Third filter is empty 88 | 89 | // Last filter 90 | builder.StartBlock(9000); 91 | builder.AddKey("box"); 92 | builder.AddKey("hello"); 93 | 94 | Slice block = builder.Finish(); 95 | FilterBlockReader reader(&policy_, block); 96 | 97 | // Check first filter 98 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 99 | ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); 100 | ASSERT_TRUE(!reader.KeyMayMatch(0, "box")); 101 | ASSERT_TRUE(!reader.KeyMayMatch(0, "hello")); 102 | 103 | // Check second filter 104 | ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); 105 | ASSERT_TRUE(!reader.KeyMayMatch(3100, "foo")); 106 | ASSERT_TRUE(!reader.KeyMayMatch(3100, "bar")); 107 | ASSERT_TRUE(!reader.KeyMayMatch(3100, "hello")); 108 | 109 | // Check third filter (empty) 110 | ASSERT_TRUE(!reader.KeyMayMatch(4100, "foo")); 111 | ASSERT_TRUE(!reader.KeyMayMatch(4100, "bar")); 112 | ASSERT_TRUE(!reader.KeyMayMatch(4100, "box")); 113 | ASSERT_TRUE(!reader.KeyMayMatch(4100, "hello")); 114 | 115 | // Check last filter 116 | ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); 117 | ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); 118 | ASSERT_TRUE(!reader.KeyMayMatch(9000, "foo")); 119 | ASSERT_TRUE(!reader.KeyMayMatch(9000, "bar")); 120 | } 121 | 122 | } // namespace leveldb 123 | -------------------------------------------------------------------------------- /table/format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ 6 | #define STORAGE_LEVELDB_TABLE_FORMAT_H_ 7 | 8 | #include 9 | #include 10 | 11 | #include "leveldb/slice.h" 12 | #include "leveldb/status.h" 13 | #include "leveldb/table_builder.h" 14 | 15 | namespace leveldb { 16 | 17 | class Block; 18 | class RandomAccessFile; 19 | struct ReadOptions; 20 | 21 | // BlockHandle is a pointer to the extent of a file that stores a data 22 | // block or a meta block. 23 | class BlockHandle { 24 | public: 25 | // Maximum encoding length of a BlockHandle 26 | enum { kMaxEncodedLength = 10 + 10 }; 27 | 28 | BlockHandle(); 29 | 30 | // The offset of the block in the file. 31 | uint64_t offset() const { return offset_; } 32 | void set_offset(uint64_t offset) { offset_ = offset; } 33 | 34 | // The size of the stored block 35 | uint64_t size() const { return size_; } 36 | void set_size(uint64_t size) { size_ = size; } 37 | 38 | void EncodeTo(std::string* dst) const; 39 | Status DecodeFrom(Slice* input); 40 | 41 | private: 42 | uint64_t offset_; 43 | uint64_t size_; 44 | }; 45 | 46 | // Footer encapsulates the fixed information stored at the tail 47 | // end of every table file. 48 | class Footer { 49 | public: 50 | // Encoded length of a Footer. Note that the serialization of a 51 | // Footer will always occupy exactly this many bytes. It consists 52 | // of two block handles and a magic number. 53 | enum { kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8 }; 54 | 55 | Footer() = default; 56 | 57 | // The block handle for the metaindex block of the table 58 | const BlockHandle& metaindex_handle() const { return metaindex_handle_; } 59 | void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } 60 | 61 | // The block handle for the index block of the table 62 | const BlockHandle& index_handle() const { return index_handle_; } 63 | void set_index_handle(const BlockHandle& h) { index_handle_ = h; } 64 | 65 | void EncodeTo(std::string* dst) const; 66 | Status DecodeFrom(Slice* input); 67 | 68 | private: 69 | BlockHandle metaindex_handle_; 70 | BlockHandle index_handle_; 71 | }; 72 | 73 | // kTableMagicNumber was picked by running 74 | // echo http://code.google.com/p/leveldb/ | sha1sum 75 | // and taking the leading 64 bits. 76 | static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; 77 | 78 | // 1-byte type + 32-bit crc 79 | static const size_t kBlockTrailerSize = 5; 80 | 81 | struct BlockContents { 82 | Slice data; // Actual contents of data 83 | bool cachable; // True iff data can be cached 84 | bool heap_allocated; // True iff caller should delete[] data.data() 85 | }; 86 | 87 | // Read the block identified by "handle" from "file". On failure 88 | // return non-OK. On success fill *result and return OK. 89 | Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, 90 | const BlockHandle& handle, BlockContents* result); 91 | 92 | // Implementation details follow. Clients should ignore, 93 | 94 | inline BlockHandle::BlockHandle() 95 | : offset_(~static_cast(0)), size_(~static_cast(0)) {} 96 | 97 | } // namespace leveldb 98 | 99 | #endif // STORAGE_LEVELDB_TABLE_FORMAT_H_ 100 | -------------------------------------------------------------------------------- /table/iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/iterator.h" 6 | 7 | namespace leveldb { 8 | 9 | Iterator::Iterator() { 10 | cleanup_head_.function = nullptr; 11 | cleanup_head_.next = nullptr; 12 | } 13 | 14 | Iterator::~Iterator() { 15 | if (!cleanup_head_.IsEmpty()) { 16 | cleanup_head_.Run(); 17 | for (CleanupNode* node = cleanup_head_.next; node != nullptr;) { 18 | node->Run(); 19 | CleanupNode* next_node = node->next; 20 | delete node; 21 | node = next_node; 22 | } 23 | } 24 | } 25 | 26 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 27 | assert(func != nullptr); 28 | CleanupNode* node; 29 | if (cleanup_head_.IsEmpty()) { 30 | node = &cleanup_head_; 31 | } else { 32 | node = new CleanupNode(); 33 | node->next = cleanup_head_.next; 34 | cleanup_head_.next = node; 35 | } 36 | node->function = func; 37 | node->arg1 = arg1; 38 | node->arg2 = arg2; 39 | } 40 | 41 | namespace { 42 | 43 | class EmptyIterator : public Iterator { 44 | public: 45 | EmptyIterator(const Status& s) : status_(s) {} 46 | ~EmptyIterator() override = default; 47 | 48 | bool Valid() const override { return false; } 49 | void Seek(const Slice& target) override {} 50 | void SeekToFirst() override {} 51 | void SeekToLast() override {} 52 | void Next() override { assert(false); } 53 | void Prev() override { assert(false); } 54 | Slice key() const override { 55 | assert(false); 56 | return Slice(); 57 | } 58 | Slice value() const override { 59 | assert(false); 60 | return Slice(); 61 | } 62 | Status status() const override { return status_; } 63 | 64 | private: 65 | Status status_; 66 | }; 67 | 68 | } // anonymous namespace 69 | 70 | Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); } 71 | 72 | Iterator* NewErrorIterator(const Status& status) { 73 | return new EmptyIterator(status); 74 | } 75 | 76 | } // namespace leveldb 77 | -------------------------------------------------------------------------------- /table/iterator_wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 6 | #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | #include "leveldb/slice.h" 10 | 11 | namespace leveldb { 12 | 13 | // A internal wrapper class with an interface similar to Iterator that 14 | // caches the valid() and key() results for an underlying iterator. 15 | // This can help avoid virtual function calls and also gives better 16 | // cache locality. 17 | class IteratorWrapper { 18 | public: 19 | IteratorWrapper() : iter_(nullptr), valid_(false) {} 20 | explicit IteratorWrapper(Iterator* iter) : iter_(nullptr) { Set(iter); } 21 | ~IteratorWrapper() { delete iter_; } 22 | Iterator* iter() const { return iter_; } 23 | 24 | // Takes ownership of "iter" and will delete it when destroyed, or 25 | // when Set() is invoked again. 26 | void Set(Iterator* iter) { 27 | delete iter_; 28 | iter_ = iter; 29 | if (iter_ == nullptr) { 30 | valid_ = false; 31 | } else { 32 | Update(); 33 | } 34 | } 35 | 36 | // Iterator interface methods 37 | bool Valid() const { return valid_; } 38 | Slice key() const { 39 | assert(Valid()); 40 | return key_; 41 | } 42 | Slice value() const { 43 | assert(Valid()); 44 | return iter_->value(); 45 | } 46 | // Methods below require iter() != nullptr 47 | Status status() const { 48 | assert(iter_); 49 | return iter_->status(); 50 | } 51 | void Next() { 52 | assert(iter_); 53 | iter_->Next(); 54 | Update(); 55 | } 56 | void Prev() { 57 | assert(iter_); 58 | iter_->Prev(); 59 | Update(); 60 | } 61 | void Seek(const Slice& k) { 62 | assert(iter_); 63 | iter_->Seek(k); 64 | Update(); 65 | } 66 | void SeekToFirst() { 67 | assert(iter_); 68 | iter_->SeekToFirst(); 69 | Update(); 70 | } 71 | void SeekToLast() { 72 | assert(iter_); 73 | iter_->SeekToLast(); 74 | Update(); 75 | } 76 | 77 | private: 78 | void Update() { 79 | valid_ = iter_->Valid(); 80 | if (valid_) { 81 | key_ = iter_->key(); 82 | } 83 | } 84 | 85 | Iterator* iter_; 86 | bool valid_; 87 | Slice key_; 88 | }; 89 | 90 | } // namespace leveldb 91 | 92 | #endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 93 | -------------------------------------------------------------------------------- /table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 6 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Comparator; 11 | class Iterator; 12 | 13 | // Return an iterator that provided the union of the data in 14 | // children[0,n-1]. Takes ownership of the child iterators and 15 | // will delete them when the result iterator is deleted. 16 | // 17 | // The result does no duplicate suppression. I.e., if a particular 18 | // key is present in K child iterators, it will be yielded K times. 19 | // 20 | // REQUIRES: n >= 0 21 | Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children, 22 | int n); 23 | 24 | } // namespace leveldb 25 | 26 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 27 | -------------------------------------------------------------------------------- /table/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | 10 | namespace leveldb { 11 | 12 | struct ReadOptions; 13 | 14 | // Return a new two level iterator. A two-level iterator contains an 15 | // index iterator whose values point to a sequence of blocks where 16 | // each block is itself a sequence of key,value pairs. The returned 17 | // two-level iterator yields the concatenation of all key/value pairs 18 | // in the sequence of blocks. Takes ownership of "index_iter" and 19 | // will delete it when no longer needed. 20 | // 21 | // Uses a supplied function to convert an index_iter value into 22 | // an iterator over the contents of the corresponding block. 23 | Iterator* NewTwoLevelIterator( 24 | Iterator* index_iter, 25 | Iterator* (*block_function)(void* arg, const ReadOptions& options, 26 | const Slice& index_value), 27 | void* arg, const ReadOptions& options); 28 | 29 | } // namespace leveldb 30 | 31 | #endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 32 | -------------------------------------------------------------------------------- /util/arena.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | namespace leveldb { 8 | 9 | static const int kBlockSize = 4096; 10 | 11 | Arena::Arena() 12 | : alloc_ptr_(nullptr), alloc_bytes_remaining_(0), memory_usage_(0) {} 13 | 14 | Arena::~Arena() { 15 | for (size_t i = 0; i < blocks_.size(); i++) { 16 | delete[] blocks_[i]; 17 | } 18 | } 19 | 20 | char* Arena::AllocateFallback(size_t bytes) { 21 | if (bytes > kBlockSize / 4) { 22 | // Object is more than a quarter of our block size. Allocate it separately 23 | // to avoid wasting too much space in leftover bytes. 24 | char* result = AllocateNewBlock(bytes); 25 | return result; 26 | } 27 | 28 | // We waste the remaining space in the current block. 29 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 30 | alloc_bytes_remaining_ = kBlockSize; 31 | 32 | char* result = alloc_ptr_; 33 | alloc_ptr_ += bytes; 34 | alloc_bytes_remaining_ -= bytes; 35 | return result; 36 | } 37 | 38 | char* Arena::AllocateAligned(size_t bytes) { 39 | const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; 40 | static_assert((align & (align - 1)) == 0, 41 | "Pointer size should be a power of 2"); 42 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align - 1); 43 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 44 | size_t needed = bytes + slop; 45 | char* result; 46 | if (needed <= alloc_bytes_remaining_) { 47 | result = alloc_ptr_ + slop; 48 | alloc_ptr_ += needed; 49 | alloc_bytes_remaining_ -= needed; 50 | } else { 51 | // AllocateFallback always returned aligned memory 52 | result = AllocateFallback(bytes); 53 | } 54 | assert((reinterpret_cast(result) & (align - 1)) == 0); 55 | return result; 56 | } 57 | 58 | char* Arena::AllocateNewBlock(size_t block_bytes) { 59 | char* result = new char[block_bytes]; 60 | blocks_.push_back(result); 61 | memory_usage_.fetch_add(block_bytes + sizeof(char*), 62 | std::memory_order_relaxed); 63 | return result; 64 | } 65 | 66 | } // namespace leveldb 67 | -------------------------------------------------------------------------------- /util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace leveldb { 15 | 16 | class Arena { 17 | public: 18 | Arena(); 19 | 20 | Arena(const Arena&) = delete; 21 | Arena& operator=(const Arena&) = delete; 22 | 23 | ~Arena(); 24 | 25 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 26 | char* Allocate(size_t bytes); 27 | 28 | // Allocate memory with the normal alignment guarantees provided by malloc. 29 | char* AllocateAligned(size_t bytes); 30 | 31 | // Returns an estimate of the total memory usage of data allocated 32 | // by the arena. 33 | size_t MemoryUsage() const { 34 | return memory_usage_.load(std::memory_order_relaxed); 35 | } 36 | 37 | private: 38 | char* AllocateFallback(size_t bytes); 39 | char* AllocateNewBlock(size_t block_bytes); 40 | 41 | // Allocation state 42 | char* alloc_ptr_; 43 | size_t alloc_bytes_remaining_; 44 | 45 | // Array of new[] allocated memory blocks 46 | std::vector blocks_; 47 | 48 | // Total memory usage of the arena. 49 | // 50 | // TODO(costan): This member is accessed via atomics, but the others are 51 | // accessed without any locking. Is this OK? 52 | std::atomic memory_usage_; 53 | }; 54 | 55 | inline char* Arena::Allocate(size_t bytes) { 56 | // The semantics of what to return are a bit messy if we allow 57 | // 0-byte allocations, so we disallow them here (we don't need 58 | // them for our internal use). 59 | assert(bytes > 0); 60 | if (bytes <= alloc_bytes_remaining_) { 61 | char* result = alloc_ptr_; 62 | alloc_ptr_ += bytes; 63 | alloc_bytes_remaining_ -= bytes; 64 | return result; 65 | } 66 | return AllocateFallback(bytes); 67 | } 68 | 69 | } // namespace leveldb 70 | 71 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 72 | -------------------------------------------------------------------------------- /util/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | #include "gtest/gtest.h" 8 | #include "util/random.h" 9 | 10 | namespace leveldb { 11 | 12 | TEST(ArenaTest, Empty) { Arena arena; } 13 | 14 | TEST(ArenaTest, Simple) { 15 | std::vector> allocated; 16 | Arena arena; 17 | const int N = 100000; 18 | size_t bytes = 0; 19 | Random rnd(301); 20 | for (int i = 0; i < N; i++) { 21 | size_t s; 22 | if (i % (N / 10) == 0) { 23 | s = i; 24 | } else { 25 | s = rnd.OneIn(4000) 26 | ? rnd.Uniform(6000) 27 | : (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 28 | } 29 | if (s == 0) { 30 | // Our arena disallows size 0 allocations. 31 | s = 1; 32 | } 33 | char* r; 34 | if (rnd.OneIn(10)) { 35 | r = arena.AllocateAligned(s); 36 | } else { 37 | r = arena.Allocate(s); 38 | } 39 | 40 | for (size_t b = 0; b < s; b++) { 41 | // Fill the "i"th allocation with a known bit pattern 42 | r[b] = i % 256; 43 | } 44 | bytes += s; 45 | allocated.push_back(std::make_pair(s, r)); 46 | ASSERT_GE(arena.MemoryUsage(), bytes); 47 | if (i > N / 10) { 48 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 49 | } 50 | } 51 | for (size_t i = 0; i < allocated.size(); i++) { 52 | size_t num_bytes = allocated[i].first; 53 | const char* p = allocated[i].second; 54 | for (size_t b = 0; b < num_bytes; b++) { 55 | // Check the "i"th allocation for the known bit pattern 56 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 57 | } 58 | } 59 | } 60 | 61 | } // namespace leveldb 62 | -------------------------------------------------------------------------------- /util/bloom.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | #include "leveldb/slice.h" 8 | #include "util/hash.h" 9 | 10 | namespace leveldb { 11 | 12 | namespace { 13 | static uint32_t BloomHash(const Slice& key) { 14 | return Hash(key.data(), key.size(), 0xbc9f1d34); 15 | } 16 | 17 | class BloomFilterPolicy : public FilterPolicy { 18 | public: 19 | explicit BloomFilterPolicy(int bits_per_key) : bits_per_key_(bits_per_key) { 20 | // We intentionally round down to reduce probing cost a little bit 21 | k_ = static_cast(bits_per_key * 0.69); // 0.69 =~ ln(2) 22 | if (k_ < 1) k_ = 1; 23 | if (k_ > 30) k_ = 30; 24 | } 25 | 26 | const char* Name() const override { return "leveldb.BuiltinBloomFilter2"; } 27 | 28 | void CreateFilter(const Slice* keys, int n, std::string* dst) const override { 29 | // Compute bloom filter size (in both bits and bytes) 30 | size_t bits = n * bits_per_key_; 31 | 32 | // For small n, we can see a very high false positive rate. Fix it 33 | // by enforcing a minimum bloom filter length. 34 | if (bits < 64) bits = 64; 35 | 36 | size_t bytes = (bits + 7) / 8; 37 | bits = bytes * 8; 38 | 39 | const size_t init_size = dst->size(); 40 | dst->resize(init_size + bytes, 0); 41 | dst->push_back(static_cast(k_)); // Remember # of probes in filter 42 | char* array = &(*dst)[init_size]; 43 | for (int i = 0; i < n; i++) { 44 | // Use double-hashing to generate a sequence of hash values. 45 | // See analysis in [Kirsch,Mitzenmacher 2006]. 46 | uint32_t h = BloomHash(keys[i]); 47 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 48 | for (size_t j = 0; j < k_; j++) { 49 | const uint32_t bitpos = h % bits; 50 | array[bitpos / 8] |= (1 << (bitpos % 8)); 51 | h += delta; 52 | } 53 | } 54 | } 55 | 56 | bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const override { 57 | const size_t len = bloom_filter.size(); 58 | if (len < 2) return false; 59 | 60 | const char* array = bloom_filter.data(); 61 | const size_t bits = (len - 1) * 8; 62 | 63 | // Use the encoded k so that we can read filters generated by 64 | // bloom filters created using different parameters. 65 | const size_t k = array[len - 1]; 66 | if (k > 30) { 67 | // Reserved for potentially new encodings for short bloom filters. 68 | // Consider it a match. 69 | return true; 70 | } 71 | 72 | uint32_t h = BloomHash(key); 73 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 74 | for (size_t j = 0; j < k; j++) { 75 | const uint32_t bitpos = h % bits; 76 | if ((array[bitpos / 8] & (1 << (bitpos % 8))) == 0) return false; 77 | h += delta; 78 | } 79 | return true; 80 | } 81 | 82 | private: 83 | size_t bits_per_key_; 84 | size_t k_; 85 | }; 86 | } // namespace 87 | 88 | const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { 89 | return new BloomFilterPolicy(bits_per_key); 90 | } 91 | 92 | } // namespace leveldb 93 | -------------------------------------------------------------------------------- /util/bloom_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "gtest/gtest.h" 6 | #include "leveldb/filter_policy.h" 7 | #include "util/coding.h" 8 | #include "util/logging.h" 9 | #include "util/testutil.h" 10 | 11 | namespace leveldb { 12 | 13 | static const int kVerbose = 1; 14 | 15 | static Slice Key(int i, char* buffer) { 16 | EncodeFixed32(buffer, i); 17 | return Slice(buffer, sizeof(uint32_t)); 18 | } 19 | 20 | class BloomTest : public testing::Test { 21 | public: 22 | BloomTest() : policy_(NewBloomFilterPolicy(10)) {} 23 | 24 | ~BloomTest() { delete policy_; } 25 | 26 | void Reset() { 27 | keys_.clear(); 28 | filter_.clear(); 29 | } 30 | 31 | void Add(const Slice& s) { keys_.push_back(s.ToString()); } 32 | 33 | void Build() { 34 | std::vector key_slices; 35 | for (size_t i = 0; i < keys_.size(); i++) { 36 | key_slices.push_back(Slice(keys_[i])); 37 | } 38 | filter_.clear(); 39 | policy_->CreateFilter(&key_slices[0], static_cast(key_slices.size()), 40 | &filter_); 41 | keys_.clear(); 42 | if (kVerbose >= 2) DumpFilter(); 43 | } 44 | 45 | size_t FilterSize() const { return filter_.size(); } 46 | 47 | void DumpFilter() { 48 | std::fprintf(stderr, "F("); 49 | for (size_t i = 0; i + 1 < filter_.size(); i++) { 50 | const unsigned int c = static_cast(filter_[i]); 51 | for (int j = 0; j < 8; j++) { 52 | std::fprintf(stderr, "%c", (c & (1 << j)) ? '1' : '.'); 53 | } 54 | } 55 | std::fprintf(stderr, ")\n"); 56 | } 57 | 58 | bool Matches(const Slice& s) { 59 | if (!keys_.empty()) { 60 | Build(); 61 | } 62 | return policy_->KeyMayMatch(s, filter_); 63 | } 64 | 65 | double FalsePositiveRate() { 66 | char buffer[sizeof(int)]; 67 | int result = 0; 68 | for (int i = 0; i < 10000; i++) { 69 | if (Matches(Key(i + 1000000000, buffer))) { 70 | result++; 71 | } 72 | } 73 | return result / 10000.0; 74 | } 75 | 76 | private: 77 | const FilterPolicy* policy_; 78 | std::string filter_; 79 | std::vector keys_; 80 | }; 81 | 82 | TEST_F(BloomTest, EmptyFilter) { 83 | ASSERT_TRUE(!Matches("hello")); 84 | ASSERT_TRUE(!Matches("world")); 85 | } 86 | 87 | TEST_F(BloomTest, Small) { 88 | Add("hello"); 89 | Add("world"); 90 | ASSERT_TRUE(Matches("hello")); 91 | ASSERT_TRUE(Matches("world")); 92 | ASSERT_TRUE(!Matches("x")); 93 | ASSERT_TRUE(!Matches("foo")); 94 | } 95 | 96 | static int NextLength(int length) { 97 | if (length < 10) { 98 | length += 1; 99 | } else if (length < 100) { 100 | length += 10; 101 | } else if (length < 1000) { 102 | length += 100; 103 | } else { 104 | length += 1000; 105 | } 106 | return length; 107 | } 108 | 109 | TEST_F(BloomTest, VaryingLengths) { 110 | char buffer[sizeof(int)]; 111 | 112 | // Count number of filters that significantly exceed the false positive rate 113 | int mediocre_filters = 0; 114 | int good_filters = 0; 115 | 116 | for (int length = 1; length <= 10000; length = NextLength(length)) { 117 | Reset(); 118 | for (int i = 0; i < length; i++) { 119 | Add(Key(i, buffer)); 120 | } 121 | Build(); 122 | 123 | ASSERT_LE(FilterSize(), static_cast((length * 10 / 8) + 40)) 124 | << length; 125 | 126 | // All added keys must match 127 | for (int i = 0; i < length; i++) { 128 | ASSERT_TRUE(Matches(Key(i, buffer))) 129 | << "Length " << length << "; key " << i; 130 | } 131 | 132 | // Check false positive rate 133 | double rate = FalsePositiveRate(); 134 | if (kVerbose >= 1) { 135 | std::fprintf(stderr, 136 | "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", 137 | rate * 100.0, length, static_cast(FilterSize())); 138 | } 139 | ASSERT_LE(rate, 0.02); // Must not be over 2% 140 | if (rate > 0.0125) 141 | mediocre_filters++; // Allowed, but not too often 142 | else 143 | good_filters++; 144 | } 145 | if (kVerbose >= 1) { 146 | std::fprintf(stderr, "Filters: %d good, %d mediocre\n", good_filters, 147 | mediocre_filters); 148 | } 149 | ASSERT_LE(mediocre_filters, good_filters / 5); 150 | } 151 | 152 | // Different bits-per-byte 153 | 154 | } // namespace leveldb 155 | -------------------------------------------------------------------------------- /util/coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/coding.h" 6 | 7 | namespace leveldb { 8 | 9 | void PutFixed32(std::string* dst, uint32_t value) { 10 | char buf[sizeof(value)]; 11 | EncodeFixed32(buf, value); 12 | dst->append(buf, sizeof(buf)); 13 | } 14 | 15 | void PutFixed64(std::string* dst, uint64_t value) { 16 | char buf[sizeof(value)]; 17 | EncodeFixed64(buf, value); 18 | dst->append(buf, sizeof(buf)); 19 | } 20 | 21 | char* EncodeVarint32(char* dst, uint32_t v) { 22 | // Operate on characters as unsigneds 23 | uint8_t* ptr = reinterpret_cast(dst); 24 | static const int B = 128; 25 | if (v < (1 << 7)) { 26 | *(ptr++) = v; 27 | } else if (v < (1 << 14)) { 28 | *(ptr++) = v | B; 29 | *(ptr++) = v >> 7; 30 | } else if (v < (1 << 21)) { 31 | *(ptr++) = v | B; 32 | *(ptr++) = (v >> 7) | B; 33 | *(ptr++) = v >> 14; 34 | } else if (v < (1 << 28)) { 35 | *(ptr++) = v | B; 36 | *(ptr++) = (v >> 7) | B; 37 | *(ptr++) = (v >> 14) | B; 38 | *(ptr++) = v >> 21; 39 | } else { 40 | *(ptr++) = v | B; 41 | *(ptr++) = (v >> 7) | B; 42 | *(ptr++) = (v >> 14) | B; 43 | *(ptr++) = (v >> 21) | B; 44 | *(ptr++) = v >> 28; 45 | } 46 | return reinterpret_cast(ptr); 47 | } 48 | 49 | void PutVarint32(std::string* dst, uint32_t v) { 50 | char buf[5]; 51 | char* ptr = EncodeVarint32(buf, v); 52 | dst->append(buf, ptr - buf); 53 | } 54 | 55 | char* EncodeVarint64(char* dst, uint64_t v) { 56 | static const int B = 128; 57 | uint8_t* ptr = reinterpret_cast(dst); 58 | while (v >= B) { 59 | *(ptr++) = v | B; 60 | v >>= 7; 61 | } 62 | *(ptr++) = static_cast(v); 63 | return reinterpret_cast(ptr); 64 | } 65 | 66 | void PutVarint64(std::string* dst, uint64_t v) { 67 | char buf[10]; 68 | char* ptr = EncodeVarint64(buf, v); 69 | dst->append(buf, ptr - buf); 70 | } 71 | 72 | void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { 73 | PutVarint32(dst, value.size()); 74 | dst->append(value.data(), value.size()); 75 | } 76 | 77 | int VarintLength(uint64_t v) { 78 | int len = 1; 79 | while (v >= 128) { 80 | v >>= 7; 81 | len++; 82 | } 83 | return len; 84 | } 85 | 86 | const char* GetVarint32PtrFallback(const char* p, const char* limit, 87 | uint32_t* value) { 88 | uint32_t result = 0; 89 | for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { 90 | uint32_t byte = *(reinterpret_cast(p)); 91 | p++; 92 | if (byte & 128) { 93 | // More bytes are present 94 | result |= ((byte & 127) << shift); 95 | } else { 96 | result |= (byte << shift); 97 | *value = result; 98 | return reinterpret_cast(p); 99 | } 100 | } 101 | return nullptr; 102 | } 103 | 104 | bool GetVarint32(Slice* input, uint32_t* value) { 105 | const char* p = input->data(); 106 | const char* limit = p + input->size(); 107 | const char* q = GetVarint32Ptr(p, limit, value); 108 | if (q == nullptr) { 109 | return false; 110 | } else { 111 | *input = Slice(q, limit - q); 112 | return true; 113 | } 114 | } 115 | 116 | const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { 117 | uint64_t result = 0; 118 | for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { 119 | uint64_t byte = *(reinterpret_cast(p)); 120 | p++; 121 | if (byte & 128) { 122 | // More bytes are present 123 | result |= ((byte & 127) << shift); 124 | } else { 125 | result |= (byte << shift); 126 | *value = result; 127 | return reinterpret_cast(p); 128 | } 129 | } 130 | return nullptr; 131 | } 132 | 133 | bool GetVarint64(Slice* input, uint64_t* value) { 134 | const char* p = input->data(); 135 | const char* limit = p + input->size(); 136 | const char* q = GetVarint64Ptr(p, limit, value); 137 | if (q == nullptr) { 138 | return false; 139 | } else { 140 | *input = Slice(q, limit - q); 141 | return true; 142 | } 143 | } 144 | 145 | bool GetLengthPrefixedSlice(Slice* input, Slice* result) { 146 | uint32_t len; 147 | if (GetVarint32(input, &len) && input->size() >= len) { 148 | *result = Slice(input->data(), len); 149 | input->remove_prefix(len); 150 | return true; 151 | } else { 152 | return false; 153 | } 154 | } 155 | 156 | } // namespace leveldb 157 | -------------------------------------------------------------------------------- /util/comparator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/comparator.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "leveldb/slice.h" 13 | #include "util/logging.h" 14 | #include "util/no_destructor.h" 15 | 16 | namespace leveldb { 17 | 18 | Comparator::~Comparator() = default; 19 | 20 | namespace { 21 | class BytewiseComparatorImpl : public Comparator { 22 | public: 23 | BytewiseComparatorImpl() = default; 24 | 25 | const char* Name() const override { return "leveldb.BytewiseComparator"; } 26 | 27 | int Compare(const Slice& a, const Slice& b) const override { 28 | return a.compare(b); 29 | } 30 | 31 | void FindShortestSeparator(std::string* start, 32 | const Slice& limit) const override { 33 | // Find length of common prefix 34 | size_t min_length = std::min(start->size(), limit.size()); 35 | size_t diff_index = 0; 36 | while ((diff_index < min_length) && 37 | ((*start)[diff_index] == limit[diff_index])) { 38 | diff_index++; 39 | } 40 | 41 | if (diff_index >= min_length) { 42 | // Do not shorten if one string is a prefix of the other 43 | } else { 44 | uint8_t diff_byte = static_cast((*start)[diff_index]); 45 | if (diff_byte < static_cast(0xff) && 46 | diff_byte + 1 < static_cast(limit[diff_index])) { 47 | (*start)[diff_index]++; 48 | start->resize(diff_index + 1); 49 | assert(Compare(*start, limit) < 0); 50 | } 51 | } 52 | } 53 | 54 | void FindShortSuccessor(std::string* key) const override { 55 | // Find first character that can be incremented 56 | size_t n = key->size(); 57 | for (size_t i = 0; i < n; i++) { 58 | const uint8_t byte = (*key)[i]; 59 | if (byte != static_cast(0xff)) { 60 | (*key)[i] = byte + 1; 61 | key->resize(i + 1); 62 | return; 63 | } 64 | } 65 | // *key is a run of 0xffs. Leave it alone. 66 | } 67 | }; 68 | } // namespace 69 | 70 | const Comparator* BytewiseComparator() { 71 | static NoDestructor singleton; 72 | return singleton.get(); 73 | } 74 | 75 | } // namespace leveldb 76 | -------------------------------------------------------------------------------- /util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ 6 | #define STORAGE_LEVELDB_UTIL_CRC32C_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace crc32c { 13 | 14 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 15 | // crc32c of some string A. Extend() is often used to maintain the 16 | // crc32c of a stream of data. 17 | uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 18 | 19 | // Return the crc32c of data[0,n-1] 20 | inline uint32_t Value(const char* data, size_t n) { return Extend(0, data, n); } 21 | 22 | static const uint32_t kMaskDelta = 0xa282ead8ul; 23 | 24 | // Return a masked representation of crc. 25 | // 26 | // Motivation: it is problematic to compute the CRC of a string that 27 | // contains embedded CRCs. Therefore we recommend that CRCs stored 28 | // somewhere (e.g., in files) should be masked before being stored. 29 | inline uint32_t Mask(uint32_t crc) { 30 | // Rotate right by 15 bits and add a constant. 31 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 32 | } 33 | 34 | // Return the crc whose masked representation is masked_crc. 35 | inline uint32_t Unmask(uint32_t masked_crc) { 36 | uint32_t rot = masked_crc - kMaskDelta; 37 | return ((rot >> 17) | (rot << 15)); 38 | } 39 | 40 | } // namespace crc32c 41 | } // namespace leveldb 42 | 43 | #endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ 44 | -------------------------------------------------------------------------------- /util/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/crc32c.h" 6 | 7 | #include "gtest/gtest.h" 8 | 9 | namespace leveldb { 10 | namespace crc32c { 11 | 12 | TEST(CRC, StandardResults) { 13 | // From rfc3720 section B.4. 14 | char buf[32]; 15 | 16 | memset(buf, 0, sizeof(buf)); 17 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 18 | 19 | memset(buf, 0xff, sizeof(buf)); 20 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 21 | 22 | for (int i = 0; i < 32; i++) { 23 | buf[i] = i; 24 | } 25 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 26 | 27 | for (int i = 0; i < 32; i++) { 28 | buf[i] = 31 - i; 29 | } 30 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 31 | 32 | uint8_t data[48] = { 33 | 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 34 | 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 35 | 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 37 | }; 38 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 39 | } 40 | 41 | TEST(CRC, Values) { ASSERT_NE(Value("a", 1), Value("foo", 3)); } 42 | 43 | TEST(CRC, Extend) { 44 | ASSERT_EQ(Value("hello world", 11), Extend(Value("hello ", 6), "world", 5)); 45 | } 46 | 47 | TEST(CRC, Mask) { 48 | uint32_t crc = Value("foo", 3); 49 | ASSERT_NE(crc, Mask(crc)); 50 | ASSERT_NE(crc, Mask(Mask(crc))); 51 | ASSERT_EQ(crc, Unmask(Mask(crc))); 52 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 53 | } 54 | 55 | } // namespace crc32c 56 | } // namespace leveldb 57 | -------------------------------------------------------------------------------- /util/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/env.h" 6 | 7 | #include 8 | 9 | // This workaround can be removed when leveldb::Env::DeleteFile is removed. 10 | // See env.h for justification. 11 | #if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) 12 | #undef DeleteFile 13 | #endif 14 | 15 | namespace leveldb { 16 | 17 | Env::Env() = default; 18 | 19 | Env::~Env() = default; 20 | 21 | Status Env::NewAppendableFile(const std::string& fname, WritableFile** result) { 22 | return Status::NotSupported("NewAppendableFile", fname); 23 | } 24 | 25 | Status Env::RemoveDir(const std::string& dirname) { return DeleteDir(dirname); } 26 | Status Env::DeleteDir(const std::string& dirname) { return RemoveDir(dirname); } 27 | 28 | Status Env::RemoveFile(const std::string& fname) { return DeleteFile(fname); } 29 | Status Env::DeleteFile(const std::string& fname) { return RemoveFile(fname); } 30 | 31 | SequentialFile::~SequentialFile() = default; 32 | 33 | RandomAccessFile::~RandomAccessFile() = default; 34 | 35 | WritableFile::~WritableFile() = default; 36 | 37 | Logger::~Logger() = default; 38 | 39 | FileLock::~FileLock() = default; 40 | 41 | void Log(Logger* info_log, const char* format, ...) { 42 | if (info_log != nullptr) { 43 | std::va_list ap; 44 | va_start(ap, format); 45 | info_log->Logv(format, ap); 46 | va_end(ap); 47 | } 48 | } 49 | 50 | static Status DoWriteStringToFile(Env* env, const Slice& data, 51 | const std::string& fname, bool should_sync) { 52 | WritableFile* file; 53 | Status s = env->NewWritableFile(fname, &file); 54 | if (!s.ok()) { 55 | return s; 56 | } 57 | s = file->Append(data); 58 | if (s.ok() && should_sync) { 59 | s = file->Sync(); 60 | } 61 | if (s.ok()) { 62 | s = file->Close(); 63 | } 64 | delete file; // Will auto-close if we did not close above 65 | if (!s.ok()) { 66 | env->RemoveFile(fname); 67 | } 68 | return s; 69 | } 70 | 71 | Status WriteStringToFile(Env* env, const Slice& data, 72 | const std::string& fname) { 73 | return DoWriteStringToFile(env, data, fname, false); 74 | } 75 | 76 | Status WriteStringToFileSync(Env* env, const Slice& data, 77 | const std::string& fname) { 78 | return DoWriteStringToFile(env, data, fname, true); 79 | } 80 | 81 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { 82 | data->clear(); 83 | SequentialFile* file; 84 | Status s = env->NewSequentialFile(fname, &file); 85 | if (!s.ok()) { 86 | return s; 87 | } 88 | static const int kBufferSize = 8192; 89 | char* space = new char[kBufferSize]; 90 | while (true) { 91 | Slice fragment; 92 | s = file->Read(kBufferSize, &fragment, space); 93 | if (!s.ok()) { 94 | break; 95 | } 96 | data->append(fragment.data(), fragment.size()); 97 | if (fragment.empty()) { 98 | break; 99 | } 100 | } 101 | delete[] space; 102 | delete file; 103 | return s; 104 | } 105 | 106 | EnvWrapper::~EnvWrapper() {} 107 | 108 | } // namespace leveldb 109 | -------------------------------------------------------------------------------- /util/env_posix_test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 6 | #define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class EnvPosixTest; 11 | 12 | // A helper for the POSIX Env to facilitate testing. 13 | class EnvPosixTestHelper { 14 | private: 15 | friend class EnvPosixTest; 16 | 17 | // Set the maximum number of read-only files that will be opened. 18 | // Must be called before creating an Env. 19 | static void SetReadOnlyFDLimit(int limit); 20 | 21 | // Set the maximum number of read-only files that will be mapped via mmap. 22 | // Must be called before creating an Env. 23 | static void SetReadOnlyMMapLimit(int limit); 24 | }; 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 29 | -------------------------------------------------------------------------------- /util/env_windows_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "gtest/gtest.h" 6 | #include "leveldb/env.h" 7 | #include "port/port.h" 8 | #include "util/env_windows_test_helper.h" 9 | #include "util/testutil.h" 10 | 11 | namespace leveldb { 12 | 13 | static const int kMMapLimit = 4; 14 | 15 | class EnvWindowsTest : public testing::Test { 16 | public: 17 | static void SetFileLimits(int mmap_limit) { 18 | EnvWindowsTestHelper::SetReadOnlyMMapLimit(mmap_limit); 19 | } 20 | 21 | EnvWindowsTest() : env_(Env::Default()) {} 22 | 23 | Env* env_; 24 | }; 25 | 26 | TEST_F(EnvWindowsTest, TestOpenOnRead) { 27 | // Write some test data to a single file that will be opened |n| times. 28 | std::string test_dir; 29 | ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); 30 | std::string test_file = test_dir + "/open_on_read.txt"; 31 | 32 | FILE* f = std::fopen(test_file.c_str(), "w"); 33 | ASSERT_TRUE(f != nullptr); 34 | const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; 35 | fputs(kFileData, f); 36 | std::fclose(f); 37 | 38 | // Open test file some number above the sum of the two limits to force 39 | // leveldb::WindowsEnv to switch from mapping the file into memory 40 | // to basic file reading. 41 | const int kNumFiles = kMMapLimit + 5; 42 | leveldb::RandomAccessFile* files[kNumFiles] = {0}; 43 | for (int i = 0; i < kNumFiles; i++) { 44 | ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i])); 45 | } 46 | char scratch; 47 | Slice read_result; 48 | for (int i = 0; i < kNumFiles; i++) { 49 | ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch)); 50 | ASSERT_EQ(kFileData[i], read_result[0]); 51 | } 52 | for (int i = 0; i < kNumFiles; i++) { 53 | delete files[i]; 54 | } 55 | ASSERT_LEVELDB_OK(env_->RemoveFile(test_file)); 56 | } 57 | 58 | } // namespace leveldb 59 | 60 | int main(int argc, char** argv) { 61 | // All tests currently run with the same read-only file limits. 62 | leveldb::EnvWindowsTest::SetFileLimits(leveldb::kMMapLimit); 63 | testing::InitGoogleTest(&argc, argv); 64 | return RUN_ALL_TESTS(); 65 | } 66 | -------------------------------------------------------------------------------- /util/env_windows_test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 (c) The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ 6 | #define STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class EnvWindowsTest; 11 | 12 | // A helper for the Windows Env to facilitate testing. 13 | class EnvWindowsTestHelper { 14 | private: 15 | friend class CorruptionTest; 16 | friend class EnvWindowsTest; 17 | 18 | // Set the maximum number of read-only files that will be mapped via mmap. 19 | // Must be called before creating an Env. 20 | static void SetReadOnlyMMapLimit(int limit); 21 | }; 22 | 23 | } // namespace leveldb 24 | 25 | #endif // STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ 26 | -------------------------------------------------------------------------------- /util/filter_policy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | namespace leveldb { 8 | 9 | FilterPolicy::~FilterPolicy() {} 10 | 11 | } // namespace leveldb 12 | -------------------------------------------------------------------------------- /util/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/hash.h" 6 | 7 | #include 8 | 9 | #include "util/coding.h" 10 | 11 | // The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through 12 | // between switch labels. The real definition should be provided externally. 13 | // This one is a fallback version for unsupported compilers. 14 | #ifndef FALLTHROUGH_INTENDED 15 | #define FALLTHROUGH_INTENDED \ 16 | do { \ 17 | } while (0) 18 | #endif 19 | 20 | namespace leveldb { 21 | 22 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 23 | // Similar to murmur hash 24 | const uint32_t m = 0xc6a4a793; 25 | const uint32_t r = 24; 26 | const char* limit = data + n; 27 | uint32_t h = seed ^ (n * m); 28 | 29 | // Pick up four bytes at a time 30 | while (limit - data >= 4) { 31 | uint32_t w = DecodeFixed32(data); 32 | data += 4; 33 | h += w; 34 | h *= m; 35 | h ^= (h >> 16); 36 | } 37 | 38 | // Pick up remaining bytes 39 | switch (limit - data) { 40 | case 3: 41 | h += static_cast(data[2]) << 16; 42 | FALLTHROUGH_INTENDED; 43 | case 2: 44 | h += static_cast(data[1]) << 8; 45 | FALLTHROUGH_INTENDED; 46 | case 1: 47 | h += static_cast(data[0]); 48 | h *= m; 49 | h ^= (h >> r); 50 | break; 51 | } 52 | return h; 53 | } 54 | 55 | } // namespace leveldb 56 | -------------------------------------------------------------------------------- /util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Simple hash function used for internal data structures 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 8 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 9 | 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | uint32_t Hash(const char* data, size_t n, uint32_t seed); 16 | 17 | } // namespace leveldb 18 | 19 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 20 | -------------------------------------------------------------------------------- /util/hash_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/hash.h" 6 | 7 | #include "gtest/gtest.h" 8 | 9 | namespace leveldb { 10 | 11 | TEST(HASH, SignedUnsignedIssue) { 12 | const uint8_t data1[1] = {0x62}; 13 | const uint8_t data2[2] = {0xc3, 0x97}; 14 | const uint8_t data3[3] = {0xe2, 0x99, 0xa5}; 15 | const uint8_t data4[4] = {0xe1, 0x80, 0xb9, 0x32}; 16 | const uint8_t data5[48] = { 17 | 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 18 | 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 19 | 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, 20 | 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 21 | }; 22 | 23 | ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); 24 | ASSERT_EQ( 25 | Hash(reinterpret_cast(data1), sizeof(data1), 0xbc9f1d34), 26 | 0xef1345c4); 27 | ASSERT_EQ( 28 | Hash(reinterpret_cast(data2), sizeof(data2), 0xbc9f1d34), 29 | 0x5b663814); 30 | ASSERT_EQ( 31 | Hash(reinterpret_cast(data3), sizeof(data3), 0xbc9f1d34), 32 | 0x323c078f); 33 | ASSERT_EQ( 34 | Hash(reinterpret_cast(data4), sizeof(data4), 0xbc9f1d34), 35 | 0xed21633a); 36 | ASSERT_EQ( 37 | Hash(reinterpret_cast(data5), sizeof(data5), 0x12345678), 38 | 0xf333dabb); 39 | } 40 | 41 | } // namespace leveldb 42 | -------------------------------------------------------------------------------- /util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 6 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Histogram { 13 | public: 14 | Histogram() {} 15 | ~Histogram() {} 16 | 17 | void Clear(); 18 | void Add(double value); 19 | void Merge(const Histogram& other); 20 | 21 | std::string ToString() const; 22 | 23 | private: 24 | enum { kNumBuckets = 154 }; 25 | 26 | double Median() const; 27 | double Percentile(double p) const; 28 | double Average() const; 29 | double StandardDeviation() const; 30 | 31 | static const double kBucketLimit[kNumBuckets]; 32 | 33 | double min_; 34 | double max_; 35 | double num_; 36 | double sum_; 37 | double sum_squares_; 38 | 39 | double buckets_[kNumBuckets]; 40 | }; 41 | 42 | } // namespace leveldb 43 | 44 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 45 | -------------------------------------------------------------------------------- /util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "leveldb/env.h" 13 | #include "leveldb/slice.h" 14 | 15 | namespace leveldb { 16 | 17 | void AppendNumberTo(std::string* str, uint64_t num) { 18 | char buf[30]; 19 | std::snprintf(buf, sizeof(buf), "%llu", static_cast(num)); 20 | str->append(buf); 21 | } 22 | 23 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 24 | for (size_t i = 0; i < value.size(); i++) { 25 | char c = value[i]; 26 | if (c >= ' ' && c <= '~') { 27 | str->push_back(c); 28 | } else { 29 | char buf[10]; 30 | std::snprintf(buf, sizeof(buf), "\\x%02x", 31 | static_cast(c) & 0xff); 32 | str->append(buf); 33 | } 34 | } 35 | } 36 | 37 | std::string NumberToString(uint64_t num) { 38 | std::string r; 39 | AppendNumberTo(&r, num); 40 | return r; 41 | } 42 | 43 | std::string EscapeString(const Slice& value) { 44 | std::string r; 45 | AppendEscapedStringTo(&r, value); 46 | return r; 47 | } 48 | 49 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 50 | // Constants that will be optimized away. 51 | constexpr const uint64_t kMaxUint64 = std::numeric_limits::max(); 52 | constexpr const char kLastDigitOfMaxUint64 = 53 | '0' + static_cast(kMaxUint64 % 10); 54 | 55 | uint64_t value = 0; 56 | 57 | // reinterpret_cast-ing from char* to uint8_t* to avoid signedness. 58 | const uint8_t* start = reinterpret_cast(in->data()); 59 | 60 | const uint8_t* end = start + in->size(); 61 | const uint8_t* current = start; 62 | for (; current != end; ++current) { 63 | const uint8_t ch = *current; 64 | if (ch < '0' || ch > '9') break; 65 | 66 | // Overflow check. 67 | // kMaxUint64 / 10 is also constant and will be optimized away. 68 | if (value > kMaxUint64 / 10 || 69 | (value == kMaxUint64 / 10 && ch > kLastDigitOfMaxUint64)) { 70 | return false; 71 | } 72 | 73 | value = (value * 10) + (ch - '0'); 74 | } 75 | 76 | *val = value; 77 | const size_t digits_consumed = current - start; 78 | in->remove_prefix(digits_consumed); 79 | return digits_consumed != 0; 80 | } 81 | 82 | } // namespace leveldb 83 | -------------------------------------------------------------------------------- /util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "port/port.h" 16 | 17 | namespace leveldb { 18 | 19 | class Slice; 20 | class WritableFile; 21 | 22 | // Append a human-readable printout of "num" to *str 23 | void AppendNumberTo(std::string* str, uint64_t num); 24 | 25 | // Append a human-readable printout of "value" to *str. 26 | // Escapes any non-printable characters found in "value". 27 | void AppendEscapedStringTo(std::string* str, const Slice& value); 28 | 29 | // Return a human-readable printout of "num" 30 | std::string NumberToString(uint64_t num); 31 | 32 | // Return a human-readable version of "value". 33 | // Escapes any non-printable characters found in "value". 34 | std::string EscapeString(const Slice& value); 35 | 36 | // Parse a human-readable number from "*in" into *value. On success, 37 | // advances "*in" past the consumed number and sets "*val" to the 38 | // numeric value. Otherwise, returns false and leaves *in in an 39 | // unspecified state. 40 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 41 | 42 | } // namespace leveldb 43 | 44 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 45 | -------------------------------------------------------------------------------- /util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | #include "port/thread_annotations.h" 10 | 11 | namespace leveldb { 12 | 13 | // Helper class that locks a mutex on construction and unlocks the mutex when 14 | // the destructor of the MutexLock object is invoked. 15 | // 16 | // Typical usage: 17 | // 18 | // void MyClass::MyMethod() { 19 | // MutexLock l(&mu_); // mu_ is an instance variable 20 | // ... some complex code, possibly with multiple return paths ... 21 | // } 22 | 23 | class SCOPED_LOCKABLE MutexLock { 24 | public: 25 | explicit MutexLock(port::Mutex* mu) EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { 26 | this->mu_->Lock(); 27 | } 28 | ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 29 | 30 | MutexLock(const MutexLock&) = delete; 31 | MutexLock& operator=(const MutexLock&) = delete; 32 | 33 | private: 34 | port::Mutex* const mu_; 35 | }; 36 | 37 | } // namespace leveldb 38 | 39 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 40 | -------------------------------------------------------------------------------- /util/no_destructor.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ 6 | #define STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | namespace leveldb { 13 | 14 | // Wraps an instance whose destructor is never called. 15 | // 16 | // This is intended for use with function-level static variables. 17 | template 18 | class NoDestructor { 19 | public: 20 | template 21 | explicit NoDestructor(ConstructorArgTypes&&... constructor_args) { 22 | static_assert(sizeof(instance_storage_) >= sizeof(InstanceType), 23 | "instance_storage_ is not large enough to hold the instance"); 24 | static_assert(std::is_standard_layout_v>); 25 | static_assert( 26 | offsetof(NoDestructor, instance_storage_) % alignof(InstanceType) == 0, 27 | "instance_storage_ does not meet the instance's alignment requirement"); 28 | static_assert( 29 | alignof(NoDestructor) % alignof(InstanceType) == 0, 30 | "instance_storage_ does not meet the instance's alignment requirement"); 31 | new (instance_storage_) 32 | InstanceType(std::forward(constructor_args)...); 33 | } 34 | 35 | ~NoDestructor() = default; 36 | 37 | NoDestructor(const NoDestructor&) = delete; 38 | NoDestructor& operator=(const NoDestructor&) = delete; 39 | 40 | InstanceType* get() { 41 | return reinterpret_cast(&instance_storage_); 42 | } 43 | 44 | private: 45 | alignas(InstanceType) char instance_storage_[sizeof(InstanceType)]; 46 | }; 47 | 48 | } // namespace leveldb 49 | 50 | #endif // STORAGE_LEVELDB_UTIL_NO_DESTRUCTOR_H_ 51 | -------------------------------------------------------------------------------- /util/no_destructor_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/no_destructor.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "gtest/gtest.h" 12 | 13 | namespace leveldb { 14 | 15 | namespace { 16 | 17 | struct DoNotDestruct { 18 | public: 19 | DoNotDestruct(uint32_t a, uint64_t b) : a(a), b(b) {} 20 | ~DoNotDestruct() { std::abort(); } 21 | 22 | // Used to check constructor argument forwarding. 23 | uint32_t a; 24 | uint64_t b; 25 | }; 26 | 27 | constexpr const uint32_t kGoldenA = 0xdeadbeef; 28 | constexpr const uint64_t kGoldenB = 0xaabbccddeeffaabb; 29 | 30 | } // namespace 31 | 32 | TEST(NoDestructorTest, StackInstance) { 33 | NoDestructor instance(kGoldenA, kGoldenB); 34 | ASSERT_EQ(kGoldenA, instance.get()->a); 35 | ASSERT_EQ(kGoldenB, instance.get()->b); 36 | } 37 | 38 | TEST(NoDestructorTest, StaticInstance) { 39 | static NoDestructor instance(kGoldenA, kGoldenB); 40 | ASSERT_EQ(kGoldenA, instance.get()->a); 41 | ASSERT_EQ(kGoldenB, instance.get()->b); 42 | } 43 | 44 | } // namespace leveldb 45 | -------------------------------------------------------------------------------- /util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/options.h" 6 | 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/env.h" 9 | 10 | namespace leveldb { 11 | 12 | Options::Options() : comparator(BytewiseComparator()), env(Env::Default()) {} 13 | 14 | } // namespace leveldb 15 | -------------------------------------------------------------------------------- /util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ 6 | #define STORAGE_LEVELDB_UTIL_RANDOM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | // A very simple random number generator. Not especially good at 13 | // generating truly random bits, but good enough for our needs in this 14 | // package. 15 | class Random { 16 | private: 17 | uint32_t seed_; 18 | 19 | public: 20 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { 21 | // Avoid bad seeds. 22 | if (seed_ == 0 || seed_ == 2147483647L) { 23 | seed_ = 1; 24 | } 25 | } 26 | uint32_t Next() { 27 | static const uint32_t M = 2147483647L; // 2^31-1 28 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 29 | // We are computing 30 | // seed_ = (seed_ * A) % M, where M = 2^31-1 31 | // 32 | // seed_ must not be zero or M, or else all subsequent computed values 33 | // will be zero or M respectively. For all other values, seed_ will end 34 | // up cycling through every number in [1,M-1] 35 | uint64_t product = seed_ * A; 36 | 37 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 38 | seed_ = static_cast((product >> 31) + (product & M)); 39 | // The first reduction may overflow by 1 bit, so we may need to 40 | // repeat. mod == M is not possible; using > allows the faster 41 | // sign-bit-based test. 42 | if (seed_ > M) { 43 | seed_ -= M; 44 | } 45 | return seed_; 46 | } 47 | // Returns a uniformly distributed value in the range [0..n-1] 48 | // REQUIRES: n > 0 49 | uint32_t Uniform(int n) { return Next() % n; } 50 | 51 | // Randomly returns true ~"1/n" of the time, and false otherwise. 52 | // REQUIRES: n > 0 53 | bool OneIn(int n) { return (Next() % n) == 0; } 54 | 55 | // Skewed: pick "base" uniformly from range [0,max_log] and then 56 | // return "base" random bits. The effect is to pick a number in the 57 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 58 | uint32_t Skewed(int max_log) { return Uniform(1 << Uniform(max_log + 1)); } 59 | }; 60 | 61 | } // namespace leveldb 62 | 63 | #endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ 64 | -------------------------------------------------------------------------------- /util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/status.h" 6 | 7 | #include 8 | 9 | #include "port/port.h" 10 | 11 | namespace leveldb { 12 | 13 | const char* Status::CopyState(const char* state) { 14 | uint32_t size; 15 | std::memcpy(&size, state, sizeof(size)); 16 | char* result = new char[size + 5]; 17 | std::memcpy(result, state, size + 5); 18 | return result; 19 | } 20 | 21 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 22 | assert(code != kOk); 23 | const uint32_t len1 = static_cast(msg.size()); 24 | const uint32_t len2 = static_cast(msg2.size()); 25 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 26 | char* result = new char[size + 5]; 27 | std::memcpy(result, &size, sizeof(size)); 28 | result[4] = static_cast(code); 29 | std::memcpy(result + 5, msg.data(), len1); 30 | if (len2) { 31 | result[5 + len1] = ':'; 32 | result[6 + len1] = ' '; 33 | std::memcpy(result + 7 + len1, msg2.data(), len2); 34 | } 35 | state_ = result; 36 | } 37 | 38 | std::string Status::ToString() const { 39 | if (state_ == nullptr) { 40 | return "OK"; 41 | } else { 42 | char tmp[30]; 43 | const char* type; 44 | switch (code()) { 45 | case kOk: 46 | type = "OK"; 47 | break; 48 | case kNotFound: 49 | type = "NotFound: "; 50 | break; 51 | case kCorruption: 52 | type = "Corruption: "; 53 | break; 54 | case kNotSupported: 55 | type = "Not implemented: "; 56 | break; 57 | case kInvalidArgument: 58 | type = "Invalid argument: "; 59 | break; 60 | case kIOError: 61 | type = "IO error: "; 62 | break; 63 | default: 64 | std::snprintf(tmp, sizeof(tmp), 65 | "Unknown code(%d): ", static_cast(code())); 66 | type = tmp; 67 | break; 68 | } 69 | std::string result(type); 70 | uint32_t length; 71 | std::memcpy(&length, state_, sizeof(length)); 72 | result.append(state_ + 5, length); 73 | return result; 74 | } 75 | } 76 | 77 | } // namespace leveldb 78 | -------------------------------------------------------------------------------- /util/status_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/status.h" 6 | 7 | #include 8 | 9 | #include "gtest/gtest.h" 10 | #include "leveldb/slice.h" 11 | 12 | namespace leveldb { 13 | 14 | TEST(Status, MoveConstructor) { 15 | { 16 | Status ok = Status::OK(); 17 | Status ok2 = std::move(ok); 18 | 19 | ASSERT_TRUE(ok2.ok()); 20 | } 21 | 22 | { 23 | Status status = Status::NotFound("custom NotFound status message"); 24 | Status status2 = std::move(status); 25 | 26 | ASSERT_TRUE(status2.IsNotFound()); 27 | ASSERT_EQ("NotFound: custom NotFound status message", status2.ToString()); 28 | } 29 | 30 | { 31 | Status self_moved = Status::IOError("custom IOError status message"); 32 | 33 | // Needed to bypass compiler warning about explicit move-assignment. 34 | Status& self_moved_reference = self_moved; 35 | self_moved_reference = std::move(self_moved); 36 | } 37 | } 38 | 39 | } // namespace leveldb 40 | -------------------------------------------------------------------------------- /util/testutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testutil.h" 6 | 7 | #include 8 | 9 | #include "util/random.h" 10 | 11 | namespace leveldb { 12 | namespace test { 13 | 14 | Slice RandomString(Random* rnd, int len, std::string* dst) { 15 | dst->resize(len); 16 | for (int i = 0; i < len; i++) { 17 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 18 | } 19 | return Slice(*dst); 20 | } 21 | 22 | std::string RandomKey(Random* rnd, int len) { 23 | // Make sure to generate a wide variety of characters so we 24 | // test the boundary conditions for short-key optimizations. 25 | static const char kTestChars[] = {'\0', '\1', 'a', 'b', 'c', 26 | 'd', 'e', '\xfd', '\xfe', '\xff'}; 27 | std::string result; 28 | for (int i = 0; i < len; i++) { 29 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 30 | } 31 | return result; 32 | } 33 | 34 | Slice CompressibleString(Random* rnd, double compressed_fraction, size_t len, 35 | std::string* dst) { 36 | int raw = static_cast(len * compressed_fraction); 37 | if (raw < 1) raw = 1; 38 | std::string raw_data; 39 | RandomString(rnd, raw, &raw_data); 40 | 41 | // Duplicate the random data until we have filled "len" bytes 42 | dst->clear(); 43 | while (dst->size() < len) { 44 | dst->append(raw_data); 45 | } 46 | dst->resize(len); 47 | return Slice(*dst); 48 | } 49 | 50 | } // namespace test 51 | } // namespace leveldb 52 | -------------------------------------------------------------------------------- /util/testutil.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 6 | #define STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 7 | 8 | #include "gmock/gmock.h" 9 | #include "gtest/gtest.h" 10 | #include "helpers/memenv/memenv.h" 11 | #include "leveldb/env.h" 12 | #include "leveldb/slice.h" 13 | #include "util/random.h" 14 | 15 | namespace leveldb { 16 | namespace test { 17 | 18 | MATCHER(IsOK, "") { return arg.ok(); } 19 | 20 | // Macros for testing the results of functions that return leveldb::Status or 21 | // absl::StatusOr (for any type T). 22 | #define EXPECT_LEVELDB_OK(expression) \ 23 | EXPECT_THAT(expression, leveldb::test::IsOK()) 24 | #define ASSERT_LEVELDB_OK(expression) \ 25 | ASSERT_THAT(expression, leveldb::test::IsOK()) 26 | 27 | // Returns the random seed used at the start of the current test run. 28 | inline int RandomSeed() { 29 | return testing::UnitTest::GetInstance()->random_seed(); 30 | } 31 | 32 | // Store in *dst a random string of length "len" and return a Slice that 33 | // references the generated data. 34 | Slice RandomString(Random* rnd, int len, std::string* dst); 35 | 36 | // Return a random key with the specified length that may contain interesting 37 | // characters (e.g. \x00, \xff, etc.). 38 | std::string RandomKey(Random* rnd, int len); 39 | 40 | // Store in *dst a string of length "len" that will compress to 41 | // "N*compressed_fraction" bytes and return a Slice that references 42 | // the generated data. 43 | Slice CompressibleString(Random* rnd, double compressed_fraction, size_t len, 44 | std::string* dst); 45 | 46 | // A wrapper that allows injection of errors. 47 | class ErrorEnv : public EnvWrapper { 48 | public: 49 | bool writable_file_error_; 50 | int num_writable_file_errors_; 51 | 52 | ErrorEnv() 53 | : EnvWrapper(NewMemEnv(Env::Default())), 54 | writable_file_error_(false), 55 | num_writable_file_errors_(0) {} 56 | ~ErrorEnv() override { delete target(); } 57 | 58 | Status NewWritableFile(const std::string& fname, 59 | WritableFile** result) override { 60 | if (writable_file_error_) { 61 | ++num_writable_file_errors_; 62 | *result = nullptr; 63 | return Status::IOError(fname, "fake error"); 64 | } 65 | return target()->NewWritableFile(fname, result); 66 | } 67 | 68 | Status NewAppendableFile(const std::string& fname, 69 | WritableFile** result) override { 70 | if (writable_file_error_) { 71 | ++num_writable_file_errors_; 72 | *result = nullptr; 73 | return Status::IOError(fname, "fake error"); 74 | } 75 | return target()->NewAppendableFile(fname, result); 76 | } 77 | }; 78 | 79 | } // namespace test 80 | } // namespace leveldb 81 | 82 | #endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 83 | --------------------------------------------------------------------------------