├── LICENSE ├── README.md └── leveldb-master ├── .gitignore ├── .travis.yml ├── AUTHORS ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NEWS ├── README.md ├── TODO ├── build_detect_platform ├── db ├── autocompact_test.cc ├── builder.cc ├── builder.h ├── c.cc ├── c_test.c ├── corruption_test.cc ├── db_bench.cc ├── db_impl.cc ├── db_impl.h ├── db_iter.cc ├── db_iter.h ├── db_test.cc ├── dbformat.cc ├── dbformat.h ├── dbformat_test.cc ├── dumpfile.cc ├── fault_injection_test.cc ├── filename.cc ├── filename.h ├── filename_test.cc ├── leveldbutil.cc ├── log_format.h ├── log_reader.cc ├── log_reader.h ├── log_test.cc ├── log_writer.cc ├── log_writer.h ├── memtable.cc ├── memtable.h ├── recovery_test.cc ├── repair.cc ├── skiplist.h ├── skiplist_test.cc ├── snapshot.h ├── table_cache.cc ├── table_cache.h ├── version_edit.cc ├── version_edit.h ├── version_edit_test.cc ├── version_set.cc ├── version_set.h ├── version_set_test.cc ├── write_batch.cc ├── write_batch_internal.h └── write_batch_test.cc ├── doc ├── bench │ ├── db_bench_sqlite3.cc │ └── db_bench_tree_db.cc ├── benchmark.html ├── impl.md ├── index.md ├── log_format.md └── table_format.md ├── helpers └── memenv │ ├── memenv.cc │ ├── memenv.h │ └── memenv_test.cc ├── include └── leveldb │ ├── c.h │ ├── cache.h │ ├── comparator.h │ ├── db.h │ ├── dumpfile.h │ ├── env.h │ ├── export.h │ ├── filter_policy.h │ ├── iterator.h │ ├── options.h │ ├── slice.h │ ├── status.h │ ├── table.h │ ├── table_builder.h │ └── write_batch.h ├── issues ├── issue178_test.cc └── issue200_test.cc ├── port ├── README ├── atomic_pointer.h ├── port.h ├── port_example.h ├── port_posix.cc ├── port_posix.h ├── thread_annotations.h └── win │ └── stdint.h ├── table ├── block.cc ├── block.h ├── block_builder.cc ├── block_builder.h ├── filter_block.cc ├── filter_block.h ├── filter_block_test.cc ├── format.cc ├── format.h ├── iterator.cc ├── iterator_wrapper.h ├── merger.cc ├── merger.h ├── table.cc ├── table_builder.cc ├── table_test.cc ├── two_level_iterator.cc └── two_level_iterator.h └── util ├── arena.cc ├── arena.h ├── arena_test.cc ├── bloom.cc ├── bloom_test.cc ├── cache.cc ├── cache_test.cc ├── coding.cc ├── coding.h ├── coding_test.cc ├── comparator.cc ├── crc32c.cc ├── crc32c.h ├── crc32c_test.cc ├── env.cc ├── env_posix.cc ├── env_posix_test.cc ├── env_posix_test_helper.h ├── env_test.cc ├── filter_policy.cc ├── hash.cc ├── hash.h ├── hash_test.cc ├── histogram.cc ├── histogram.h ├── logging.cc ├── logging.h ├── mutexlock.h ├── options.cc ├── posix_logger.h ├── random.h ├── status.cc ├── testharness.cc ├── testharness.h ├── testutil.cc └── testutil.h /README.md: -------------------------------------------------------------------------------- 1 | # leveldb_comment 2 | 详细且纯粹的leveldb源码注解，注解总结链接：https://TitenWang.github.io 3 | 4 | # 注解进度 5 | 6 | 2017-12-24 注解用于封装leveldb所有操作结果的Status模块(util/status.cc、include/leveldb/status.h)，内存管理模块(util/arena.cc、util/arena.h)，整型序列化模块(util/coding.cc、util/coding.h)。 7 | 8 | 2017-12-30 注解操作日志模块(db/log_format.h、db/log_writer.cc、db/log_writer.h、db/log_reader.cc、db/log_reader.h)、LRUCache模块(util/cache.cc、include/leveldb/cache.h) 9 | 10 | 2017-12-31 注解布隆过滤器模块(util/bloom.cc、util/bloom.h)，memtable模块(db/memtable.cc、db/memtable.h)，跳表实现(db/skiplist.h)，leveldb内部多种key封装(db/dbformat.cc、db/dbformat.h) 11 | 12 | 2018-01-03 注解Block模块(table/block.h、table/block.cc、table/block_builder.h、table/block_builder.cc)，BlockHandle/Footer模块(table/format.h、table/format.cc) 13 | 14 | 2018-01-06 注解二层迭代器(table/two_level_iterator.h、table/two_level_iterator.cc)，Table模块(table/table.cc、include/leveldb/table.h、table/table_builder.cc、include/leveldb/talbe_builder.h)，FilterBlock模块(table/filter_block.h、table/filter_block.cc) 15 | 16 | 2018-01-14 注解版本变更信息类VersionEdit(db/version_edit.cc、db/version_edit.h)，用于构造Version的类VersionSet::Builder(db/version_set.cc、db/version_set.h) 17 | 18 | 2018-01-28 注解版本信息类Version(db/version_set.cc、db/version_set.h) 19 | 20 | # 注解风格 21 | 注解风格和leveldb本身的注释风格保持一致。以table/block.h的注释为例，如下： 22 | ```c 23 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 24 | // Use of this source code is governed by a BSD-style license that can be 25 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 26 | 27 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 28 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 29 | 30 | #include 31 | #include 32 | #include "leveldb/iterator.h" 33 | 34 | namespace leveldb { 35 | 36 | struct BlockContents; 37 | class Comparator; 38 | 39 | // 一个Block的内存布局： 40 | // ------------------ 41 | // | Record 1 | 42 | // ------------------ 43 | // | Record 2 | 44 | // ------------------ 45 | // | .... | 46 | // ------------------ 47 | // | Record n | 48 | // ------------------ 49 | // | Restart 1 | 50 | // ------------------ 51 | // | Restart 2 | 52 | // ------------------ 53 | // | .... | 54 | // ------------------ 55 | // | Restart m | 56 | // ------------------ 57 | // | num_restarts | 58 | // ------------------ 59 | // 从上面的内存布局可以看到，一个Block内部可以分成两个部分，前面的Record数组 60 | // 存放的是一些K-V记录，其顺序是根据Key值由小到大排列的。后面则是一个称为 61 | // "重启点"的数组，存储是距离block开始处的偏移值，存放了Record数组中一些记录位置。 62 | // "重启点"是干什么的呢？我们知道Block内部的k-v记录是从小到大排列的，这样的话， 63 | // 相邻的两条或者多条记录的key值很有可能存在重叠部分，比如上图中的Record 1的 64 | // key值为"my friend"，Record 2的key值为"my girlfriend"，那么两者就存在重叠部分 65 | // "my "，为了减少key的存储量，Record 2中的key可能只存储和Record 1的key的不同部分， 66 | // 即"girlfriend"，两者的相同部分则可以从Record 1中获取。基于上述的设计，"重启点"的 67 | // 意思是：从这条记录开始，不再采取只存储不同的key部分，而是记录这条记录完整的 68 | // key值，不管该记录的key值是否前一条记录有相同部分，例如Record 3就是一个重启点， 69 | // 且他的key值为"your friend" 或者"my monther"，都将记录完整的key值，而不只是 70 | // 和前一个记录的不同部分。而num_restarts则是"重启点"的个数。 71 | 72 | // 前面我们说过，一个Record记录包含了k-v值，并且其中存储的key可能不是完整的key值， 73 | // 而只是key的一部分值，那么为了适应这种设计，Record记录本身的内存布局是怎么样的呢？ 74 | // ---------------------------------------------------------- 75 | // |key共享长度|key非共享长度|value长度|key非共享内容|value内容| 76 | // ---------------------------------------------------------- 77 | // 以上面的Record 1和Record 2为例，对于Record 2来说，key共享长度为"my "的长度，为3： 78 | // key非共享长度为"girlfriend"的长度，为10，value长度的就是k-v的v的长度，key非共享内容 79 | // 就是"girlfriend"，value内容就是k-v的v的内容。 80 | class Block { 81 | public: 82 | // Initialize the block with the specified contents. 83 | explicit Block(const BlockContents& contents); 84 | 85 | ~Block(); 86 | 87 | size_t size() const { return size_; } 88 | Iterator* NewIterator(const Comparator* comparator); 89 | 90 | private: 91 | 92 | // NumRestarts()方法从data_数组中根据block的内存布局获取到重启点的个数。 93 | uint32_t NumRestarts() const; 94 | 95 |  // data_就是block存放具体内容的内存首地址。 96 | const char* data_; 97 | 98 |  // size_就是block的大小，也就是data_所指向的内存的大小。 99 | size_t size_; 100 | 101 | // restart_offset_就是“重启点”数组首地址在整个Block内的偏移。 102 | uint32_t restart_offset_; // Offset in data_ of restart array 103 | 104 | // 用来表示data_数组指向的内存是不是动态申请的，如果说是，则为true。 105 | bool owned_; // Block owns data_[] 106 | 107 | // No copying allowed 108 | Block(const Block&); 109 | void operator=(const Block&); 110 | 111 | // Iter是Block的内部迭代器。 112 | class Iter; 113 | }; 114 | 115 | } // namespace leveldb 116 | 117 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 118 | ``` 119 | -------------------------------------------------------------------------------- /leveldb-master/.gitignore: -------------------------------------------------------------------------------- 1 | build_config.mk 2 | *.a 3 | *.o 4 | *.dylib* 5 | *.so 6 | *.so.* 7 | *_test 8 | db_bench 9 | leveldbutil 10 | -------------------------------------------------------------------------------- /leveldb-master/.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | compiler: 3 | - clang 4 | - gcc 5 | os: 6 | - linux 7 | - osx 8 | sudo: false 9 | before_install: 10 | - echo $LANG 11 | - echo $LC_ALL 12 | script: 13 | - make -j 4 check 14 | -------------------------------------------------------------------------------- /leveldb-master/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | 10 | # Partial list of contributors: 11 | Kevin Regan 12 | Johan Bilien 13 | -------------------------------------------------------------------------------- /leveldb-master/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We'd love to accept your code patches! However, before we can take them, we 4 | have to jump a couple of legal hurdles. 5 | 6 | ## Contributor License Agreements 7 | 8 | Please fill out either the individual or corporate Contributor License 9 | Agreement as appropriate. 10 | 11 | * If you are an individual writing original source code and you're sure you 12 | own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual). 13 | * If you work for a company that wants to allow you to contribute your work, 14 | then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate). 15 | 16 | Follow either of the two links above to access the appropriate CLA and 17 | instructions for how to sign and return it. 18 | 19 | ## Submitting a Patch 20 | 21 | 1. Sign the contributors license agreement above. 22 | 2. Decide which code you want to submit. A submission should be a set of changes 23 | that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues). 24 | Please don't mix more than one logical change per submission, because it makes 25 | the history hard to follow. If you want to make a change 26 | (e.g. add a sample or feature) that doesn't have a corresponding issue in the 27 | issue tracker, please create one. 28 | 3. **Submitting**: When you are ready to submit, send us a Pull Request. Be 29 | sure to include the issue number you fixed and the name you used to sign 30 | the CLA. 31 | 32 | ## Writing Code ## 33 | 34 | If your contribution contains code, please make sure that it follows 35 | [the style guide](http://google.github.io/styleguide/cppguide.html). 36 | Otherwise we will have to ask you to make changes, and that's no fun for anyone. 37 | -------------------------------------------------------------------------------- /leveldb-master/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /leveldb-master/NEWS: -------------------------------------------------------------------------------- 1 | Release 1.2 2011-05-16 2 | ---------------------- 3 | 4 | Fixes for larger databases (tested up to one billion 100-byte entries, 5 | i.e., ~100GB). 6 | 7 | (1) Place hard limit on number of level-0 files. This fixes errors 8 | of the form "too many open files". 9 | 10 | (2) Fixed memtable management. Before the fix, a heavy write burst 11 | could cause unbounded memory usage. 12 | 13 | A fix for a logging bug where the reader would incorrectly complain 14 | about corruption. 15 | 16 | Allow public access to WriteBatch contents so that users can easily 17 | wrap a DB. 18 | -------------------------------------------------------------------------------- /leveldb-master/TODO: -------------------------------------------------------------------------------- 1 | ss 2 | - Stats 3 | 4 | db 5 | - Maybe implement DB::BulkDeleteForRange(start_key, end_key) 6 | that would blow away files whose ranges are entirely contained 7 | within [start_key..end_key]? For Chrome, deletion of obsolete 8 | object stores, etc. can be done in the background anyway, so 9 | probably not that important. 10 | - There have been requests for MultiGet. 11 | 12 | After a range is completely deleted, what gets rid of the 13 | corresponding files if we do no future changes to that range. Make 14 | the conditions for triggering compactions fire in more situations? 15 | -------------------------------------------------------------------------------- /leveldb-master/db/autocompact_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/db.h" 6 | #include "db/db_impl.h" 7 | #include "leveldb/cache.h" 8 | #include "util/testharness.h" 9 | #include "util/testutil.h" 10 | 11 | namespace leveldb { 12 | 13 | class AutoCompactTest { 14 | public: 15 | std::string dbname_; 16 | Cache* tiny_cache_; 17 | Options options_; 18 | DB* db_; 19 | 20 | AutoCompactTest() { 21 | dbname_ = test::TmpDir() + "/autocompact_test"; 22 | tiny_cache_ = NewLRUCache(100); 23 | options_.block_cache = tiny_cache_; 24 | DestroyDB(dbname_, options_); 25 | options_.create_if_missing = true; 26 | options_.compression = kNoCompression; 27 | ASSERT_OK(DB::Open(options_, dbname_, &db_)); 28 | } 29 | 30 | ~AutoCompactTest() { 31 | delete db_; 32 | DestroyDB(dbname_, Options()); 33 | delete tiny_cache_; 34 | } 35 | 36 | std::string Key(int i) { 37 | char buf[100]; 38 | snprintf(buf, sizeof(buf), "key%06d", i); 39 | return std::string(buf); 40 | } 41 | 42 | uint64_t Size(const Slice& start, const Slice& limit) { 43 | Range r(start, limit); 44 | uint64_t size; 45 | db_->GetApproximateSizes(&r, 1, &size); 46 | return size; 47 | } 48 | 49 | void DoReads(int n); 50 | }; 51 | 52 | static const int kValueSize = 200 * 1024; 53 | static const int kTotalSize = 100 * 1024 * 1024; 54 | static const int kCount = kTotalSize / kValueSize; 55 | 56 | // Read through the first n keys repeatedly and check that they get 57 | // compacted (verified by checking the size of the key space). 58 | void AutoCompactTest::DoReads(int n) { 59 | std::string value(kValueSize, 'x'); 60 | DBImpl* dbi = reinterpret_cast(db_); 61 | 62 | // Fill database 63 | for (int i = 0; i < kCount; i++) { 64 | ASSERT_OK(db_->Put(WriteOptions(), Key(i), value)); 65 | } 66 | ASSERT_OK(dbi->TEST_CompactMemTable()); 67 | 68 | // Delete everything 69 | for (int i = 0; i < kCount; i++) { 70 | ASSERT_OK(db_->Delete(WriteOptions(), Key(i))); 71 | } 72 | ASSERT_OK(dbi->TEST_CompactMemTable()); 73 | 74 | // Get initial measurement of the space we will be reading. 75 | const int64_t initial_size = Size(Key(0), Key(n)); 76 | const int64_t initial_other_size = Size(Key(n), Key(kCount)); 77 | 78 | // Read until size drops significantly. 79 | std::string limit_key = Key(n); 80 | for (int read = 0; true; read++) { 81 | ASSERT_LT(read, 100) << "Taking too long to compact"; 82 | Iterator* iter = db_->NewIterator(ReadOptions()); 83 | for (iter->SeekToFirst(); 84 | iter->Valid() && iter->key().ToString() < limit_key; 85 | iter->Next()) { 86 | // Drop data 87 | } 88 | delete iter; 89 | // Wait a little bit to allow any triggered compactions to complete. 90 | Env::Default()->SleepForMicroseconds(1000000); 91 | uint64_t size = Size(Key(0), Key(n)); 92 | fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", 93 | read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0); 94 | if (size <= initial_size/10) { 95 | break; 96 | } 97 | } 98 | 99 | // Verify that the size of the key space not touched by the reads 100 | // is pretty much unchanged. 101 | const int64_t final_other_size = Size(Key(n), Key(kCount)); 102 | ASSERT_LE(final_other_size, initial_other_size + 1048576); 103 | ASSERT_GE(final_other_size, initial_other_size/5 - 1048576); 104 | } 105 | 106 | TEST(AutoCompactTest, ReadAll) { 107 | DoReads(kCount); 108 | } 109 | 110 | TEST(AutoCompactTest, ReadHalf) { 111 | DoReads(kCount/2); 112 | } 113 | 114 | } // namespace leveldb 115 | 116 | int main(int argc, char** argv) { 117 | return leveldb::test::RunAllTests(); 118 | } 119 | -------------------------------------------------------------------------------- /leveldb-master/db/builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/builder.h" 6 | 7 | #include "db/filename.h" 8 | #include "db/dbformat.h" 9 | #include "db/table_cache.h" 10 | #include "db/version_edit.h" 11 | #include "leveldb/db.h" 12 | #include "leveldb/env.h" 13 | #include "leveldb/iterator.h" 14 | 15 | namespace leveldb { 16 | 17 | Status BuildTable(const std::string& dbname, 18 | Env* env, 19 | const Options& options, 20 | TableCache* table_cache, 21 | Iterator* iter, 22 | FileMetaData* meta) { 23 | Status s; 24 | meta->file_size = 0; 25 | iter->SeekToFirst(); 26 | 27 | std::string fname = TableFileName(dbname, meta->number); 28 | if (iter->Valid()) { 29 | WritableFile* file; 30 | s = env->NewWritableFile(fname, &file); 31 | if (!s.ok()) { 32 | return s; 33 | } 34 | 35 | TableBuilder* builder = new TableBuilder(options, file); 36 | meta->smallest.DecodeFrom(iter->key()); 37 | for (; iter->Valid(); iter->Next()) { 38 | Slice key = iter->key(); 39 | meta->largest.DecodeFrom(key); 40 | builder->Add(key, iter->value()); 41 | } 42 | 43 | // Finish and check for builder errors 44 | s = builder->Finish(); 45 | if (s.ok()) { 46 | meta->file_size = builder->FileSize(); 47 | assert(meta->file_size > 0); 48 | } 49 | delete builder; 50 | 51 | // Finish and check for file errors 52 | if (s.ok()) { 53 | s = file->Sync(); 54 | } 55 | if (s.ok()) { 56 | s = file->Close(); 57 | } 58 | delete file; 59 | file = NULL; 60 | 61 | if (s.ok()) { 62 | // Verify that the table is usable 63 | Iterator* it = table_cache->NewIterator(ReadOptions(), 64 | meta->number, 65 | meta->file_size); 66 | s = it->status(); 67 | delete it; 68 | } 69 | } 70 | 71 | // Check for input iterator errors 72 | if (!iter->status().ok()) { 73 | s = iter->status(); 74 | } 75 | 76 | if (s.ok() && meta->file_size > 0) { 77 | // Keep it 78 | } else { 79 | env->DeleteFile(fname); 80 | } 81 | return s; 82 | } 83 | 84 | } // namespace leveldb 85 | -------------------------------------------------------------------------------- /leveldb-master/db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 6 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 7 | 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | 12 | struct Options; 13 | struct FileMetaData; 14 | 15 | class Env; 16 | class Iterator; 17 | class TableCache; 18 | class VersionEdit; 19 | 20 | // Build a Table file from the contents of *iter. The generated file 21 | // will be named according to meta->number. On success, the rest of 22 | // *meta will be filled with metadata about the generated table. 23 | // If no data is present in *iter, meta->file_size will be set to 24 | // zero, and no Table file will be produced. 25 | extern Status BuildTable(const std::string& dbname, 26 | Env* env, 27 | const Options& options, 28 | TableCache* table_cache, 29 | Iterator* iter, 30 | FileMetaData* meta); 31 | 32 | } // namespace leveldb 33 | 34 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 35 | -------------------------------------------------------------------------------- /leveldb-master/db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 6 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 7 | 8 | #include 9 | #include "leveldb/db.h" 10 | #include "db/dbformat.h" 11 | 12 | namespace leveldb { 13 | 14 | class DBImpl; 15 | 16 | // Return a new iterator that converts internal keys (yielded by 17 | // "*internal_iter") that were live at the specified "sequence" number 18 | // into appropriate user keys. 19 | extern Iterator* NewDBIterator( 20 | DBImpl* db, 21 | const Comparator* user_key_comparator, 22 | Iterator* internal_iter, 23 | SequenceNumber sequence, 24 | uint32_t seed); 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 29 | -------------------------------------------------------------------------------- /leveldb-master/db/dbformat_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/dbformat.h" 6 | #include "util/logging.h" 7 | #include "util/testharness.h" 8 | 9 | namespace leveldb { 10 | 11 | static std::string IKey(const std::string& user_key, 12 | uint64_t seq, 13 | ValueType vt) { 14 | std::string encoded; 15 | AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); 16 | return encoded; 17 | } 18 | 19 | static std::string Shorten(const std::string& s, const std::string& l) { 20 | std::string result = s; 21 | InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); 22 | return result; 23 | } 24 | 25 | static std::string ShortSuccessor(const std::string& s) { 26 | std::string result = s; 27 | InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); 28 | return result; 29 | } 30 | 31 | static void TestKey(const std::string& key, 32 | uint64_t seq, 33 | ValueType vt) { 34 | std::string encoded = IKey(key, seq, vt); 35 | 36 | Slice in(encoded); 37 | ParsedInternalKey decoded("", 0, kTypeValue); 38 | 39 | ASSERT_TRUE(ParseInternalKey(in, &decoded)); 40 | ASSERT_EQ(key, decoded.user_key.ToString()); 41 | ASSERT_EQ(seq, decoded.sequence); 42 | ASSERT_EQ(vt, decoded.type); 43 | 44 | ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); 45 | } 46 | 47 | class FormatTest { }; 48 | 49 | TEST(FormatTest, InternalKey_EncodeDecode) { 50 | const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; 51 | const uint64_t seq[] = { 52 | 1, 2, 3, 53 | (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, 54 | (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, 55 | (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 56 | }; 57 | for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { 58 | for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { 59 | TestKey(keys[k], seq[s], kTypeValue); 60 | TestKey("hello", 1, kTypeDeletion); 61 | } 62 | } 63 | } 64 | 65 | TEST(FormatTest, InternalKeyShortSeparator) { 66 | // When user keys are same 67 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 68 | Shorten(IKey("foo", 100, kTypeValue), 69 | IKey("foo", 99, kTypeValue))); 70 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 71 | Shorten(IKey("foo", 100, kTypeValue), 72 | IKey("foo", 101, kTypeValue))); 73 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 74 | Shorten(IKey("foo", 100, kTypeValue), 75 | IKey("foo", 100, kTypeValue))); 76 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 77 | Shorten(IKey("foo", 100, kTypeValue), 78 | IKey("foo", 100, kTypeDeletion))); 79 | 80 | // When user keys are misordered 81 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 82 | Shorten(IKey("foo", 100, kTypeValue), 83 | IKey("bar", 99, kTypeValue))); 84 | 85 | // When user keys are different, but correctly ordered 86 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 87 | Shorten(IKey("foo", 100, kTypeValue), 88 | IKey("hello", 200, kTypeValue))); 89 | 90 | // When start user key is prefix of limit user key 91 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 92 | Shorten(IKey("foo", 100, kTypeValue), 93 | IKey("foobar", 200, kTypeValue))); 94 | 95 | // When limit user key is prefix of start user key 96 | ASSERT_EQ(IKey("foobar", 100, kTypeValue), 97 | Shorten(IKey("foobar", 100, kTypeValue), 98 | IKey("foo", 200, kTypeValue))); 99 | } 100 | 101 | TEST(FormatTest, InternalKeyShortestSuccessor) { 102 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 103 | ShortSuccessor(IKey("foo", 100, kTypeValue))); 104 | ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), 105 | ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); 106 | } 107 | 108 | } // namespace leveldb 109 | 110 | int main(int argc, char** argv) { 111 | return leveldb::test::RunAllTests(); 112 | } 113 | -------------------------------------------------------------------------------- /leveldb-master/db/filename.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include "db/filename.h" 8 | #include "db/dbformat.h" 9 | #include "leveldb/env.h" 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | 14 | // A utility routine: write "data" to the named file and Sync() it. 15 | extern Status WriteStringToFileSync(Env* env, const Slice& data, 16 | const std::string& fname); 17 | 18 | static std::string MakeFileName(const std::string& name, uint64_t number, 19 | const char* suffix) { 20 | char buf[100]; 21 | snprintf(buf, sizeof(buf), "/%06llu.%s", 22 | static_cast(number), 23 | suffix); 24 | return name + buf; 25 | } 26 | 27 | std::string LogFileName(const std::string& name, uint64_t number) { 28 | assert(number > 0); 29 | return MakeFileName(name, number, "log"); 30 | } 31 | 32 | std::string TableFileName(const std::string& name, uint64_t number) { 33 | assert(number > 0); 34 | return MakeFileName(name, number, "ldb"); 35 | } 36 | 37 | std::string SSTTableFileName(const std::string& name, uint64_t number) { 38 | assert(number > 0); 39 | return MakeFileName(name, number, "sst"); 40 | } 41 | 42 | std::string DescriptorFileName(const std::string& dbname, uint64_t number) { 43 | assert(number > 0); 44 | char buf[100]; 45 | snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", 46 | static_cast(number)); 47 | return dbname + buf; 48 | } 49 | 50 | std::string CurrentFileName(const std::string& dbname) { 51 | return dbname + "/CURRENT"; 52 | } 53 | 54 | std::string LockFileName(const std::string& dbname) { 55 | return dbname + "/LOCK"; 56 | } 57 | 58 | std::string TempFileName(const std::string& dbname, uint64_t number) { 59 | assert(number > 0); 60 | return MakeFileName(dbname, number, "dbtmp"); 61 | } 62 | 63 | std::string InfoLogFileName(const std::string& dbname) { 64 | return dbname + "/LOG"; 65 | } 66 | 67 | // Return the name of the old info log file for "dbname". 68 | std::string OldInfoLogFileName(const std::string& dbname) { 69 | return dbname + "/LOG.old"; 70 | } 71 | 72 | 73 | // Owned filenames have the form: 74 | // dbname/CURRENT 75 | // dbname/LOCK 76 | // dbname/LOG 77 | // dbname/LOG.old 78 | // dbname/MANIFEST-[0-9]+ 79 | // dbname/[0-9]+.(log|sst|ldb) 80 | bool ParseFileName(const std::string& fname, 81 | uint64_t* number, 82 | FileType* type) { 83 | Slice rest(fname); 84 | if (rest == "CURRENT") { 85 | *number = 0; 86 | *type = kCurrentFile; 87 | } else if (rest == "LOCK") { 88 | *number = 0; 89 | *type = kDBLockFile; 90 | } else if (rest == "LOG" || rest == "LOG.old") { 91 | *number = 0; 92 | *type = kInfoLogFile; 93 | } else if (rest.starts_with("MANIFEST-")) { 94 | rest.remove_prefix(strlen("MANIFEST-")); 95 | uint64_t num; 96 | if (!ConsumeDecimalNumber(&rest, &num)) { 97 | return false; 98 | } 99 | if (!rest.empty()) { 100 | return false; 101 | } 102 | *type = kDescriptorFile; 103 | *number = num; 104 | } else { 105 | // Avoid strtoull() to keep filename format independent of the 106 | // current locale 107 | uint64_t num; 108 | if (!ConsumeDecimalNumber(&rest, &num)) { 109 | return false; 110 | } 111 | Slice suffix = rest; 112 | if (suffix == Slice(".log")) { 113 | *type = kLogFile; 114 | } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) { 115 | *type = kTableFile; 116 | } else if (suffix == Slice(".dbtmp")) { 117 | *type = kTempFile; 118 | } else { 119 | return false; 120 | } 121 | *number = num; 122 | } 123 | return true; 124 | } 125 | 126 | Status SetCurrentFile(Env* env, const std::string& dbname, 127 | uint64_t descriptor_number) { 128 | // Remove leading "dbname/" and add newline to manifest file name 129 | std::string manifest = DescriptorFileName(dbname, descriptor_number); 130 | Slice contents = manifest; 131 | assert(contents.starts_with(dbname + "/")); 132 | contents.remove_prefix(dbname.size() + 1); 133 | std::string tmp = TempFileName(dbname, descriptor_number); 134 | Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp); 135 | if (s.ok()) { 136 | s = env->RenameFile(tmp, CurrentFileName(dbname)); 137 | } 138 | if (!s.ok()) { 139 | env->DeleteFile(tmp); 140 | } 141 | return s; 142 | } 143 | 144 | } // namespace leveldb 145 | -------------------------------------------------------------------------------- /leveldb-master/db/filename.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // File names used by DB code 6 | 7 | #ifndef STORAGE_LEVELDB_DB_FILENAME_H_ 8 | #define STORAGE_LEVELDB_DB_FILENAME_H_ 9 | 10 | #include 11 | #include 12 | #include "leveldb/slice.h" 13 | #include "leveldb/status.h" 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Env; 19 | 20 | enum FileType { 21 | kLogFile, 22 | kDBLockFile, 23 | kTableFile, 24 | kDescriptorFile, 25 | kCurrentFile, 26 | kTempFile, 27 | kInfoLogFile // Either the current one, or an old one 28 | }; 29 | 30 | // Return the name of the log file with the specified number 31 | // in the db named by "dbname". The result will be prefixed with 32 | // "dbname". 33 | extern std::string LogFileName(const std::string& dbname, uint64_t number); 34 | 35 | // Return the name of the sstable with the specified number 36 | // in the db named by "dbname". The result will be prefixed with 37 | // "dbname". 38 | extern std::string TableFileName(const std::string& dbname, uint64_t number); 39 | 40 | // Return the legacy file name for an sstable with the specified number 41 | // in the db named by "dbname". The result will be prefixed with 42 | // "dbname". 43 | extern std::string SSTTableFileName(const std::string& dbname, uint64_t number); 44 | 45 | // Return the name of the descriptor file for the db named by 46 | // "dbname" and the specified incarnation number. The result will be 47 | // prefixed with "dbname". 48 | extern std::string DescriptorFileName(const std::string& dbname, 49 | uint64_t number); 50 | 51 | // Return the name of the current file. This file contains the name 52 | // of the current manifest file. The result will be prefixed with 53 | // "dbname". 54 | extern std::string CurrentFileName(const std::string& dbname); 55 | 56 | // Return the name of the lock file for the db named by 57 | // "dbname". The result will be prefixed with "dbname". 58 | extern std::string LockFileName(const std::string& dbname); 59 | 60 | // Return the name of a temporary file owned by the db named "dbname". 61 | // The result will be prefixed with "dbname". 62 | extern std::string TempFileName(const std::string& dbname, uint64_t number); 63 | 64 | // Return the name of the info log file for "dbname". 65 | extern std::string InfoLogFileName(const std::string& dbname); 66 | 67 | // Return the name of the old info log file for "dbname". 68 | extern std::string OldInfoLogFileName(const std::string& dbname); 69 | 70 | // If filename is a leveldb file, store the type of the file in *type. 71 | // The number encoded in the filename is stored in *number. If the 72 | // filename was successfully parsed, returns true. Else return false. 73 | extern bool ParseFileName(const std::string& filename, 74 | uint64_t* number, 75 | FileType* type); 76 | 77 | // Make the CURRENT file point to the descriptor file with the 78 | // specified number. 79 | extern Status SetCurrentFile(Env* env, const std::string& dbname, 80 | uint64_t descriptor_number); 81 | 82 | 83 | } // namespace leveldb 84 | 85 | #endif // STORAGE_LEVELDB_DB_FILENAME_H_ 86 | -------------------------------------------------------------------------------- /leveldb-master/db/filename_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/filename.h" 6 | 7 | #include "db/dbformat.h" 8 | #include "port/port.h" 9 | #include "util/logging.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class FileNameTest { }; 15 | 16 | TEST(FileNameTest, Parse) { 17 | Slice db; 18 | FileType type; 19 | uint64_t number; 20 | 21 | // Successful parses 22 | static struct { 23 | const char* fname; 24 | uint64_t number; 25 | FileType type; 26 | } cases[] = { 27 | { "100.log", 100, kLogFile }, 28 | { "0.log", 0, kLogFile }, 29 | { "0.sst", 0, kTableFile }, 30 | { "0.ldb", 0, kTableFile }, 31 | { "CURRENT", 0, kCurrentFile }, 32 | { "LOCK", 0, kDBLockFile }, 33 | { "MANIFEST-2", 2, kDescriptorFile }, 34 | { "MANIFEST-7", 7, kDescriptorFile }, 35 | { "LOG", 0, kInfoLogFile }, 36 | { "LOG.old", 0, kInfoLogFile }, 37 | { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, 38 | }; 39 | for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { 40 | std::string f = cases[i].fname; 41 | ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; 42 | ASSERT_EQ(cases[i].type, type) << f; 43 | ASSERT_EQ(cases[i].number, number) << f; 44 | } 45 | 46 | // Errors 47 | static const char* errors[] = { 48 | "", 49 | "foo", 50 | "foo-dx-100.log", 51 | ".log", 52 | "", 53 | "manifest", 54 | "CURREN", 55 | "CURRENTX", 56 | "MANIFES", 57 | "MANIFEST", 58 | "MANIFEST-", 59 | "XMANIFEST-3", 60 | "MANIFEST-3x", 61 | "LOC", 62 | "LOCKx", 63 | "LO", 64 | "LOGx", 65 | "18446744073709551616.log", 66 | "184467440737095516150.log", 67 | "100", 68 | "100.", 69 | "100.lop" 70 | }; 71 | for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { 72 | std::string f = errors[i]; 73 | ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; 74 | } 75 | } 76 | 77 | TEST(FileNameTest, Construction) { 78 | uint64_t number; 79 | FileType type; 80 | std::string fname; 81 | 82 | fname = CurrentFileName("foo"); 83 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 84 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 85 | ASSERT_EQ(0, number); 86 | ASSERT_EQ(kCurrentFile, type); 87 | 88 | fname = LockFileName("foo"); 89 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 90 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 91 | ASSERT_EQ(0, number); 92 | ASSERT_EQ(kDBLockFile, type); 93 | 94 | fname = LogFileName("foo", 192); 95 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 96 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 97 | ASSERT_EQ(192, number); 98 | ASSERT_EQ(kLogFile, type); 99 | 100 | fname = TableFileName("bar", 200); 101 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 102 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 103 | ASSERT_EQ(200, number); 104 | ASSERT_EQ(kTableFile, type); 105 | 106 | fname = DescriptorFileName("bar", 100); 107 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 108 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 109 | ASSERT_EQ(100, number); 110 | ASSERT_EQ(kDescriptorFile, type); 111 | 112 | fname = TempFileName("tmp", 999); 113 | ASSERT_EQ("tmp/", std::string(fname.data(), 4)); 114 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 115 | ASSERT_EQ(999, number); 116 | ASSERT_EQ(kTempFile, type); 117 | } 118 | 119 | } // namespace leveldb 120 | 121 | int main(int argc, char** argv) { 122 | return leveldb::test::RunAllTests(); 123 | } 124 | -------------------------------------------------------------------------------- /leveldb-master/db/leveldbutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "leveldb/dumpfile.h" 7 | #include "leveldb/env.h" 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | namespace { 12 | 13 | class StdoutPrinter : public WritableFile { 14 | public: 15 | virtual Status Append(const Slice& data) { 16 | fwrite(data.data(), 1, data.size(), stdout); 17 | return Status::OK(); 18 | } 19 | virtual Status Close() { return Status::OK(); } 20 | virtual Status Flush() { return Status::OK(); } 21 | virtual Status Sync() { return Status::OK(); } 22 | }; 23 | 24 | bool HandleDumpCommand(Env* env, char** files, int num) { 25 | StdoutPrinter printer; 26 | bool ok = true; 27 | for (int i = 0; i < num; i++) { 28 | Status s = DumpFile(env, files[i], &printer); 29 | if (!s.ok()) { 30 | fprintf(stderr, "%s\n", s.ToString().c_str()); 31 | ok = false; 32 | } 33 | } 34 | return ok; 35 | } 36 | 37 | } // namespace 38 | } // namespace leveldb 39 | 40 | static void Usage() { 41 | fprintf( 42 | stderr, 43 | "Usage: leveldbutil command...\n" 44 | " dump files... -- dump contents of specified files\n" 45 | ); 46 | } 47 | 48 | int main(int argc, char** argv) { 49 | leveldb::Env* env = leveldb::Env::Default(); 50 | bool ok = true; 51 | if (argc < 2) { 52 | Usage(); 53 | ok = false; 54 | } else { 55 | std::string command = argv[1]; 56 | if (command == "dump") { 57 | ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); 58 | } else { 59 | Usage(); 60 | ok = false; 61 | } 62 | } 63 | return (ok ? 0 : 1); 64 | } 65 | -------------------------------------------------------------------------------- /leveldb-master/db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Log format information shared by reader and writer. 6 | // See ../doc/log_format.md for more detail. 7 | 8 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 9 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 10 | 11 | namespace leveldb { 12 | namespace log { 13 | 14 | // RecordType列举了日志中的记录类型 15 | enum RecordType { 16 | // Zero is reserved for preallocated files 17 | kZeroType = 0, 18 | 19 | kFullType = 1, 20 | 21 | // For fragments 22 | kFirstType = 2, 23 | kMiddleType = 3, 24 | kLastType = 4 25 | }; 26 | static const int kMaxRecordType = kLastType; 27 | 28 | // log日志中是按照kBlockSize为单位进行写的 29 | static const int kBlockSize = 32768; 30 | 31 | // Header is checksum (4 bytes), length (2 bytes), type (1 byte). 32 | static const int kHeaderSize = 4 + 2 + 1; 33 | 34 | } // namespace log 35 | } // namespace leveldb 36 | 37 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 38 | -------------------------------------------------------------------------------- /leveldb-master/db/log_reader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_READER_H_ 7 | 8 | #include 9 | 10 | #include "db/log_format.h" 11 | #include "leveldb/slice.h" 12 | #include "leveldb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | class SequentialFile; 17 | 18 | namespace log { 19 | 20 | class Reader { 21 | public: 22 | // Interface for reporting errors. 23 | class Reporter { 24 | public: 25 | virtual ~Reporter(); 26 | 27 | // Some corruption was detected. "size" is the approximate number 28 | // of bytes dropped due to the corruption. 29 | virtual void Corruption(size_t bytes, const Status& status) = 0; 30 | }; 31 | 32 | // Create a reader that will return log records from "*file". 33 | // "*file" must remain live while this Reader is in use. 34 | // 35 | // If "reporter" is non-NULL, it is notified whenever some data is 36 | // dropped due to a detected corruption. "*reporter" must remain 37 | // live while this Reader is in use. 38 | // 39 | // If "checksum" is true, verify checksums if available. 40 | // 41 | // The Reader will start reading at the first record located at physical 42 | // position >= initial_offset within the file. 43 | Reader(SequentialFile* file, Reporter* reporter, bool checksum, 44 | uint64_t initial_offset); 45 | 46 | ~Reader(); 47 | 48 | // Read the next record into *record. Returns true if read 49 | // successfully, false if we hit end of the input. May use 50 | // "*scratch" as temporary storage. The contents filled in *record 51 | // will only be valid until the next mutating operation on this 52 | // reader or the next mutation to *scratch. 53 | bool ReadRecord(Slice* record, std::string* scratch); 54 | 55 | // Returns the physical offset of the last record returned by ReadRecord. 56 | // 57 | // Undefined before the first call to ReadRecord. 58 | uint64_t LastRecordOffset(); 59 | 60 | private: 61 | 62 | // log文件封装对象 63 | SequentialFile* const file_; 64 | 65 | // reporter_用于错误上报 66 | Reporter* const reporter_; 67 | 68 | // checksum_用于表示是否需要对校验和做校验处理 69 | bool const checksum_; 70 | 71 | // backing_store_缓冲区用于存放从log文件中读取到的block 72 | char* const backing_store_; 73 | 74 | // buffer_用于封装backing_store_缓冲区中剩余未被处理的内容，将未被处理的内容的 75 | // 起始地址和大小用一个Slice来保存。 76 | Slice buffer_; 77 | 78 | // 表示log文件是否已经读取完毕了。 79 | bool eof_; // Last Read() indicated EOF by returning < kBlockSize 80 | 81 | // Offset of the last record returned by ReadRecord. 82 | // last_record_offset_存放的是上一次调用ReadRecord()函数返回的逻辑记录的 83 | // 起始字节在整个log文件中偏移。 84 | uint64_t last_record_offset_; 85 | 86 | 87 | // Offset of the first location past the end of buffer_. 88 | // end_of_buffer_offset_存放的是当前buffer结束位置存放的内容的在log文件中 89 | // 紧接着的下一个字节在整个文件中的偏移。往buffer中存放内容的地方只有一个， 90 | // 那就是从文件中读取一个block的时候，每次往buffer中存放一个block大小的内容 91 | // 时，就会更新一下end_of_buffer_offset_，所以end_of_buffer_offset_更准确的说 92 | // 是下一个将要被读取的block在整个文件中的偏移。 93 | uint64_t end_of_buffer_offset_; 94 | 95 | // Offset at which to start looking for the first record to return 96 | // log文件中初始逻辑记录距离log文件头部的偏移。 97 | uint64_t const initial_offset_; 98 | 99 | // True if we are resynchronizing after a seek (initial_offset_ > 0). In 100 | // particular, a run of kMiddleType and kLastType records can be silently 101 | // skipped in this mode 102 | bool resyncing_; 103 | 104 | // Extend record types with the following special values 105 | enum { 106 | kEof = kMaxRecordType + 1, 107 | // Returned whenever we find an invalid physical record. 108 | // Currently there are three situations in which this happens: 109 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 110 | // * The record is a 0-length record (No drop is reported) 111 | // * The record is below constructor's initial_offset (No drop is reported) 112 | kBadRecord = kMaxRecordType + 2 113 | }; 114 | 115 | // Skips all blocks that are completely before "initial_offset_". 116 | // 117 | // Returns true on success. Handles reporting. 118 | bool SkipToInitialBlock(); 119 | 120 | // Return type, or one of the preceding special values 121 | unsigned int ReadPhysicalRecord(Slice* result); 122 | 123 | // Reports dropped bytes to the reporter. 124 | // buffer_ must be updated to remove the dropped bytes prior to invocation. 125 | void ReportCorruption(uint64_t bytes, const char* reason); 126 | void ReportDrop(uint64_t bytes, const Status& reason); 127 | 128 | // No copying allowed 129 | Reader(const Reader&); 130 | void operator=(const Reader&); 131 | }; 132 | 133 | } // namespace log 134 | } // namespace leveldb 135 | 136 | #endif // STORAGE_LEVELDB_DB_LOG_READER_H_ 137 | -------------------------------------------------------------------------------- /leveldb-master/db/log_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/log_writer.h" 6 | 7 | #include 8 | #include "leveldb/env.h" 9 | #include "util/coding.h" 10 | #include "util/crc32c.h" 11 | 12 | namespace leveldb { 13 | namespace log { 14 | 15 | static void InitTypeCrc(uint32_t* type_crc) { 16 | for (int i = 0; i <= kMaxRecordType; i++) { 17 | char t = static_cast(i); 18 | type_crc[i] = crc32c::Value(&t, 1); 19 | } 20 | } 21 | 22 | Writer::Writer(WritableFile* dest) 23 | : dest_(dest), 24 | block_offset_(0) { 25 | InitTypeCrc(type_crc_); 26 | } 27 | 28 | Writer::Writer(WritableFile* dest, uint64_t dest_length) 29 | : dest_(dest), block_offset_(dest_length % kBlockSize) { 30 | InitTypeCrc(type_crc_); 31 | } 32 | 33 | Writer::~Writer() { 34 | } 35 | 36 | // 该函数用于向日志模块中添加一条操作日志，操作日志实际内容由调用者封装成一个数据片段Slice。 37 | Status Writer::AddRecord(const Slice& slice) { 38 | 39 | // 获取数据片段的值和大小。 40 | const char* ptr = slice.data(); 41 | size_t left = slice.size(); 42 | 43 | // Fragment the record if necessary and emit it. Note that if slice 44 | // is empty, we still want to iterate once to emit a single 45 | // zero-length record 46 | Status s; 47 | bool begin = true; 48 | do { 49 | // 当前block中还可以用来存放日志记录(未被使用)的空间。 50 | const int leftover = kBlockSize - block_offset_; 51 | assert(leftover >= 0); 52 | 53 | // 如果当前block中的可用空间已经不足以容纳下一条记录的头部内容，则直接切换到 54 | // 一个新的block，那么这部分剩余空间中也需要写入零值数据，作为一条零长记录。 55 | if (leftover < kHeaderSize) { 56 | // Switch to a new block 57 | if (leftover > 0) { 58 | // Fill the trailer (literal below relies on kHeaderSize being 7) 59 | assert(kHeaderSize == 7); 60 | // 向dest_中写入内容 61 | dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); 62 | } 63 | 64 | // 因为要切换到一个新的block，所以这里将剩余可用的空间相对于block首地址的偏移清零。 65 | block_offset_ = 0; 66 | } 67 | 68 | // Invariant: we never leave < kHeaderSize bytes in a block. 69 | // 上面的if语句判断程序走到这里一定有剩余空间用来存放实际的记录内容。 70 | assert(kBlockSize - block_offset_ - kHeaderSize >= 0); 71 | 72 | // 计算当前block除去头部长度之后剩余可用的空间大小， 73 | const size_t avail = kBlockSize - block_offset_ - kHeaderSize; 74 | 75 | // 如果当前block中剩余可用空间多余记录剩余未写入的内容长度，那么可以直接将 76 | // 记录内容写入当前block中；如果当前block中剩余空间内容不足以写入记录剩余 77 | // 未写入内容，则尽可能多地写入记录内容，余下的写入到后面的block中；如果 78 | // 剩余未写入内容长度刚好等于当前block可用空间的话，那就直接写入。 79 | // fragment_length存放的是本次写入的记录长度。 80 | const size_t fragment_length = (left < avail) ? left : avail; 81 | 82 | RecordType type; 83 | // left == fragment_length说明这是要写的是记录最后一部分内容了 84 | const bool end = (left == fragment_length); 85 | 86 | if (begin && end) { 87 | // kFulltType说明本条记录在一个block中就完全写完了。 88 | type = kFullType; 89 | } else if (begin) { 90 | // kFirstType说明本条记录的第一部分写入到了当前block中。 91 | type = kFirstType; 92 | } else if (end) { 93 | // kLastType说明本条记录的最后一部分写入到了当前block中。 94 | type = kLastType; 95 | } else { 96 | // kMiddleType说明本条记录的中间部分写入到了当前block中。 97 | type = kMiddleType; 98 | } 99 | 100 | // 将本次待写的记录内容写入到文件中。 101 | s = EmitPhysicalRecord(type, ptr, fragment_length); 102 | 103 | // 更新指向记录中剩余未写入的内容的指针及剩余未写入文件的记录长度 104 | ptr += fragment_length; 105 | left -= fragment_length; 106 | 107 | // 对于一条记录来说，写了一次如果没写完，则后续的写就不再是begin了， 108 | // 也就是不可能为kFullType或者kFirstType了。 109 | begin = false; 110 | } while (s.ok() && left > 0); 111 | 112 | return s; 113 | } 114 | 115 | Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { 116 | assert(n <= 0xffff); // Must fit in two bytes 117 | assert(block_offset_ + kHeaderSize + n <= kBlockSize); 118 | 119 | // Format the header 120 | // 构造本次写入的头部信息，头部信息长度为7个字节，其中四个字节的校验和， 121 | // 两个字节的内容长度，一个字节的记录类型信息 122 | char buf[kHeaderSize]; 123 | buf[4] = static_cast(n & 0xff); 124 | buf[5] = static_cast(n >> 8); 125 | buf[6] = static_cast(t); 126 | 127 | // Compute the crc of the record type and the payload. 128 | uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); 129 | crc = crc32c::Mask(crc); // Adjust for storage 130 | EncodeFixed32(buf, crc); 131 | 132 | // Write the header and the payload 133 | // 写入头部信息，写成功后紧接着写入记录内容信息，写成功后刷新到文件中。 134 | Status s = dest_->Append(Slice(buf, kHeaderSize)); 135 | if (s.ok()) { 136 | s = dest_->Append(Slice(ptr, n)); 137 | if (s.ok()) { 138 | s = dest_->Flush(); 139 | } 140 | } 141 | 142 | // 更新当前block中剩余可用空间起始地址相对于block起始地址的偏移。 143 | block_offset_ += kHeaderSize + n; 144 | return s; 145 | } 146 | 147 | } // namespace log 148 | } // namespace leveldb 149 | -------------------------------------------------------------------------------- /leveldb-master/db/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ 7 | 8 | #include 9 | #include "db/log_format.h" 10 | #include "leveldb/slice.h" 11 | #include "leveldb/status.h" 12 | 13 | namespace leveldb { 14 | 15 | class WritableFile; 16 | 17 | namespace log { 18 | 19 | class Writer { 20 | public: 21 | // Create a writer that will append data to "*dest". 22 | // "*dest" must be initially empty. 23 | // "*dest" must remain live while this Writer is in use. 24 | explicit Writer(WritableFile* dest); 25 | 26 | // Create a writer that will append data to "*dest". 27 | // "*dest" must have initial length "dest_length". 28 | // "*dest" must remain live while this Writer is in use. 29 | Writer(WritableFile* dest, uint64_t dest_length); 30 | 31 | ~Writer(); 32 | 33 | // 该函数用于向日志模块中添加一条操作日志。 34 | Status AddRecord(const Slice& slice); 35 | 36 | private: 37 | WritableFile* dest_; 38 | // 当前正在使用的block中未使用部分相对于block起始的偏移。 39 | int block_offset_; // Current offset in block 40 | 41 | // crc32c values for all supported record types. These are 42 | // pre-computed to reduce the overhead of computing the crc of the 43 | // record type stored in the header. 44 | uint32_t type_crc_[kMaxRecordType + 1]; 45 | 46 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 47 | 48 | // No copying allowed 49 | Writer(const Writer&); 50 | void operator=(const Writer&); 51 | }; 52 | 53 | } // namespace log 54 | } // namespace leveldb 55 | 56 | #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ 57 | -------------------------------------------------------------------------------- /leveldb-master/db/memtable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_ 6 | #define STORAGE_LEVELDB_DB_MEMTABLE_H_ 7 | 8 | #include 9 | #include "leveldb/db.h" 10 | #include "db/dbformat.h" 11 | #include "db/skiplist.h" 12 | #include "util/arena.h" 13 | 14 | namespace leveldb { 15 | 16 | class InternalKeyComparator; 17 | class Mutex; 18 | class MemTableIterator; 19 | 20 | class MemTable { 21 | public: 22 | // MemTables are reference counted. The initial reference count 23 | // is zero and the caller must call Ref() at least once. 24 | explicit MemTable(const InternalKeyComparator& comparator); 25 | 26 | // Increase reference count. 27 | void Ref() { ++refs_; } 28 | 29 | // Drop reference count. Delete if no more references exist. 30 | void Unref() { 31 | --refs_; 32 | assert(refs_ >= 0); 33 | if (refs_ <= 0) { 34 | delete this; 35 | } 36 | } 37 | 38 | // Returns an estimate of the number of bytes of data in use by this 39 | // data structure. It is safe to call when MemTable is being modified. 40 | size_t ApproximateMemoryUsage(); 41 | 42 | // Return an iterator that yields the contents of the memtable. 43 | // 44 | // The caller must ensure that the underlying MemTable remains live 45 | // while the returned iterator is live. The keys returned by this 46 | // iterator are internal keys encoded by AppendInternalKey in the 47 | // db/format.{h,cc} module. 48 | Iterator* NewIterator(); 49 | 50 | // Add an entry into memtable that maps key to value at the 51 | // specified sequence number and with the specified type. 52 | // Typically value will be empty if type==kTypeDeletion. 53 | void Add(SequenceNumber seq, ValueType type, 54 | const Slice& key, 55 | const Slice& value); 56 | 57 | // If memtable contains a value for key, store it in *value and return true. 58 | // If memtable contains a deletion for key, store a NotFound() error 59 | // in *status and return true. 60 | // Else, return false. 61 | bool Get(const LookupKey& key, std::string* value, Status* s); 62 | 63 | private: 64 | ~MemTable(); // Private since only Unref() should be used to delete it 65 | 66 | // MemTable类内部使用的键比较器类型。 67 | struct KeyComparator { 68 | const InternalKeyComparator comparator; 69 | explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } 70 | int operator()(const char* a, const char* b) const; 71 | }; 72 | 73 | friend class MemTableIterator; 74 | friend class MemTableBackwardIterator; 75 | 76 | // 定义一个MemTable用来存储元素的skiplist类型，该skiplist内部的key类型是字符串， 77 | // 所用的key比较器类型是KeyComparator 78 | typedef SkipList Table; 79 | 80 | 81 | // MemTable类实例用来比较内部元素大小的比较器。 82 | KeyComparator comparator_; 83 | 84 | // MemTable类实例的内部引用计数器。 85 | int refs_; 86 | 87 | // MemTable类实例用来给元素分配内存时使用的内存管理器。 88 | Arena arena_; 89 | 90 | // MemTable类实例用来存储元素的表，其内部实现其实是一个跳表，skiplist。 91 | Table table_; 92 | 93 | // No copying allowed 94 | MemTable(const MemTable&); 95 | void operator=(const MemTable&); 96 | }; 97 | 98 | } // namespace leveldb 99 | 100 | #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_ 101 | -------------------------------------------------------------------------------- /leveldb-master/db/snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ 6 | #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/db.h" 10 | 11 | namespace leveldb { 12 | 13 | class SnapshotList; 14 | 15 | // Snapshots are kept in a doubly-linked list in the DB. 16 | // Each SnapshotImpl corresponds to a particular sequence number. 17 | class SnapshotImpl : public Snapshot { 18 | public: 19 | SequenceNumber number_; // const after creation 20 | 21 | private: 22 | friend class SnapshotList; 23 | 24 | // SnapshotImpl is kept in a doubly-linked circular list 25 | SnapshotImpl* prev_; 26 | SnapshotImpl* next_; 27 | 28 | SnapshotList* list_; // just for sanity checks 29 | }; 30 | 31 | class SnapshotList { 32 | public: 33 | SnapshotList() { 34 | list_.prev_ = &list_; 35 | list_.next_ = &list_; 36 | } 37 | 38 | bool empty() const { return list_.next_ == &list_; } 39 | SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } 40 | SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } 41 | 42 | const SnapshotImpl* New(SequenceNumber seq) { 43 | SnapshotImpl* s = new SnapshotImpl; 44 | s->number_ = seq; 45 | s->list_ = this; 46 | s->next_ = &list_; 47 | s->prev_ = list_.prev_; 48 | s->prev_->next_ = s; 49 | s->next_->prev_ = s; 50 | return s; 51 | } 52 | 53 | void Delete(const SnapshotImpl* s) { 54 | assert(s->list_ == this); 55 | s->prev_->next_ = s->next_; 56 | s->next_->prev_ = s->prev_; 57 | delete s; 58 | } 59 | 60 | private: 61 | // Dummy head of doubly-linked list of snapshots 62 | SnapshotImpl list_; 63 | }; 64 | 65 | } // namespace leveldb 66 | 67 | #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ 68 | -------------------------------------------------------------------------------- /leveldb-master/db/table_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Thread-safe (provides internal synchronization) 6 | 7 | #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 8 | #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 9 | 10 | #include 11 | #include 12 | #include "db/dbformat.h" 13 | #include "leveldb/cache.h" 14 | #include "leveldb/table.h" 15 | #include "port/port.h" 16 | 17 | namespace leveldb { 18 | 19 | class Env; 20 | 21 | class TableCache { 22 | public: 23 | TableCache(const std::string& dbname, const Options* options, int entries); 24 | ~TableCache(); 25 | 26 | // Return an iterator for the specified file number (the corresponding 27 | // file length must be exactly "file_size" bytes). If "tableptr" is 28 | // non-NULL, also sets "*tableptr" to point to the Table object 29 | // underlying the returned iterator, or NULL if no Table object underlies 30 | // the returned iterator. The returned "*tableptr" object is owned by 31 | // the cache and should not be deleted, and is valid for as long as the 32 | // returned iterator is live. 33 | Iterator* NewIterator(const ReadOptions& options, 34 | uint64_t file_number, 35 | uint64_t file_size, 36 | Table** tableptr = NULL); 37 | 38 | // If a seek to internal key "k" in specified file finds an entry, 39 | // call (*handle_result)(arg, found_key, found_value). 40 | Status Get(const ReadOptions& options, 41 | uint64_t file_number, 42 | uint64_t file_size, 43 | const Slice& k, 44 | void* arg, 45 | void (*handle_result)(void*, const Slice&, const Slice&)); 46 | 47 | // Evict any entry for the specified file number 48 | void Evict(uint64_t file_number); 49 | 50 | private: 51 | Env* const env_; // env_是leveldb所处的环境信息 52 | const std::string dbname_; // dbname_是数据库名字 53 | const Options* options_; 54 | // LRUCache类实例指针，这个LRUCache用来存放TableAndFile类实例，其key值是 55 | // table对应的file_number。 56 | Cache* cache_; 57 | 58 | Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); 59 | }; 60 | 61 | } // namespace leveldb 62 | 63 | #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 64 | -------------------------------------------------------------------------------- /leveldb-master/db/version_edit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 6 | #define STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "db/dbformat.h" 12 | 13 | namespace leveldb { 14 | 15 | class VersionSet; 16 | 17 | // struct FileMetaData结构体存放着一个sstable文件的元信息 18 | struct FileMetaData { 19 | int refs; // sstable文件的引用计数 20 | 21 | // allowed_seeks存放的是该sstable文件在进行compact之前最大允许访问的次数 22 | int allowed_seeks; // Seeks allowed until compaction 23 | 24 | // sstable文件对应的file number 25 | uint64_t number; 26 | 27 | // sstable文件的大小 28 | uint64_t file_size; // File size in bytes 29 | 30 | // sstable文件中存放的最小的key 31 | InternalKey smallest; // Smallest internal key served by table 32 | 33 | // sstable文件中存放的最大的key 34 | InternalKey largest; // Largest internal key served by table 35 | 36 | FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { } 37 | }; 38 | 39 | // VersionEdit类保存的是一个版本变动的信息，在某个基准版本上面，应用一个或者多个 40 | // VersionEdit就可以得到新的版本。VersionEdit中存放了基于上一个版本增加的文件信息， 41 | // 删除的文件信息。 42 | class VersionEdit { 43 | public: 44 | VersionEdit() { Clear(); } 45 | ~VersionEdit() { } 46 | 47 | void Clear(); 48 | 49 | void SetComparatorName(const Slice& name) { 50 | has_comparator_ = true; 51 | comparator_ = name.ToString(); 52 | } 53 | void SetLogNumber(uint64_t num) { 54 | has_log_number_ = true; 55 | log_number_ = num; 56 | } 57 | void SetPrevLogNumber(uint64_t num) { 58 | has_prev_log_number_ = true; 59 | prev_log_number_ = num; 60 | } 61 | 62 | //设置下一个file number 63 | void SetNextFile(uint64_t num) { 64 | has_next_file_number_ = true; 65 | next_file_number_ = num; 66 | } 67 | void SetLastSequence(SequenceNumber seq) { 68 | has_last_sequence_ = true; 69 | last_sequence_ = seq; 70 | } 71 | void SetCompactPointer(int level, const InternalKey& key) { 72 | compact_pointers_.push_back(std::make_pair(level, key)); 73 | } 74 | 75 | // Add the specified file at the specified number. 76 | // REQUIRES: This version has not been saved (see VersionSet::SaveTo) 77 | // REQUIRES: "smallest" and "largest" are smallest and largest keys in file 78 | 79 | // AddFile()方法用于往VersionEdit类实例中添加版本变动需要增加的一个sstable文件， 80 | // 其中level是sstable文件所在的层数，file是sstable文件的file number，file_size是 81 | // sstable文件的大小，smallest和largest分别是sstable文件中存放的最小和最大的key信息。 82 | void AddFile(int level, uint64_t file, 83 | uint64_t file_size, 84 | const InternalKey& smallest, 85 | const InternalKey& largest) { 86 | FileMetaData f; 87 | f.number = file; 88 | f.file_size = file_size; 89 | f.smallest = smallest; 90 | f.largest = largest; 91 | new_files_.push_back(std::make_pair(level, f)); 92 | } 93 | 94 | // Delete the specified "file" from the specified "level". 95 | // DeleteFile()方法用于往VersionEdit类实例中添加一个版本变动需要被删除的sstable文件， 96 | // level是sstable文件所在的层级数，而file则是sstable文件的file number。 97 | void DeleteFile(int level, uint64_t file) { 98 | deleted_files_.insert(std::make_pair(level, file)); 99 | } 100 | 101 | void EncodeTo(std::string* dst) const; 102 | Status DecodeFrom(const Slice& src); 103 | 104 | std::string DebugString() const; 105 | 106 | private: 107 | friend class VersionSet; 108 | 109 | typedef std::set< std::pair > DeletedFileSet; 110 | 111 | // 比较器名字 112 | std::string comparator_; 113 | 114 | // log文件对应的FileNumber 115 | uint64_t log_number_; 116 | 117 | // 上一个log文件对应的FileNumber 118 | uint64_t prev_log_number_; 119 | 120 | // 下一个可用的FileNumber 121 | uint64_t next_file_number_; 122 | 123 | // 用过的最后一个SequenceNumber 124 | SequenceNumber last_sequence_; 125 | bool has_comparator_; 126 | bool has_log_number_; 127 | bool has_prev_log_number_; 128 | bool has_next_file_number_; 129 | bool has_last_sequence_; 130 | 131 | // 为了尽量均匀compact每个层级，所以会将这一次compact的end-key作为下一次 132 | // compact的start-key，compact_pointers_就保存了每一个level下一次compact的 133 | // start-key，compact_pointers_中的每个元素类型为std::pair， 134 | // pair中的first就是level信息，而second就是对应的key信息。 135 | std::vector< std::pair > compact_pointers_; 136 | 137 | // deleted_files_是一个集合set，存放了版本变动相对于基线版本来说要删除的sstable文件信息， 138 | // 其中std::pair的first部分是sstable所在的层级数level，而second部分则是 139 | // sstable文件的对应的file number。这里为什么要使用set呢？set里面的元素具有唯一性，这里 140 | // 使用set可以避免重复删除同一个文件。 141 | DeletedFileSet deleted_files_; 142 | 143 | // 版本变动相对于基线版本来说要增加的sstable文件信息，其中std::pair 144 | // 的first部分是sstable所在的层级数level，而second部分则是sstable文件的 145 | // 元信息。 146 | std::vector< std::pair > new_files_; 147 | }; 148 | 149 | } // namespace leveldb 150 | 151 | #endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 152 | -------------------------------------------------------------------------------- /leveldb-master/db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/version_edit.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | static void TestEncodeDecode(const VersionEdit& edit) { 11 | std::string encoded, encoded2; 12 | edit.EncodeTo(&encoded); 13 | VersionEdit parsed; 14 | Status s = parsed.DecodeFrom(encoded); 15 | ASSERT_TRUE(s.ok()) << s.ToString(); 16 | parsed.EncodeTo(&encoded2); 17 | ASSERT_EQ(encoded, encoded2); 18 | } 19 | 20 | class VersionEditTest { }; 21 | 22 | TEST(VersionEditTest, EncodeDecode) { 23 | static const uint64_t kBig = 1ull << 50; 24 | 25 | VersionEdit edit; 26 | for (int i = 0; i < 4; i++) { 27 | TestEncodeDecode(edit); 28 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 29 | InternalKey("foo", kBig + 500 + i, kTypeValue), 30 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 31 | edit.DeleteFile(4, kBig + 700 + i); 32 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 33 | } 34 | 35 | edit.SetComparatorName("foo"); 36 | edit.SetLogNumber(kBig + 100); 37 | edit.SetNextFile(kBig + 200); 38 | edit.SetLastSequence(kBig + 1000); 39 | TestEncodeDecode(edit); 40 | } 41 | 42 | } // namespace leveldb 43 | 44 | int main(int argc, char** argv) { 45 | return leveldb::test::RunAllTests(); 46 | } 47 | -------------------------------------------------------------------------------- /leveldb-master/db/write_batch.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch::rep_ := 6 | // sequence: fixed64 7 | // count: fixed32 8 | // data: record[count] 9 | // record := 10 | // kTypeValue varstring varstring | 11 | // kTypeDeletion varstring 12 | // varstring := 13 | // len: varint32 14 | // data: uint8[len] 15 | 16 | #include "leveldb/write_batch.h" 17 | 18 | #include "leveldb/db.h" 19 | #include "db/dbformat.h" 20 | #include "db/memtable.h" 21 | #include "db/write_batch_internal.h" 22 | #include "util/coding.h" 23 | 24 | namespace leveldb { 25 | 26 | // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. 27 | static const size_t kHeader = 12; 28 | 29 | WriteBatch::WriteBatch() { 30 | Clear(); 31 | } 32 | 33 | WriteBatch::~WriteBatch() { } 34 | 35 | WriteBatch::Handler::~Handler() { } 36 | 37 | void WriteBatch::Clear() { 38 | rep_.clear(); 39 | rep_.resize(kHeader); 40 | } 41 | 42 | size_t WriteBatch::ApproximateSize() { 43 | return rep_.size(); 44 | } 45 | 46 | Status WriteBatch::Iterate(Handler* handler) const { 47 | Slice input(rep_); 48 | if (input.size() < kHeader) { 49 | return Status::Corruption("malformed WriteBatch (too small)"); 50 | } 51 | 52 | input.remove_prefix(kHeader); 53 | Slice key, value; 54 | int found = 0; 55 | while (!input.empty()) { 56 | found++; 57 | char tag = input[0]; 58 | input.remove_prefix(1); 59 | switch (tag) { 60 | case kTypeValue: 61 | if (GetLengthPrefixedSlice(&input, &key) && 62 | GetLengthPrefixedSlice(&input, &value)) { 63 | handler->Put(key, value); 64 | } else { 65 | return Status::Corruption("bad WriteBatch Put"); 66 | } 67 | break; 68 | case kTypeDeletion: 69 | if (GetLengthPrefixedSlice(&input, &key)) { 70 | handler->Delete(key); 71 | } else { 72 | return Status::Corruption("bad WriteBatch Delete"); 73 | } 74 | break; 75 | default: 76 | return Status::Corruption("unknown WriteBatch tag"); 77 | } 78 | } 79 | if (found != WriteBatchInternal::Count(this)) { 80 | return Status::Corruption("WriteBatch has wrong count"); 81 | } else { 82 | return Status::OK(); 83 | } 84 | } 85 | 86 | int WriteBatchInternal::Count(const WriteBatch* b) { 87 | return DecodeFixed32(b->rep_.data() + 8); 88 | } 89 | 90 | void WriteBatchInternal::SetCount(WriteBatch* b, int n) { 91 | EncodeFixed32(&b->rep_[8], n); 92 | } 93 | 94 | SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { 95 | return SequenceNumber(DecodeFixed64(b->rep_.data())); 96 | } 97 | 98 | void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { 99 | EncodeFixed64(&b->rep_[0], seq); 100 | } 101 | 102 | void WriteBatch::Put(const Slice& key, const Slice& value) { 103 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 104 | rep_.push_back(static_cast(kTypeValue)); 105 | PutLengthPrefixedSlice(&rep_, key); 106 | PutLengthPrefixedSlice(&rep_, value); 107 | } 108 | 109 | void WriteBatch::Delete(const Slice& key) { 110 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 111 | rep_.push_back(static_cast(kTypeDeletion)); 112 | PutLengthPrefixedSlice(&rep_, key); 113 | } 114 | 115 | namespace { 116 | class MemTableInserter : public WriteBatch::Handler { 117 | public: 118 | SequenceNumber sequence_; 119 | MemTable* mem_; 120 | 121 | virtual void Put(const Slice& key, const Slice& value) { 122 | mem_->Add(sequence_, kTypeValue, key, value); 123 | sequence_++; 124 | } 125 | virtual void Delete(const Slice& key) { 126 | mem_->Add(sequence_, kTypeDeletion, key, Slice()); 127 | sequence_++; 128 | } 129 | }; 130 | } // namespace 131 | 132 | Status WriteBatchInternal::InsertInto(const WriteBatch* b, 133 | MemTable* memtable) { 134 | MemTableInserter inserter; 135 | inserter.sequence_ = WriteBatchInternal::Sequence(b); 136 | inserter.mem_ = memtable; 137 | return b->Iterate(&inserter); 138 | } 139 | 140 | void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { 141 | assert(contents.size() >= kHeader); 142 | b->rep_.assign(contents.data(), contents.size()); 143 | } 144 | 145 | void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) { 146 | SetCount(dst, Count(dst) + Count(src)); 147 | assert(src->rep_.size() >= kHeader); 148 | dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader); 149 | } 150 | 151 | } // namespace leveldb 152 | -------------------------------------------------------------------------------- /leveldb-master/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include "db/dbformat.h" 9 | #include "leveldb/write_batch.h" 10 | 11 | namespace leveldb { 12 | 13 | class MemTable; 14 | 15 | // WriteBatchInternal provides static methods for manipulating a 16 | // WriteBatch that we don't want in the public WriteBatch interface. 17 | class WriteBatchInternal { 18 | public: 19 | // Return the number of entries in the batch. 20 | static int Count(const WriteBatch* batch); 21 | 22 | // Set the count for the number of entries in the batch. 23 | static void SetCount(WriteBatch* batch, int n); 24 | 25 | // Return the sequence number for the start of this batch. 26 | static SequenceNumber Sequence(const WriteBatch* batch); 27 | 28 | // Store the specified number as the sequence number for the start of 29 | // this batch. 30 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 31 | 32 | static Slice Contents(const WriteBatch* batch) { 33 | return Slice(batch->rep_); 34 | } 35 | 36 | static size_t ByteSize(const WriteBatch* batch) { 37 | return batch->rep_.size(); 38 | } 39 | 40 | static void SetContents(WriteBatch* batch, const Slice& contents); 41 | 42 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 43 | 44 | static void Append(WriteBatch* dst, const WriteBatch* src); 45 | }; 46 | 47 | } // namespace leveldb 48 | 49 | 50 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 51 | -------------------------------------------------------------------------------- /leveldb-master/db/write_batch_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/db.h" 6 | 7 | #include "db/memtable.h" 8 | #include "db/write_batch_internal.h" 9 | #include "leveldb/env.h" 10 | #include "util/logging.h" 11 | #include "util/testharness.h" 12 | 13 | namespace leveldb { 14 | 15 | static std::string PrintContents(WriteBatch* b) { 16 | InternalKeyComparator cmp(BytewiseComparator()); 17 | MemTable* mem = new MemTable(cmp); 18 | mem->Ref(); 19 | std::string state; 20 | Status s = WriteBatchInternal::InsertInto(b, mem); 21 | int count = 0; 22 | Iterator* iter = mem->NewIterator(); 23 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 24 | ParsedInternalKey ikey; 25 | ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); 26 | switch (ikey.type) { 27 | case kTypeValue: 28 | state.append("Put("); 29 | state.append(ikey.user_key.ToString()); 30 | state.append(", "); 31 | state.append(iter->value().ToString()); 32 | state.append(")"); 33 | count++; 34 | break; 35 | case kTypeDeletion: 36 | state.append("Delete("); 37 | state.append(ikey.user_key.ToString()); 38 | state.append(")"); 39 | count++; 40 | break; 41 | } 42 | state.append("@"); 43 | state.append(NumberToString(ikey.sequence)); 44 | } 45 | delete iter; 46 | if (!s.ok()) { 47 | state.append("ParseError()"); 48 | } else if (count != WriteBatchInternal::Count(b)) { 49 | state.append("CountMismatch()"); 50 | } 51 | mem->Unref(); 52 | return state; 53 | } 54 | 55 | class WriteBatchTest { }; 56 | 57 | TEST(WriteBatchTest, Empty) { 58 | WriteBatch batch; 59 | ASSERT_EQ("", PrintContents(&batch)); 60 | ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); 61 | } 62 | 63 | TEST(WriteBatchTest, Multiple) { 64 | WriteBatch batch; 65 | batch.Put(Slice("foo"), Slice("bar")); 66 | batch.Delete(Slice("box")); 67 | batch.Put(Slice("baz"), Slice("boo")); 68 | WriteBatchInternal::SetSequence(&batch, 100); 69 | ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); 70 | ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); 71 | ASSERT_EQ("Put(baz, boo)@102" 72 | "Delete(box)@101" 73 | "Put(foo, bar)@100", 74 | PrintContents(&batch)); 75 | } 76 | 77 | TEST(WriteBatchTest, Corruption) { 78 | WriteBatch batch; 79 | batch.Put(Slice("foo"), Slice("bar")); 80 | batch.Delete(Slice("box")); 81 | WriteBatchInternal::SetSequence(&batch, 200); 82 | Slice contents = WriteBatchInternal::Contents(&batch); 83 | WriteBatchInternal::SetContents(&batch, 84 | Slice(contents.data(),contents.size()-1)); 85 | ASSERT_EQ("Put(foo, bar)@200" 86 | "ParseError()", 87 | PrintContents(&batch)); 88 | } 89 | 90 | TEST(WriteBatchTest, Append) { 91 | WriteBatch b1, b2; 92 | WriteBatchInternal::SetSequence(&b1, 200); 93 | WriteBatchInternal::SetSequence(&b2, 300); 94 | WriteBatchInternal::Append(&b1, &b2); 95 | ASSERT_EQ("", 96 | PrintContents(&b1)); 97 | b2.Put("a", "va"); 98 | WriteBatchInternal::Append(&b1, &b2); 99 | ASSERT_EQ("Put(a, va)@200", 100 | PrintContents(&b1)); 101 | b2.Clear(); 102 | b2.Put("b", "vb"); 103 | WriteBatchInternal::Append(&b1, &b2); 104 | ASSERT_EQ("Put(a, va)@200" 105 | "Put(b, vb)@201", 106 | PrintContents(&b1)); 107 | b2.Delete("foo"); 108 | WriteBatchInternal::Append(&b1, &b2); 109 | ASSERT_EQ("Put(a, va)@200" 110 | "Put(b, vb)@202" 111 | "Put(b, vb)@201" 112 | "Delete(foo)@203", 113 | PrintContents(&b1)); 114 | } 115 | 116 | TEST(WriteBatchTest, ApproximateSize) { 117 | WriteBatch batch; 118 | size_t empty_size = batch.ApproximateSize(); 119 | 120 | batch.Put(Slice("foo"), Slice("bar")); 121 | size_t one_key_size = batch.ApproximateSize(); 122 | ASSERT_LT(empty_size, one_key_size); 123 | 124 | batch.Put(Slice("baz"), Slice("boo")); 125 | size_t two_keys_size = batch.ApproximateSize(); 126 | ASSERT_LT(one_key_size, two_keys_size); 127 | 128 | batch.Delete(Slice("box")); 129 | size_t post_delete_size = batch.ApproximateSize(); 130 | ASSERT_LT(two_keys_size, post_delete_size); 131 | } 132 | 133 | } // namespace leveldb 134 | 135 | int main(int argc, char** argv) { 136 | return leveldb::test::RunAllTests(); 137 | } 138 | -------------------------------------------------------------------------------- /leveldb-master/doc/log_format.md: -------------------------------------------------------------------------------- 1 | leveldb Log format 2 | ================== 3 | The log file contents are a sequence of 32KB blocks. The only exception is that 4 | the tail of the file may contain a partial block. 5 | 6 | Each block consists of a sequence of records: 7 | 8 | block := record* trailer? 9 | record := 10 | checksum: uint32 // crc32c of type and data[] ; little-endian 11 | length: uint16 // little-endian 12 | type: uint8 // One of FULL, FIRST, MIDDLE, LAST 13 | data: uint8[length] 14 | 15 | A record never starts within the last six bytes of a block (since it won't fit). 16 | Any leftover bytes here form the trailer, which must consist entirely of zero 17 | bytes and must be skipped by readers. 18 | 19 | Aside: if exactly seven bytes are left in the current block, and a new non-zero 20 | length record is added, the writer must emit a FIRST record (which contains zero 21 | bytes of user data) to fill up the trailing seven bytes of the block and then 22 | emit all of the user data in subsequent blocks. 23 | 24 | More types may be added in the future. Some Readers may skip record types they 25 | do not understand, others may report that some data was skipped. 26 | 27 | FULL == 1 28 | FIRST == 2 29 | MIDDLE == 3 30 | LAST == 4 31 | 32 | The FULL record contains the contents of an entire user record. 33 | 34 | FIRST, MIDDLE, LAST are types used for user records that have been split into 35 | multiple fragments (typically because of block boundaries). FIRST is the type 36 | of the first fragment of a user record, LAST is the type of the last fragment of 37 | a user record, and MIDDLE is the type of all interior fragments of a user 38 | record. 39 | 40 | Example: consider a sequence of user records: 41 | 42 | A: length 1000 43 | B: length 97270 44 | C: length 8000 45 | 46 | **A** will be stored as a FULL record in the first block. 47 | 48 | **B** will be split into three fragments: first fragment occupies the rest of 49 | the first block, second fragment occupies the entirety of the second block, and 50 | the third fragment occupies a prefix of the third block. This will leave six 51 | bytes free in the third block, which will be left empty as the trailer. 52 | 53 | **C** will be stored as a FULL record in the fourth block. 54 | 55 | ---- 56 | 57 | ## Some benefits over the recordio format: 58 | 59 | 1. We do not need any heuristics for resyncing - just go to next block boundary 60 | and scan. If there is a corruption, skip to the next block. As a 61 | side-benefit, we do not get confused when part of the contents of one log 62 | file are embedded as a record inside another log file. 63 | 64 | 2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the 65 | next block boundary and skip records until we hit a FULL or FIRST record. 66 | 67 | 3. We do not need extra buffering for large records. 68 | 69 | ## Some downsides compared to recordio format: 70 | 71 | 1. No packing of tiny records. This could be fixed by adding a new record type, 72 | so it is a shortcoming of the current implementation, not necessarily the 73 | format. 74 | 75 | 2. No compression. Again, this could be fixed by adding new record types. 76 | -------------------------------------------------------------------------------- /leveldb-master/doc/table_format.md: -------------------------------------------------------------------------------- 1 | leveldb File format 2 | =================== 3 | 4 | 5 | [data block 1] 6 | [data block 2] 7 | ... 8 | [data block N] 9 | [meta block 1] 10 | ... 11 | [meta block K] 12 | [metaindex block] 13 | [index block] 14 | [Footer] (fixed size; starts at file_size - sizeof(Footer)) 15 | 16 | 17 | The file contains internal pointers. Each such pointer is called 18 | a BlockHandle and contains the following information: 19 | 20 | offset: varint64 21 | size: varint64 22 | 23 | See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints) 24 | for an explanation of varint64 format. 25 | 26 | 1. The sequence of key/value pairs in the file are stored in sorted 27 | order and partitioned into a sequence of data blocks. These blocks 28 | come one after another at the beginning of the file. Each data block 29 | is formatted according to the code in `block_builder.cc`, and then 30 | optionally compressed. 31 | 32 | 2. After the data blocks we store a bunch of meta blocks. The 33 | supported meta block types are described below. More meta block types 34 | may be added in the future. Each meta block is again formatted using 35 | `block_builder.cc` and then optionally compressed. 36 | 37 | 3. A "metaindex" block. It contains one entry for every other meta 38 | block where the key is the name of the meta block and the value is a 39 | BlockHandle pointing to that meta block. 40 | 41 | 4. An "index" block. This block contains one entry per data block, 42 | where the key is a string >= last key in that data block and before 43 | the first key in the successive data block. The value is the 44 | BlockHandle for the data block. 45 | 46 | 5. At the very end of the file is a fixed length footer that contains 47 | the BlockHandle of the metaindex and index blocks as well as a magic number. 48 | 49 | metaindex_handle: char[p]; // Block handle for metaindex 50 | index_handle: char[q]; // Block handle for index 51 | padding: char[40-p-q];// zeroed bytes to make fixed length 52 | // (40==2*BlockHandle::kMaxEncodedLength) 53 | magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) 54 | 55 | ## "filter" Meta Block 56 | 57 | If a `FilterPolicy` was specified when the database was opened, a 58 | filter block is stored in each table. The "metaindex" block contains 59 | an entry that maps from `filter.` to the BlockHandle for the filter 60 | block where `` is the string returned by the filter policy's 61 | `Name()` method. 62 | 63 | The filter block stores a sequence of filters, where filter i contains 64 | the output of `FilterPolicy::CreateFilter()` on all keys that are stored 65 | in a block whose file offset falls within the range 66 | 67 | [ i*base ... (i+1)*base-1 ] 68 | 69 | Currently, "base" is 2KB. So for example, if blocks X and Y start in 70 | the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be 71 | converted to a filter by calling `FilterPolicy::CreateFilter()`, and the 72 | resulting filter will be stored as the first filter in the filter 73 | block. 74 | 75 | The filter block is formatted as follows: 76 | 77 | [filter 0] 78 | [filter 1] 79 | [filter 2] 80 | ... 81 | [filter N-1] 82 | 83 | [offset of filter 0] : 4 bytes 84 | [offset of filter 1] : 4 bytes 85 | [offset of filter 2] : 4 bytes 86 | ... 87 | [offset of filter N-1] : 4 bytes 88 | 89 | [offset of beginning of offset array] : 4 bytes 90 | lg(base) : 1 byte 91 | 92 | The offset array at the end of the filter block allows efficient 93 | mapping from a data block offset to the corresponding filter. 94 | 95 | ## "stats" Meta Block 96 | 97 | This meta block contains a bunch of stats. The key is the name 98 | of the statistic. The value contains the statistic. 99 | 100 | TODO(postrelease): record following stats. 101 | 102 | data size 103 | index size 104 | key size (uncompressed) 105 | value size (uncompressed) 106 | number of entries 107 | number of data blocks 108 | -------------------------------------------------------------------------------- /leveldb-master/helpers/memenv/memenv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 6 | #define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Env; 11 | 12 | // Returns a new environment that stores its data in memory and delegates 13 | // all non-file-storage tasks to base_env. The caller must delete the result 14 | // when it is no longer needed. 15 | // *base_env must remain live while the result is in use. 16 | Env* NewMemEnv(Env* base_env); 17 | 18 | } // namespace leveldb 19 | 20 | #endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_ 21 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Cache is an interface that maps keys to values. It has internal 6 | // synchronization and may be safely accessed concurrently from 7 | // multiple threads. It may automatically evict entries to make room 8 | // for new entries. Values have a specified charge against the cache 9 | // capacity. For example, a cache where the values are variable 10 | // length strings, may use the length of the string as the charge for 11 | // the string. 12 | // 13 | // A builtin cache implementation with a least-recently-used eviction 14 | // policy is provided. Clients may use their own implementations if 15 | // they want something more sophisticated (like scan-resistance, a 16 | // custom eviction policy, variable cache sizing, etc.) 17 | 18 | #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ 19 | #define STORAGE_LEVELDB_INCLUDE_CACHE_H_ 20 | 21 | #include 22 | #include "leveldb/export.h" 23 | #include "leveldb/slice.h" 24 | 25 | namespace leveldb { 26 | 27 | class LEVELDB_EXPORT Cache; 28 | 29 | // Create a new cache with a fixed size capacity. This implementation 30 | // of Cache uses a least-recently-used eviction policy. 31 | LEVELDB_EXPORT Cache* NewLRUCache(size_t capacity); 32 | 33 | // Cache类是一个抽象类，定义了缓存类的基本接口 34 | class LEVELDB_EXPORT Cache { 35 | public: 36 | Cache() { } 37 | 38 | // Destroys all existing entries by calling the "deleter" 39 | // function that was passed to the constructor. 40 | virtual ~Cache(); 41 | 42 | // Opaque handle to an entry stored in the cache. 43 | struct Handle { }; 44 | 45 | // Insert a mapping from key->value into the cache and assign it 46 | // the specified charge against the total cache capacity. 47 | // 48 | // Returns a handle that corresponds to the mapping. The caller 49 | // must call this->Release(handle) when the returned mapping is no 50 | // longer needed. 51 | // 52 | // When the inserted entry is no longer needed, the key and 53 | // value will be passed to "deleter". 54 | virtual Handle* Insert(const Slice& key, void* value, size_t charge, 55 | void (*deleter)(const Slice& key, void* value)) = 0; 56 | 57 | // If the cache has no mapping for "key", returns NULL. 58 | // 59 | // Else return a handle that corresponds to the mapping. The caller 60 | // must call this->Release(handle) when the returned mapping is no 61 | // longer needed. 62 | virtual Handle* Lookup(const Slice& key) = 0; 63 | 64 | // Release a mapping returned by a previous Lookup(). 65 | // REQUIRES: handle must not have been released yet. 66 | // REQUIRES: handle must have been returned by a method on *this. 67 | virtual void Release(Handle* handle) = 0; 68 | 69 | // Return the value encapsulated in a handle returned by a 70 | // successful Lookup(). 71 | // REQUIRES: handle must not have been released yet. 72 | // REQUIRES: handle must have been returned by a method on *this. 73 | virtual void* Value(Handle* handle) = 0; 74 | 75 | // If the cache contains entry for key, erase it. Note that the 76 | // underlying entry will be kept around until all existing handles 77 | // to it have been released. 78 | virtual void Erase(const Slice& key) = 0; 79 | 80 | // Return a new numeric id. May be used by multiple clients who are 81 | // sharing the same cache to partition the key space. Typically the 82 | // client will allocate a new id at startup and prepend the id to 83 | // its cache keys. 84 | virtual uint64_t NewId() = 0; 85 | 86 | // Remove all cache entries that are not actively in use. Memory-constrained 87 | // applications may wish to call this method to reduce memory usage. 88 | // Default implementation of Prune() does nothing. Subclasses are strongly 89 | // encouraged to override the default implementation. A future release of 90 | // leveldb may change Prune() to a pure abstract method. 91 | virtual void Prune() {} 92 | 93 | // Return an estimate of the combined charges of all elements stored in the 94 | // cache. 95 | virtual size_t TotalCharge() const = 0; 96 | 97 | private: 98 | void LRU_Remove(Handle* e); 99 | void LRU_Append(Handle* e); 100 | void Unref(Handle* e); 101 | 102 | struct Rep; 103 | Rep* rep_; 104 | 105 | // No copying allowed 106 | Cache(const Cache&); 107 | void operator=(const Cache&); 108 | }; 109 | 110 | } // namespace leveldb 111 | 112 | #endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_ 113 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 7 | 8 | #include 9 | #include "leveldb/export.h" 10 | 11 | namespace leveldb { 12 | 13 | class Slice; 14 | 15 | // A Comparator object provides a total order across slices that are 16 | // used as keys in an sstable or a database. A Comparator implementation 17 | // must be thread-safe since leveldb may invoke its methods concurrently 18 | // from multiple threads. 19 | // Comparator类是一个抽象类，定义了比较器应该实现的接口。 20 | class LEVELDB_EXPORT Comparator { 21 | public: 22 | virtual ~Comparator(); 23 | 24 | // Three-way comparison. Returns value: 25 | // < 0 iff "a" < "b", 26 | // == 0 iff "a" == "b", 27 | // > 0 iff "a" > "b" 28 | virtual int Compare(const Slice& a, const Slice& b) const = 0; 29 | 30 | // The name of the comparator. Used to check for comparator 31 | // mismatches (i.e., a DB created with one comparator is 32 | // accessed using a different comparator. 33 | // 34 | // The client of this package should switch to a new name whenever 35 | // the comparator implementation changes in a way that will cause 36 | // the relative ordering of any two keys to change. 37 | // 38 | // Names starting with "leveldb." are reserved and should not be used 39 | // by any clients of this package. 40 | virtual const char* Name() const = 0; 41 | 42 | // Advanced functions: these are used to reduce the space requirements 43 | // for internal data structures like index blocks. 44 | 45 | // If *start < limit, changes *start to a short string in [start,limit). 46 | // Simple comparator implementations may return with *start unchanged, 47 | // i.e., an implementation of this method that does nothing is correct. 48 | virtual void FindShortestSeparator( 49 | std::string* start, 50 | const Slice& limit) const = 0; 51 | 52 | // Changes *key to a short string >= *key. 53 | // Simple comparator implementations may return with *key unchanged, 54 | // i.e., an implementation of this method that does nothing is correct. 55 | virtual void FindShortSuccessor(std::string* key) const = 0; 56 | }; 57 | 58 | // Return a builtin comparator that uses lexicographic byte-wise 59 | // ordering. The result remains the property of this module and 60 | // must not be deleted. 61 | LEVELDB_EXPORT const Comparator* BytewiseComparator(); 62 | 63 | } // namespace leveldb 64 | 65 | #endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 66 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/dumpfile.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 7 | 8 | #include 9 | #include "leveldb/env.h" 10 | #include "leveldb/export.h" 11 | #include "leveldb/status.h" 12 | 13 | namespace leveldb { 14 | 15 | // Dump the contents of the file named by fname in text format to 16 | // *dst. Makes a sequence of dst->Append() calls; each call is passed 17 | // the newline-terminated text corresponding to a single item found 18 | // in the file. 19 | // 20 | // Returns a non-OK result if fname does not name a leveldb storage 21 | // file, or if the file cannot be read. 22 | LEVELDB_EXPORT Status DumpFile(Env* env, const std::string& fname, 23 | WritableFile* dst); 24 | 25 | } // namespace leveldb 26 | 27 | #endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_ 28 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/export.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 7 | 8 | #if !defined(LEVELDB_EXPORT) 9 | 10 | #if defined(LEVELDB_SHARED_LIBRARY) 11 | #if defined(OS_WIN) 12 | 13 | #if defined(LEVELDB_COMPILE_LIBRARY) 14 | #define LEVELDB_EXPORT __declspec(dllexport) 15 | #else 16 | #define LEVELDB_EXPORT __declspec(dllimport) 17 | #endif // defined(LEVELDB_COMPILE_LIBRARY) 18 | 19 | #else // defined(OS_WIN) 20 | #if defined(LEVELDB_COMPILE_LIBRARY) 21 | #define LEVELDB_EXPORT __attribute__((visibility("default"))) 22 | #else 23 | #define LEVELDB_EXPORT 24 | #endif 25 | #endif // defined(OS_WIN) 26 | 27 | #else // defined(LEVELDB_SHARED_LIBRARY) 28 | #define LEVELDB_EXPORT 29 | #endif 30 | 31 | #endif // !defined(LEVELDB_EXPORT) 32 | 33 | #endif // STORAGE_LEVELDB_INCLUDE_EXPORT_H_ 34 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/filter_policy.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A database can be configured with a custom FilterPolicy object. 6 | // This object is responsible for creating a small filter from a set 7 | // of keys. These filters are stored in leveldb and are consulted 8 | // automatically by leveldb to decide whether or not to read some 9 | // information from disk. In many cases, a filter can cut down the 10 | // number of disk seeks form a handful to a single disk seek per 11 | // DB::Get() call. 12 | // 13 | // Most people will want to use the builtin bloom filter support (see 14 | // NewBloomFilterPolicy() below). 15 | 16 | #ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 17 | #define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 18 | 19 | #include 20 | #include "leveldb/export.h" 21 | 22 | namespace leveldb { 23 | 24 | class Slice; 25 | 26 | // 过滤器的抽象类 27 | class LEVELDB_EXPORT FilterPolicy { 28 | public: 29 | virtual ~FilterPolicy(); 30 | 31 | // Return the name of this policy. Note that if the filter encoding 32 | // changes in an incompatible way, the name returned by this method 33 | // must be changed. Otherwise, old incompatible filters may be 34 | // passed to methods of this type. 35 | virtual const char* Name() const = 0; 36 | 37 | // keys[0,n-1] contains a list of keys (potentially with duplicates) 38 | // that are ordered according to the user supplied comparator. 39 | // Append a filter that summarizes keys[0,n-1] to *dst. 40 | // 41 | // Warning: do not change the initial contents of *dst. Instead, 42 | // append the newly constructed filter to *dst. 43 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) 44 | const = 0; 45 | 46 | // "filter" contains the data appended by a preceding call to 47 | // CreateFilter() on this class. This method must return true if 48 | // the key was in the list of keys passed to CreateFilter(). 49 | // This method may return true or false if the key was not on the 50 | // list, but it should aim to return false with a high probability. 51 | virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0; 52 | }; 53 | 54 | // Return a new filter policy that uses a bloom filter with approximately 55 | // the specified number of bits per key. A good value for bits_per_key 56 | // is 10, which yields a filter with ~ 1% false positive rate. 57 | // 58 | // Callers must delete the result after any database that is using the 59 | // result has been closed. 60 | // 61 | // Note: if you are using a custom comparator that ignores some parts 62 | // of the keys being compared, you must not use NewBloomFilterPolicy() 63 | // and must provide your own FilterPolicy that also ignores the 64 | // corresponding parts of the keys. For example, if the comparator 65 | // ignores trailing spaces, it would be incorrect to use a 66 | // FilterPolicy (like NewBloomFilterPolicy) that does not ignore 67 | // trailing spaces in keys. 68 | LEVELDB_EXPORT const FilterPolicy* NewBloomFilterPolicy(int bits_per_key); 69 | } 70 | 71 | #endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ 72 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // An iterator yields a sequence of key/value pairs from a source. 6 | // The following class defines the interface. Multiple implementations 7 | // are provided by this library. In particular, iterators are provided 8 | // to access the contents of a Table or a DB. 9 | // 10 | // Multiple threads can invoke const methods on an Iterator without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Iterator must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 17 | 18 | #include "leveldb/export.h" 19 | #include "leveldb/slice.h" 20 | #include "leveldb/status.h" 21 | 22 | namespace leveldb { 23 | 24 | // Itertor类是一个抽象类，定义了一个迭代器实现类需要实现的接口。 25 | class LEVELDB_EXPORT Iterator { 26 | public: 27 | Iterator(); 28 | virtual ~Iterator(); 29 | 30 | // An iterator is either positioned at a key/value pair, or 31 | // not valid. This method returns true iff the iterator is valid. 32 | virtual bool Valid() const = 0; 33 | 34 | // Position at the first key in the source. The iterator is Valid() 35 | // after this call iff the source is not empty. 36 | virtual void SeekToFirst() = 0; 37 | 38 | // Position at the last key in the source. The iterator is 39 | // Valid() after this call iff the source is not empty. 40 | virtual void SeekToLast() = 0; 41 | 42 | // Position at the first key in the source that is at or past target. 43 | // The iterator is Valid() after this call iff the source contains 44 | // an entry that comes at or past target. 45 | virtual void Seek(const Slice& target) = 0; 46 | 47 | // Moves to the next entry in the source. After this call, Valid() is 48 | // true iff the iterator was not positioned at the last entry in the source. 49 | // REQUIRES: Valid() 50 | virtual void Next() = 0; 51 | 52 | // Moves to the previous entry in the source. After this call, Valid() is 53 | // true iff the iterator was not positioned at the first entry in source. 54 | // REQUIRES: Valid() 55 | virtual void Prev() = 0; 56 | 57 | // Return the key for the current entry. The underlying storage for 58 | // the returned slice is valid only until the next modification of 59 | // the iterator. 60 | // REQUIRES: Valid() 61 | virtual Slice key() const = 0; 62 | 63 | // Return the value for the current entry. The underlying storage for 64 | // the returned slice is valid only until the next modification of 65 | // the iterator. 66 | // REQUIRES: Valid() 67 | virtual Slice value() const = 0; 68 | 69 | // If an error has occurred, return it. Else return an ok status. 70 | virtual Status status() const = 0; 71 | 72 | // Clients are allowed to register function/arg1/arg2 triples that 73 | // will be invoked when this iterator is destroyed. 74 | // 75 | // Note that unlike all of the preceding methods, this method is 76 | // not abstract and therefore clients should not override it. 77 | typedef void (*CleanupFunction)(void* arg1, void* arg2); 78 | void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); 79 | 80 | private: 81 | struct Cleanup { 82 | CleanupFunction function; 83 | void* arg1; 84 | void* arg2; 85 | Cleanup* next; 86 | }; 87 | Cleanup cleanup_; 88 | 89 | // No copying allowed 90 | Iterator(const Iterator&); 91 | void operator=(const Iterator&); 92 | }; 93 | 94 | // Return an empty iterator (yields nothing). 95 | LEVELDB_EXPORT Iterator* NewEmptyIterator(); 96 | 97 | // Return an empty iterator with the specified status. 98 | LEVELDB_EXPORT Iterator* NewErrorIterator(const Status& status); 99 | 100 | } // namespace leveldb 101 | 102 | #endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 103 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Slice is a simple structure containing a pointer into some external 6 | // storage and a size. The user of a Slice must ensure that the slice 7 | // is not used after the corresponding external storage has been 8 | // deallocated. 9 | // 10 | // Multiple threads can invoke const methods on a Slice without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Slice must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_SLICE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include "leveldb/export.h" 23 | 24 | namespace leveldb { 25 | 26 | // Slice类封装了一个数据片段及其相关的操作。数据片段主要是通过内部的data_和size_成员 27 | // 实现的，data_字段是一个char*类型，一般指向一个字符数组，size_则是这个字符数组的长度。 28 | class LEVELDB_EXPORT Slice { 29 | public: 30 | // Create an empty slice. 31 | Slice() : data_(""), size_(0) { } 32 | 33 | // Create a slice that refers to d[0,n-1]. 34 | Slice(const char* d, size_t n) : data_(d), size_(n) { } 35 | 36 | // Create a slice that refers to the contents of "s" 37 | Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } 38 | 39 | // Create a slice that refers to s[0,strlen(s)-1] 40 | Slice(const char* s) : data_(s), size_(strlen(s)) { } 41 | 42 | // Return a pointer to the beginning of the referenced data 43 | const char* data() const { return data_; } 44 | 45 | // Return the length (in bytes) of the referenced data 46 | size_t size() const { return size_; } 47 | 48 | // Return true iff the length of the referenced data is zero 49 | bool empty() const { return size_ == 0; } 50 | 51 | // Return the ith byte in the referenced data. 52 | // REQUIRES: n < size() 53 | // 重载Slice类实例的下标操作符。 54 | char operator[](size_t n) const { 55 | assert(n < size()); 56 | return data_[n]; 57 | } 58 | 59 | // Change this slice to refer to an empty array 60 | // clear()用于清空Slice类实例的内部数据 61 | void clear() { data_ = ""; size_ = 0; } 62 | 63 | // Drop the first "n" bytes from this slice. 64 | // remove_prefix()函数用于移除Slice实例自身的前n个字节数据。 65 | void remove_prefix(size_t n) { 66 | assert(n <= size()); 67 | data_ += n; 68 | size_ -= n; 69 | } 70 | 71 | // Return a string that contains the copy of the referenced data. 72 | // ToString()用于将Slice实例信息封装成一个string类型的对象 73 | std::string ToString() const { return std::string(data_, size_); } 74 | 75 | // Three-way comparison. Returns value: 76 | // < 0 iff "*this" < "b", 77 | // == 0 iff "*this" == "b", 78 | // > 0 iff "*this" > "b" 79 | int compare(const Slice& b) const; 80 | 81 | // Return true iff "x" is a prefix of "*this" 82 | // starts_with()函数用于判断Slice实例x是不是以Slice实例this作为前缀。 83 | bool starts_with(const Slice& x) const { 84 | return ((size_ >= x.size_) && 85 | (memcmp(data_, x.data_, x.size_) == 0)); 86 | } 87 | 88 | private: 89 | //data_成员通常指向一个字符数组，而size_则是字符数组的长度。 90 | const char* data_; 91 | size_t size_; 92 | 93 | // Intentionally copyable 94 | }; 95 | 96 | // 重载Slice类实例的相等判断操作。 97 | inline bool operator==(const Slice& x, const Slice& y) { 98 | return ((x.size() == y.size()) && 99 | (memcmp(x.data(), y.data(), x.size()) == 0)); 100 | } 101 | 102 | // 重载Slice类实例的不想等判断逻辑，其利用了已经实现的相等判断操作。 103 | inline bool operator!=(const Slice& x, const Slice& y) { 104 | return !(x == y); 105 | } 106 | 107 | // Slice::compare()函数用于判断Slice实例b和this之间的大小关系 108 | // 其中如果this < b，则返回-1；this == b，返回 0，this > b，返回1； 109 | inline int Slice::compare(const Slice& b) const { 110 | const size_t min_len = (size_ < b.size_) ? size_ : b.size_; 111 | // 先利用memcmp()库函数进行初步判断，如果这一步可以判断出大于和小于关系，则可以直接返回 112 | // 两者之间的逻辑关系；但是如果这一步判断出两者相等，则需要进一步判断两者的大小关系， 113 | // 这个时候长度大的就大。 114 | int r = memcmp(data_, b.data_, min_len); 115 | if (r == 0) { 116 | if (size_ < b.size_) r = -1; 117 | else if (size_ > b.size_) r = +1; 118 | } 119 | return r; 120 | } 121 | 122 | } // namespace leveldb 123 | 124 | 125 | #endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ 126 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/status.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Status encapsulates the result of an operation. It may indicate success, 6 | // or it may indicate an error with an associated error message. 7 | // 8 | // Multiple threads can invoke const methods on a Status without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same Status must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_STATUS_H_ 15 | 16 | #include 17 | #include "leveldb/export.h" 18 | #include "leveldb/slice.h" 19 | 20 | namespace leveldb { 21 | 22 | class LEVELDB_EXPORT Status { 23 | public: 24 | // Create a success status. 25 | Status() : state_(NULL) { } 26 | ~Status() { delete[] state_; } 27 | 28 | // Copy the specified status. 29 | Status(const Status& s); 30 | void operator=(const Status& s); 31 | 32 | // Return a success status. 33 | static Status OK() { return Status(); } 34 | 35 | // 下面这一族函数用于编码对应状态码的状态信息。 36 | // Return error status of an appropriate type. 37 | static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { 38 | return Status(kNotFound, msg, msg2); 39 | } 40 | static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { 41 | return Status(kCorruption, msg, msg2); 42 | } 43 | static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { 44 | return Status(kNotSupported, msg, msg2); 45 | } 46 | static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { 47 | return Status(kInvalidArgument, msg, msg2); 48 | } 49 | static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { 50 | return Status(kIOError, msg, msg2); 51 | } 52 | 53 | // Returns true iff the status indicates success. 54 | bool ok() const { return (state_ == NULL); } 55 | 56 | // Returns true iff the status indicates a NotFound error. 57 | bool IsNotFound() const { return code() == kNotFound; } 58 | 59 | // Returns true iff the status indicates a Corruption error. 60 | bool IsCorruption() const { return code() == kCorruption; } 61 | 62 | // Returns true iff the status indicates an IOError. 63 | bool IsIOError() const { return code() == kIOError; } 64 | 65 | // Returns true iff the status indicates a NotSupportedError. 66 | bool IsNotSupportedError() const { return code() == kNotSupported; } 67 | 68 | // Returns true iff the status indicates an InvalidArgument. 69 | bool IsInvalidArgument() const { return code() == kInvalidArgument; } 70 | 71 | // Return a string representation of this status suitable for printing. 72 | // Returns the string "OK" for success. 73 | std::string ToString() const; 74 | 75 | private: 76 | // OK status has a NULL state_. Otherwise, state_ is a new[] array 77 | // of the following form: 78 | // state_[0..3] == length of message 79 | // state_[4] == code 80 | // state_[5..] == message 81 | const char* state_; 82 | 83 | enum Code { 84 | kOk = 0, 85 | kNotFound = 1, 86 | kCorruption = 2, 87 | kNotSupported = 3, 88 | kInvalidArgument = 4, 89 | kIOError = 5 90 | }; 91 | 92 | // 从state_字符数组中获取到状态码 93 | Code code() const { 94 | return (state_ == NULL) ? kOk : static_cast(state_[4]); 95 | } 96 | 97 | Status(Code code, const Slice& msg, const Slice& msg2); 98 | static const char* CopyState(const char* s); 99 | }; 100 | 101 | // 从Status类实例s中获取其状态信息，并设置到自己的state_成员中 102 | inline Status::Status(const Status& s) { 103 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 104 | } 105 | 106 | // 实现了Status类的赋值操作。 107 | inline void Status::operator=(const Status& s) { 108 | // The following condition catches both aliasing (when this == &s), 109 | // and the common case where both s and *this are ok. 110 | if (state_ != s.state_) { 111 | delete[] state_; 112 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 113 | } 114 | } 115 | 116 | } // namespace leveldb 117 | 118 | #endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_ 119 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/table.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_TABLE_H_ 7 | 8 | #include 9 | #include "leveldb/export.h" 10 | #include "leveldb/iterator.h" 11 | 12 | namespace leveldb { 13 | 14 | class Block; 15 | class BlockHandle; 16 | class Footer; 17 | struct Options; 18 | class RandomAccessFile; 19 | struct ReadOptions; 20 | class TableCache; 21 | 22 | // A Table is a sorted map from strings to strings. Tables are 23 | // immutable and persistent. A Table may be safely accessed from 24 | // multiple threads without external synchronization. 25 | class LEVELDB_EXPORT Table { 26 | public: 27 | // Attempt to open the table that is stored in bytes [0..file_size) 28 | // of "file", and read the metadata entries necessary to allow 29 | // retrieving data from the table. 30 | // 31 | // If successful, returns ok and sets "*table" to the newly opened 32 | // table. The client should delete "*table" when no longer needed. 33 | // If there was an error while initializing the table, sets "*table" 34 | // to NULL and returns a non-ok status. Does not take ownership of 35 | // "*source", but the client must ensure that "source" remains live 36 | // for the duration of the returned table's lifetime. 37 | // 38 | // *file must remain live while this Table is in use. 39 | // Open()函数用于从sstable 文件中读取大小为size的内容，并用一个Table实例来管理 40 | // 这些信息，说白了就是将sstable文件中的内容从磁盘读入到内存中。 41 | static Status Open(const Options& options, 42 | RandomAccessFile* file, 43 | uint64_t file_size, 44 | Table** table); 45 | 46 | ~Table(); 47 | 48 | // Returns a new iterator over the table contents. 49 | // The result of NewIterator() is initially invalid (caller must 50 | // call one of the Seek methods on the iterator before using it). 51 | Iterator* NewIterator(const ReadOptions&) const; 52 | 53 | // Given a key, return an approximate byte offset in the file where 54 | // the data for that key begins (or would begin if the key were 55 | // present in the file). The returned value is in terms of file 56 | // bytes, and so includes effects like compression of the underlying data. 57 | // E.g., the approximate offset of the last key in the table will 58 | // be close to the file length. 59 | uint64_t ApproximateOffsetOf(const Slice& key) const; 60 | 61 | private: 62 | struct Rep; 63 | Rep* rep_; 64 | 65 | explicit Table(Rep* rep) { rep_ = rep; } 66 | static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); 67 | 68 | // Calls (*handle_result)(arg, ...) with the entry found after a call 69 | // to Seek(key). May not make such a call if filter policy says 70 | // that key is not present. 71 | friend class TableCache; 72 | Status InternalGet( 73 | const ReadOptions&, const Slice& key, 74 | void* arg, 75 | void (*handle_result)(void* arg, const Slice& k, const Slice& v)); 76 | 77 | 78 | void ReadMeta(const Footer& footer); 79 | void ReadFilter(const Slice& filter_handle_value); 80 | 81 | // No copying allowed 82 | Table(const Table&); 83 | void operator=(const Table&); 84 | }; 85 | 86 | } // namespace leveldb 87 | 88 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_ 89 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/table_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // TableBuilder provides the interface used to build a Table 6 | // (an immutable and sorted map from keys to values). 7 | // 8 | // Multiple threads can invoke const methods on a TableBuilder without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same TableBuilder must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 15 | 16 | #include 17 | #include "leveldb/export.h" 18 | #include "leveldb/options.h" 19 | #include "leveldb/status.h" 20 | 21 | namespace leveldb { 22 | 23 | class BlockBuilder; 24 | class BlockHandle; 25 | class WritableFile; 26 | 27 | // 在leveldb中，一个Table类实例就对应了一个sstable文件内容，而TableBuilder类就是 28 | // 用来构建一个sstable文件的。 29 | class LEVELDB_EXPORT TableBuilder { 30 | public: 31 | // Create a builder that will store the contents of the table it is 32 | // building in *file. Does not close the file. It is up to the 33 | // caller to close the file after calling Finish(). 34 | // TableBuilder类的构造函数，其中的file对象封装了sstable文件。 35 | TableBuilder(const Options& options, WritableFile* file); 36 | 37 | // REQUIRES: Either Finish() or Abandon() has been called. 38 | ~TableBuilder(); 39 | 40 | // Change the options used by this builder. Note: only some of the 41 | // option fields can be changed after construction. If a field is 42 | // not allowed to change dynamically and its value in the structure 43 | // passed to the constructor is different from its value in the 44 | // structure passed to this method, this method will return an error 45 | // without changing any fields. 46 | Status ChangeOptions(const Options& options); 47 | 48 | // Add key,value to the table being constructed. 49 | // REQUIRES: key is after any previously added key according to comparator. 50 | // REQUIRES: Finish(), Abandon() have not been called 51 | // Add()方法用于将参数中的key-value信息加入到当前的table中，其实也就是sstable 52 | // 文件中，当然这个key-value最终会存放到sstable中的data block中。 53 | void Add(const Slice& key, const Slice& value); 54 | 55 | // Advanced operation: flush any buffered key/value pairs to file. 56 | // Can be used to ensure that two adjacent entries never live in 57 | // the same data block. Most clients should not need to use this method. 58 | // REQUIRES: Finish(), Abandon() have not been called 59 | // 将当前正在构建的data block写入到sstable文件中，表示一个data block构建完毕。 60 | void Flush(); 61 | 62 | // Return non-ok iff some error has been detected. 63 | Status status() const; 64 | 65 | // Finish building the table. Stops using the file passed to the 66 | // constructor after this function returns. 67 | // REQUIRES: Finish(), Abandon() have not been called 68 | // Finish()方法用来结束当前sstable的构建，用于往data block(如果有meta block的话， 69 | // 则是meta block)后面写入metaindex block和index block，接着写入footer来完成 70 | // 整个sstable的构建。 71 | Status Finish(); 72 | 73 | // Indicate that the contents of this builder should be abandoned. Stops 74 | // using the file passed to the constructor after this function returns. 75 | // If the caller is not going to call Finish(), it must call Abandon() 76 | // before destroying this builder. 77 | // REQUIRES: Finish(), Abandon() have not been called 78 | void Abandon(); 79 | 80 | // Number of calls to Add() so far. 81 | // 当前已经加入到sstable中的key-value记录个数。 82 | uint64_t NumEntries() const; 83 | 84 | // Size of the file generated so far. If invoked after a successful 85 | // Finish() call, returns the size of the final generated file. 86 | // 返回当前文件的大小。 87 | uint64_t FileSize() const; 88 | 89 | private: 90 | bool ok() const { return status().ok(); } 91 | 92 | // WriteBlock()方法用于往 93 | void WriteBlock(BlockBuilder* block, BlockHandle* handle); 94 | void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle); 95 | 96 | struct Rep; 97 | Rep* rep_; 98 | 99 | // No copying allowed 100 | TableBuilder(const TableBuilder&); 101 | void operator=(const TableBuilder&); 102 | }; 103 | 104 | } // namespace leveldb 105 | 106 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 107 | -------------------------------------------------------------------------------- /leveldb-master/include/leveldb/write_batch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch holds a collection of updates to apply atomically to a DB. 6 | // 7 | // The updates are applied in the order in which they are added 8 | // to the WriteBatch. For example, the value of "key" will be "v3" 9 | // after the following batch is written: 10 | // 11 | // batch.Put("key", "v1"); 12 | // batch.Delete("key"); 13 | // batch.Put("key", "v2"); 14 | // batch.Put("key", "v3"); 15 | // 16 | // Multiple threads can invoke const methods on a WriteBatch without 17 | // external synchronization, but if any of the threads may call a 18 | // non-const method, all threads accessing the same WriteBatch must use 19 | // external synchronization. 20 | 21 | #ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 22 | #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 23 | 24 | #include 25 | #include "leveldb/export.h" 26 | #include "leveldb/status.h" 27 | 28 | namespace leveldb { 29 | 30 | class Slice; 31 | 32 | class LEVELDB_EXPORT WriteBatch { 33 | public: 34 | WriteBatch(); 35 | ~WriteBatch(); 36 | 37 | // Store the mapping "key->value" in the database. 38 | void Put(const Slice& key, const Slice& value); 39 | 40 | // If the database contains a mapping for "key", erase it. Else do nothing. 41 | void Delete(const Slice& key); 42 | 43 | // Clear all updates buffered in this batch. 44 | void Clear(); 45 | 46 | // The size of the database changes caused by this batch. 47 | // 48 | // This number is tied to implementation details, and may change across 49 | // releases. It is intended for LevelDB usage metrics. 50 | size_t ApproximateSize(); 51 | 52 | // Support for iterating over the contents of a batch. 53 | class Handler { 54 | public: 55 | virtual ~Handler(); 56 | virtual void Put(const Slice& key, const Slice& value) = 0; 57 | virtual void Delete(const Slice& key) = 0; 58 | }; 59 | Status Iterate(Handler* handler) const; 60 | 61 | private: 62 | friend class WriteBatchInternal; 63 | 64 | std::string rep_; // See comment in write_batch.cc for the format of rep_ 65 | 66 | // Intentionally copyable 67 | }; 68 | 69 | } // namespace leveldb 70 | 71 | #endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 72 | -------------------------------------------------------------------------------- /leveldb-master/issues/issue178_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 178: a manual compaction causes deleted data to reappear. 6 | #include 7 | #include 8 | #include 9 | 10 | #include "leveldb/db.h" 11 | #include "leveldb/write_batch.h" 12 | #include "util/testharness.h" 13 | 14 | namespace { 15 | 16 | const int kNumKeys = 1100000; 17 | 18 | std::string Key1(int i) { 19 | char buf[100]; 20 | snprintf(buf, sizeof(buf), "my_key_%d", i); 21 | return buf; 22 | } 23 | 24 | std::string Key2(int i) { 25 | return Key1(i) + "_xxx"; 26 | } 27 | 28 | class Issue178 { }; 29 | 30 | TEST(Issue178, Test) { 31 | // Get rid of any state from an old run. 32 | std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; 33 | DestroyDB(dbpath, leveldb::Options()); 34 | 35 | // Open database. Disable compression since it affects the creation 36 | // of layers and the code below is trying to test against a very 37 | // specific scenario. 38 | leveldb::DB* db; 39 | leveldb::Options db_options; 40 | db_options.create_if_missing = true; 41 | db_options.compression = leveldb::kNoCompression; 42 | ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); 43 | 44 | // create first key range 45 | leveldb::WriteBatch batch; 46 | for (size_t i = 0; i < kNumKeys; i++) { 47 | batch.Put(Key1(i), "value for range 1 key"); 48 | } 49 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 50 | 51 | // create second key range 52 | batch.Clear(); 53 | for (size_t i = 0; i < kNumKeys; i++) { 54 | batch.Put(Key2(i), "value for range 2 key"); 55 | } 56 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 57 | 58 | // delete second key range 59 | batch.Clear(); 60 | for (size_t i = 0; i < kNumKeys; i++) { 61 | batch.Delete(Key2(i)); 62 | } 63 | ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); 64 | 65 | // compact database 66 | std::string start_key = Key1(0); 67 | std::string end_key = Key1(kNumKeys - 1); 68 | leveldb::Slice least(start_key.data(), start_key.size()); 69 | leveldb::Slice greatest(end_key.data(), end_key.size()); 70 | 71 | // commenting out the line below causes the example to work correctly 72 | db->CompactRange(&least, &greatest); 73 | 74 | // count the keys 75 | leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); 76 | size_t num_keys = 0; 77 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 78 | num_keys++; 79 | } 80 | delete iter; 81 | ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; 82 | 83 | // close database 84 | delete db; 85 | DestroyDB(dbpath, leveldb::Options()); 86 | } 87 | 88 | } // anonymous namespace 89 | 90 | int main(int argc, char** argv) { 91 | return leveldb::test::RunAllTests(); 92 | } 93 | -------------------------------------------------------------------------------- /leveldb-master/issues/issue200_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // Test for issue 200: when iterator switches direction from backward 6 | // to forward, the current key can be yielded unexpectedly if a new 7 | // mutation has been added just before the current key. 8 | 9 | #include "leveldb/db.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class Issue200 { }; 15 | 16 | TEST(Issue200, Test) { 17 | // Get rid of any state from an old run. 18 | std::string dbpath = test::TmpDir() + "/leveldb_issue200_test"; 19 | DestroyDB(dbpath, Options()); 20 | 21 | DB *db; 22 | Options options; 23 | options.create_if_missing = true; 24 | ASSERT_OK(DB::Open(options, dbpath, &db)); 25 | 26 | WriteOptions write_options; 27 | ASSERT_OK(db->Put(write_options, "1", "b")); 28 | ASSERT_OK(db->Put(write_options, "2", "c")); 29 | ASSERT_OK(db->Put(write_options, "3", "d")); 30 | ASSERT_OK(db->Put(write_options, "4", "e")); 31 | ASSERT_OK(db->Put(write_options, "5", "f")); 32 | 33 | ReadOptions read_options; 34 | Iterator *iter = db->NewIterator(read_options); 35 | 36 | // Add an element that should not be reflected in the iterator. 37 | ASSERT_OK(db->Put(write_options, "25", "cd")); 38 | 39 | iter->Seek("5"); 40 | ASSERT_EQ(iter->key().ToString(), "5"); 41 | iter->Prev(); 42 | ASSERT_EQ(iter->key().ToString(), "4"); 43 | iter->Prev(); 44 | ASSERT_EQ(iter->key().ToString(), "3"); 45 | iter->Next(); 46 | ASSERT_EQ(iter->key().ToString(), "4"); 47 | iter->Next(); 48 | ASSERT_EQ(iter->key().ToString(), "5"); 49 | 50 | delete iter; 51 | delete db; 52 | DestroyDB(dbpath, options); 53 | } 54 | 55 | } // namespace leveldb 56 | 57 | int main(int argc, char** argv) { 58 | return leveldb::test::RunAllTests(); 59 | } 60 | -------------------------------------------------------------------------------- /leveldb-master/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /leveldb-master/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_posix.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /leveldb-master/port/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_posix.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace port { 13 | 14 | static void PthreadCall(const char* label, int result) { 15 | if (result != 0) { 16 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 17 | abort(); 18 | } 19 | } 20 | 21 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 22 | 23 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 24 | 25 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 26 | 27 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 28 | 29 | CondVar::CondVar(Mutex* mu) 30 | : mu_(mu) { 31 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 32 | } 33 | 34 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 35 | 36 | void CondVar::Wait() { 37 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 38 | } 39 | 40 | void CondVar::Signal() { 41 | PthreadCall("signal", pthread_cond_signal(&cv_)); 42 | } 43 | 44 | void CondVar::SignalAll() { 45 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 46 | } 47 | 48 | void InitOnce(OnceType* once, void (*initializer)()) { 49 | PthreadCall("once", pthread_once(once, initializer)); 50 | } 51 | 52 | } // namespace port 53 | } // namespace leveldb 54 | -------------------------------------------------------------------------------- /leveldb-master/port/port_posix.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // See port_example.h for documentation for the following types/functions. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 8 | #define STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 9 | 10 | #undef PLATFORM_IS_LITTLE_ENDIAN 11 | #if defined(__APPLE__) 12 | #include 13 | #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) 14 | #define PLATFORM_IS_LITTLE_ENDIAN \ 15 | (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) 16 | #endif 17 | #elif defined(OS_SOLARIS) 18 | #include 19 | #ifdef _LITTLE_ENDIAN 20 | #define PLATFORM_IS_LITTLE_ENDIAN true 21 | #else 22 | #define PLATFORM_IS_LITTLE_ENDIAN false 23 | #endif 24 | #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\ 25 | defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD) 26 | #include 27 | #include 28 | #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) 29 | #elif defined(OS_HPUX) 30 | #define PLATFORM_IS_LITTLE_ENDIAN false 31 | #elif defined(OS_ANDROID) 32 | // Due to a bug in the NDK x86 definition, 33 | // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. 34 | // See http://code.google.com/p/android/issues/detail?id=39824 35 | #include 36 | #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) 37 | #else 38 | #include 39 | #endif 40 | 41 | #include 42 | #if defined(HAVE_CRC32C) 43 | #include 44 | #endif // defined(HAVE_CRC32C) 45 | #ifdef HAVE_SNAPPY 46 | #include 47 | #endif // defined(HAVE_SNAPPY) 48 | #include 49 | #include 50 | #include "port/atomic_pointer.h" 51 | 52 | #ifndef PLATFORM_IS_LITTLE_ENDIAN 53 | #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) 54 | #endif 55 | 56 | #if defined(__APPLE__) || defined(OS_FREEBSD) ||\ 57 | defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) 58 | // Use fsync() on platforms without fdatasync() 59 | #define fdatasync fsync 60 | #endif 61 | 62 | #if defined(OS_ANDROID) && __ANDROID_API__ < 9 63 | // fdatasync() was only introduced in API level 9 on Android. Use fsync() 64 | // when targetting older platforms. 65 | #define fdatasync fsync 66 | #endif 67 | 68 | namespace leveldb { 69 | namespace port { 70 | 71 | static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; 72 | #undef PLATFORM_IS_LITTLE_ENDIAN 73 | 74 | class CondVar; 75 | 76 | class Mutex { 77 | public: 78 | Mutex(); 79 | ~Mutex(); 80 | 81 | void Lock(); 82 | void Unlock(); 83 | void AssertHeld() { } 84 | 85 | private: 86 | friend class CondVar; 87 | pthread_mutex_t mu_; 88 | 89 | // No copying 90 | Mutex(const Mutex&); 91 | void operator=(const Mutex&); 92 | }; 93 | 94 | class CondVar { 95 | public: 96 | explicit CondVar(Mutex* mu); 97 | ~CondVar(); 98 | void Wait(); 99 | void Signal(); 100 | void SignalAll(); 101 | private: 102 | pthread_cond_t cv_; 103 | Mutex* mu_; 104 | }; 105 | 106 | typedef pthread_once_t OnceType; 107 | #define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT 108 | extern void InitOnce(OnceType* once, void (*initializer)()); 109 | 110 | inline bool Snappy_Compress(const char* input, size_t length, 111 | ::std::string* output) { 112 | #ifdef HAVE_SNAPPY 113 | output->resize(snappy::MaxCompressedLength(length)); 114 | size_t outlen; 115 | snappy::RawCompress(input, length, &(*output)[0], &outlen); 116 | output->resize(outlen); 117 | return true; 118 | #endif // defined(HAVE_SNAPPY) 119 | 120 | return false; 121 | } 122 | 123 | inline bool Snappy_GetUncompressedLength(const char* input, size_t length, 124 | size_t* result) { 125 | #ifdef HAVE_SNAPPY 126 | return snappy::GetUncompressedLength(input, length, result); 127 | #else 128 | return false; 129 | #endif // defined(HAVE_SNAPPY) 130 | } 131 | 132 | inline bool Snappy_Uncompress(const char* input, size_t length, 133 | char* output) { 134 | #ifdef HAVE_SNAPPY 135 | return snappy::RawUncompress(input, length, output); 136 | #else 137 | return false; 138 | #endif // defined(HAVE_SNAPPY) 139 | } 140 | 141 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 142 | return false; 143 | } 144 | 145 | inline uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { 146 | #if defined(HAVE_CRC32C) 147 | return ::crc32c::Extend(crc, reinterpret_cast(buf), size); 148 | #else 149 | return 0; 150 | #endif // defined(HAVE_CRC32C) 151 | } 152 | 153 | } // namespace port 154 | } // namespace leveldb 155 | 156 | #endif // STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 157 | -------------------------------------------------------------------------------- /leveldb-master/port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 6 | #define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 7 | 8 | // Some environments provide custom macros to aid in static thread-safety 9 | // analysis. Provide empty definitions of such macros unless they are already 10 | // defined. 11 | 12 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 13 | #define EXCLUSIVE_LOCKS_REQUIRED(...) 14 | #endif 15 | 16 | #ifndef SHARED_LOCKS_REQUIRED 17 | #define SHARED_LOCKS_REQUIRED(...) 18 | #endif 19 | 20 | #ifndef LOCKS_EXCLUDED 21 | #define LOCKS_EXCLUDED(...) 22 | #endif 23 | 24 | #ifndef LOCK_RETURNED 25 | #define LOCK_RETURNED(x) 26 | #endif 27 | 28 | #ifndef LOCKABLE 29 | #define LOCKABLE 30 | #endif 31 | 32 | #ifndef SCOPED_LOCKABLE 33 | #define SCOPED_LOCKABLE 34 | #endif 35 | 36 | #ifndef EXCLUSIVE_LOCK_FUNCTION 37 | #define EXCLUSIVE_LOCK_FUNCTION(...) 38 | #endif 39 | 40 | #ifndef SHARED_LOCK_FUNCTION 41 | #define SHARED_LOCK_FUNCTION(...) 42 | #endif 43 | 44 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 45 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) 46 | #endif 47 | 48 | #ifndef SHARED_TRYLOCK_FUNCTION 49 | #define SHARED_TRYLOCK_FUNCTION(...) 50 | #endif 51 | 52 | #ifndef UNLOCK_FUNCTION 53 | #define UNLOCK_FUNCTION(...) 54 | #endif 55 | 56 | #ifndef NO_THREAD_SAFETY_ANALYSIS 57 | #define NO_THREAD_SAFETY_ANALYSIS 58 | #endif 59 | 60 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 61 | -------------------------------------------------------------------------------- /leveldb-master/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /leveldb-master/table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 7 | 8 | #include 9 | #include 10 | #include "leveldb/iterator.h" 11 | 12 | namespace leveldb { 13 | 14 | struct BlockContents; 15 | class Comparator; 16 | 17 | // 一个Block的内存布局： 18 | // ------------------ 19 | // | Record 1 | 20 | // ------------------ 21 | // | Record 2 | 22 | // ------------------ 23 | // | .... | 24 | // ------------------ 25 | // | Record n | 26 | // ------------------ 27 | // | Restart 1 | 28 | // ------------------ 29 | // | Restart 2 | 30 | // ------------------ 31 | // | .... | 32 | // ------------------ 33 | // | Restart m | 34 | // ------------------ 35 | // | num_restarts | 36 | // ------------------ 37 | // 从上面的内存布局可以看到，一个Block内部可以分成两个部分，前面的Record数组 38 | // 存放的是一些K-V记录，其顺序是根据Key值由小到大排列的。后面则是一个称为 39 | // "重启点"的数组，存储是距离block开始处的偏移值，存放了Record数组中一些记录位置。 40 | // "重启点"是干什么的呢？我们知道Block内部的k-v记录是从小到大排列的，这样的话， 41 | // 相邻的两条或者多条记录的key值很有可能存在重叠部分，比如上图中的Record 1的 42 | // key值为"my friend"，Record 2的key值为"my girlfriend"，那么两者就存在重叠部分 43 | // "my "，为了减少key的存储量，Record 2中的key可能只存储和Record 1的key的不同部分， 44 | // 即"girlfriend"，两者的相同部分则可以从Record 1中获取。基于上述的设计，"重启点"的 45 | // 意思是：从这条记录开始，不再采取只存储不同的key部分，而是记录这条记录完整的 46 | // key值，不管该记录的key值是否前一条记录有相同部分，例如Record 3就是一个重启点， 47 | // 且他的key值为"your friend" 或者"my monther"，都将记录完整的key值，而不只是 48 | // 和前一个记录的不同部分。而num_restarts则是"重启点"的个数。 49 | 50 | // 前面我们说过，一个Record记录包含了k-v值，并且其中存储的key可能不是完整的key值， 51 | // 而只是key的一部分值，那么为了适应这种设计，Record记录本身的内存布局是怎么样的呢？ 52 | // ------------------------------------------------------------- 53 | // |key共享长度|key非共享长度|value长度|key非共享内容|value内容| 54 | // ------------------------------------------------------------- 55 | // 以上面的Record 1和Record 2为例，对于Record 2来说，key共享长度为"my "的长度，为3： 56 | // key非共享长度为"girlfriend"的长度，为10，value长度的就是k-v的v的长度，key非共享内容 57 | // 就是"girlfriend"，value内容就是k-v的v的内容。 58 | class Block { 59 | public: 60 | // Initialize the block with the specified contents. 61 | explicit Block(const BlockContents& contents); 62 | 63 | ~Block(); 64 | 65 | size_t size() const { return size_; } 66 | Iterator* NewIterator(const Comparator* comparator); 67 | 68 | private: 69 | 70 | // NumRestarts()方法从data_数组中根据block的内存布局获取到重启点的个数。 71 | uint32_t NumRestarts() const; 72 | 73 | // data_就是block存放具体内容的内存首地址。 74 | const char* data_; 75 | 76 | // size_就是block的大小，也就是data_所指向的内存的大小。 77 | size_t size_; 78 | 79 | // restart_offset_就是“重启点”数组首地址在整个Block内的偏移。 80 | uint32_t restart_offset_; // Offset in data_ of restart array 81 | 82 | // 用来表示data_数组指向的内存是不是动态申请的，如果说是，则为true。 83 | bool owned_; // Block owns data_[] 84 | 85 | // No copying allowed 86 | Block(const Block&); 87 | void operator=(const Block&); 88 | 89 | // Iter是Block的内部迭代器。 90 | class Iter; 91 | }; 92 | 93 | } // namespace leveldb 94 | 95 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 96 | -------------------------------------------------------------------------------- /leveldb-master/table/block_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 7 | 8 | #include 9 | 10 | #include 11 | #include "leveldb/slice.h" 12 | 13 | namespace leveldb { 14 | 15 | struct Options; 16 | 17 | // BlockBuilder类是用来创建一个Block对象的，其创建的过程其实也就是 18 | // 形成Block内存布局的过程。Block的内存布局如下： 19 | // 一个Block的内存布局： 20 | // ------------------ 21 | // | Record 1 | 22 | // ------------------ 23 | // | Record 2 | 24 | // ------------------ 25 | // | .... | 26 | // ------------------ 27 | // | Record n | 28 | // ------------------ 29 | // | Restart 1 | 30 | // ------------------ 31 | // | Restart 2 | 32 | // ------------------ 33 | // | .... | 34 | // ------------------ 35 | // | Restart m | 36 | // ------------------ 37 | // | num_restarts | 38 | // ------------------ 39 | // 从上面的内存布局可以看到，一个Block内部可以分成两个部分，前面的Record数组 40 | // 存放的是一些K-V记录，其顺序是根据Key值由小到大排列的。后面则是一个称为 41 | // "重启点"的数组，存储是距离block开始处的偏移值，存放了Record数组中一些记录位置。 42 | // "重启点"是干什么的呢？我们知道Block内部的k-v记录是从小到大排列的，这样的话， 43 | // 相邻的两条或者多条记录的key值很有可能存在重叠部分，比如上图中的Record 1的 44 | // key值为"my friend"，Record 2的key值为"my girlfriend"，那么两者就存在重叠部分 45 | // "my "，为了减少key的存储量，Record 2中的key可能只存储和Record 1的key的不同部分， 46 | // 即"girlfriend"，两者的相同部分则可以从Record 1中获取。基于上述的设计，"重启点"的 47 | // 意思是：从这条记录开始，不再采取只存储不同的key部分，而是记录这条记录完整的 48 | // key值，不管该记录的key值是否前一条记录有相同部分，例如Record 3就是一个重启点， 49 | // 且他的key值为"your friend" 或者"my monther"，都将记录完整的key值，而不只是 50 | // 和前一个记录的不同部分。而num_restarts则是"重启点"的个数。 51 | 52 | // 前面我们说过，一个Record记录包含了k-v值，并且其中存储的key可能不是完整的key值， 53 | // 而只是key的一部分值，那么为了适应这种设计，Record记录本身的内存布局是怎么样的呢？ 54 | // ------------------------------------------------------------- 55 | // |key共享长度|key非共享长度|value长度|key非共享内容|value内容| 56 | // ------------------------------------------------------------- 57 | // 以上面的Record 1和Record 2为例，对于Record 2来说，key共享长度为"my "的长度，为3： 58 | // key非共享长度为"girlfriend"的长度，为10，value长度的就是k-v的v的长度，key非共享内容 59 | // 就是"girlfriend"，value内容就是k-v的v的内容。 60 | 61 | class BlockBuilder { 62 | public: 63 | explicit BlockBuilder(const Options* options); 64 | 65 | // Reset the contents as if the BlockBuilder was just constructed. 66 | void Reset(); 67 | 68 | // REQUIRES: Finish() has not been called since the last call to Reset(). 69 | // REQUIRES: key is larger than any previously added key 70 | // Add()方法是用来往当前正在构建的Block中添加一个key-value记录。 71 | void Add(const Slice& key, const Slice& value); 72 | 73 | // Finish building the block and return a slice that refers to the 74 | // block contents. The returned slice will remain valid for the 75 | // lifetime of this builder or until Reset() is called. 76 | // Finish()方法用来往Block中key-value记录后面追加重启点数组以及重启点数组长度信息的。 77 | // 当一个正在构建的Block的内存占用达到阈值时就不会再往这个Block中继续添加key-value记录了， 78 | // 这个时候就会在Block的末尾追加重启点数组以及重启点数组长度信息来结束这个Block的构建。 79 | // 这个block的内存布局是一致的。 80 | Slice Finish(); 81 | 82 | // Returns an estimate of the current (uncompressed) size of the block 83 | // we are building. 84 | // CurrentSizeEstimate()方法用来计算当前正在构建的Block的内存占用情况。 85 | // 也正是根据这个方法的返回值判断是否要结束这个Block的构建。 86 | size_t CurrentSizeEstimate() const; 87 | 88 | // Return true iff no entries have been added since the last Reset() 89 | bool empty() const { 90 | return buffer_.empty(); 91 | } 92 | 93 | private: 94 | const Options* options_; 95 | 96 | // buffer_是用来存放block具体内容的缓冲区。 97 | std::string buffer_; // Destination buffer 98 | 99 | // restarts_动态数组中存放的就是每个重启点相对于block开始处的偏移，也就是 100 | // 重启点数组中元素的值。 101 | std::vector restarts_; // Restart points 102 | 103 | // counter_存放的是一个重启点对应的记录后面跟的key-value记录个数， 104 | // 如果这个个数达到了options_->block_restart_interval，那么就要重新 105 | // 创建一个重启点了，即要开始存放完整的key信息，而不仅仅只存放和前面key的 106 | // 不同部分了。 107 | int counter_; // Number of entries emitted since restart 108 | 109 | // finished_是Finish()方法被调用过的标志。 110 | bool finished_; // Has Finish() been called? 111 | 112 | // 上一次通过调用Add()方法加入到当前block中的key值。 113 | std::string last_key_; 114 | 115 | // No copying allowed 116 | BlockBuilder(const BlockBuilder&); 117 | void operator=(const BlockBuilder&); 118 | }; 119 | 120 | } // namespace leveldb 121 | 122 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 123 | -------------------------------------------------------------------------------- /leveldb-master/table/filter_block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A filter block is stored near the end of a Table file. It contains 6 | // filters (e.g., bloom filters) for all data blocks in the table combined 7 | // into a single filter block. 8 | 9 | #ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 10 | #define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "leveldb/slice.h" 17 | #include "util/hash.h" 18 | 19 | namespace leveldb { 20 | 21 | class FilterPolicy; 22 | 23 | // A FilterBlockBuilder is used to construct all of the filters for a 24 | // particular Table. It generates a single string which is stored as 25 | // a special block in the Table. 26 | // 27 | // The sequence of calls to FilterBlockBuilder must match the regexp: 28 | // (StartBlock AddKey*)* Finish 29 | class FilterBlockBuilder { 30 | public: 31 | explicit FilterBlockBuilder(const FilterPolicy*); 32 | 33 | void StartBlock(uint64_t block_offset); 34 | void AddKey(const Slice& key); 35 | Slice Finish(); 36 | 37 | private: 38 | void GenerateFilter(); 39 | 40 | // 过滤策略 41 | const FilterPolicy* policy_; 42 | 43 | // keys_动态字符串中存放了加入到filter中的key值。keys_中的key值个数达到一个阈值之后 44 | // 就会将这一个key集合用来生成一个过滤器，生成过滤器之后会将keys_内容清除，为创建下一个 45 | // key集合对应的过滤器做准备。 46 | std::string keys_; // Flattened key contents 47 | 48 | // start_动态数组中存放的是每个key在keys_动态字符串中的索引起始值。当keys_中的key集合 49 | // 对应的过滤器创建完毕之后就会将start_清空，为创建下一个key集合对应的过滤器做准备。 50 | std::vector start_; // Starting index in keys_ of each key 51 | 52 | // 存放每一个key集合生成的过滤器。多个key集合对应的过滤器会以追加的方式 53 | // 存放到result_中 54 | std::string result_; // Filter data computed so far 55 | 56 | // 在创建Filter的时候，用来存放从keys_中取出来的所有key值。如何结合start_ 57 | // 从keys_中取出每一个key值可以参考GenerateFilter()的实现。当tmp_keys中 58 | // 的key集合对应的过滤器创建完毕之后就会将tmp_keys清空，为创建下一个key集合对应的 59 | // 过滤器做好准备。 60 | std::vector tmp_keys_; // policy_->CreateFilter() argument 61 | 62 | // filter_offsets_动态数组存放的是每一个过滤器在result_动态字符串中的索引起始值。 63 | std::vector filter_offsets_; 64 | 65 | // No copying allowed 66 | FilterBlockBuilder(const FilterBlockBuilder&); 67 | void operator=(const FilterBlockBuilder&); 68 | }; 69 | 70 | class FilterBlockReader { 71 | public: 72 | // REQUIRES: "contents" and *policy must stay live while *this is live. 73 | FilterBlockReader(const FilterPolicy* policy, const Slice& contents); 74 | bool KeyMayMatch(uint64_t block_offset, const Slice& key); 75 | 76 | private: 77 | const FilterPolicy* policy_; // 78 | // data_指向了存放着多个过滤器的内存起始地址 79 | const char* data_; // Pointer to filter data (at block-start) 80 | 81 | // offset_指向了存放着过滤器在data_中的起始索引值的内存。 82 | const char* offset_; // Pointer to beginning of offset array (at block-end) 83 | 84 | // num_记录的是data_中存放的过滤器个数。 85 | size_t num_; // Number of entries in offset array 86 | 87 | // kFilterBaseLg的值 88 | size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file) 89 | }; 90 | 91 | } 92 | 93 | #endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_ 94 | -------------------------------------------------------------------------------- /leveldb-master/table/filter_block_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/filter_block.h" 6 | 7 | #include "leveldb/filter_policy.h" 8 | #include "util/coding.h" 9 | #include "util/hash.h" 10 | #include "util/logging.h" 11 | #include "util/testharness.h" 12 | #include "util/testutil.h" 13 | 14 | namespace leveldb { 15 | 16 | // For testing: emit an array with one hash value per key 17 | class TestHashFilter : public FilterPolicy { 18 | public: 19 | virtual const char* Name() const { 20 | return "TestHashFilter"; 21 | } 22 | 23 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { 24 | for (int i = 0; i < n; i++) { 25 | uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); 26 | PutFixed32(dst, h); 27 | } 28 | } 29 | 30 | virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { 31 | uint32_t h = Hash(key.data(), key.size(), 1); 32 | for (size_t i = 0; i + 4 <= filter.size(); i += 4) { 33 | if (h == DecodeFixed32(filter.data() + i)) { 34 | return true; 35 | } 36 | } 37 | return false; 38 | } 39 | }; 40 | 41 | class FilterBlockTest { 42 | public: 43 | TestHashFilter policy_; 44 | }; 45 | 46 | TEST(FilterBlockTest, EmptyBuilder) { 47 | FilterBlockBuilder builder(&policy_); 48 | Slice block = builder.Finish(); 49 | ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block)); 50 | FilterBlockReader reader(&policy_, block); 51 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 52 | ASSERT_TRUE(reader.KeyMayMatch(100000, "foo")); 53 | } 54 | 55 | TEST(FilterBlockTest, SingleChunk) { 56 | FilterBlockBuilder builder(&policy_); 57 | builder.StartBlock(100); 58 | builder.AddKey("foo"); 59 | builder.AddKey("bar"); 60 | builder.AddKey("box"); 61 | builder.StartBlock(200); 62 | builder.AddKey("box"); 63 | builder.StartBlock(300); 64 | builder.AddKey("hello"); 65 | Slice block = builder.Finish(); 66 | FilterBlockReader reader(&policy_, block); 67 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 68 | ASSERT_TRUE(reader.KeyMayMatch(100, "bar")); 69 | ASSERT_TRUE(reader.KeyMayMatch(100, "box")); 70 | ASSERT_TRUE(reader.KeyMayMatch(100, "hello")); 71 | ASSERT_TRUE(reader.KeyMayMatch(100, "foo")); 72 | ASSERT_TRUE(! reader.KeyMayMatch(100, "missing")); 73 | ASSERT_TRUE(! reader.KeyMayMatch(100, "other")); 74 | } 75 | 76 | TEST(FilterBlockTest, MultiChunk) { 77 | FilterBlockBuilder builder(&policy_); 78 | 79 | // First filter 80 | builder.StartBlock(0); 81 | builder.AddKey("foo"); 82 | builder.StartBlock(2000); 83 | builder.AddKey("bar"); 84 | 85 | // Second filter 86 | builder.StartBlock(3100); 87 | builder.AddKey("box"); 88 | 89 | // Third filter is empty 90 | 91 | // Last filter 92 | builder.StartBlock(9000); 93 | builder.AddKey("box"); 94 | builder.AddKey("hello"); 95 | 96 | Slice block = builder.Finish(); 97 | FilterBlockReader reader(&policy_, block); 98 | 99 | // Check first filter 100 | ASSERT_TRUE(reader.KeyMayMatch(0, "foo")); 101 | ASSERT_TRUE(reader.KeyMayMatch(2000, "bar")); 102 | ASSERT_TRUE(! reader.KeyMayMatch(0, "box")); 103 | ASSERT_TRUE(! reader.KeyMayMatch(0, "hello")); 104 | 105 | // Check second filter 106 | ASSERT_TRUE(reader.KeyMayMatch(3100, "box")); 107 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo")); 108 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar")); 109 | ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello")); 110 | 111 | // Check third filter (empty) 112 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo")); 113 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar")); 114 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "box")); 115 | ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello")); 116 | 117 | // Check last filter 118 | ASSERT_TRUE(reader.KeyMayMatch(9000, "box")); 119 | ASSERT_TRUE(reader.KeyMayMatch(9000, "hello")); 120 | ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo")); 121 | ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar")); 122 | } 123 | 124 | } // namespace leveldb 125 | 126 | int main(int argc, char** argv) { 127 | return leveldb::test::RunAllTests(); 128 | } 129 | -------------------------------------------------------------------------------- /leveldb-master/table/format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ 6 | #define STORAGE_LEVELDB_TABLE_FORMAT_H_ 7 | 8 | #include 9 | #include 10 | #include "leveldb/slice.h" 11 | #include "leveldb/status.h" 12 | #include "leveldb/table_builder.h" 13 | 14 | namespace leveldb { 15 | 16 | class Block; 17 | class RandomAccessFile; 18 | struct ReadOptions; 19 | 20 | // BlockHandle is a pointer to the extent of a file that stores a data 21 | // block or a meta block. 22 | // BlockHandle类描述了一个block在文件中的偏移和大小。这个大小不包含 23 | // 紧随在block后面的type和crc的5个字节大小。 24 | class BlockHandle { 25 | public: 26 | BlockHandle(); 27 | 28 | // The offset of the block in the file. 29 | uint64_t offset() const { return offset_; } 30 | void set_offset(uint64_t offset) { offset_ = offset; } 31 | 32 | // The size of the stored block 33 | uint64_t size() const { return size_; } 34 | void set_size(uint64_t size) { size_ = size; } 35 | 36 | // EncodeTo()方法将offset_和size_以varint64的编码方式编码后追加到dst中。 37 | void EncodeTo(std::string* dst) const; 38 | 39 | // DecodeFrom()方法从Slice对象input中解码出整形数字，并存放到offset_和size_。 40 | Status DecodeFrom(Slice* input); 41 | 42 | // Maximum encoding length of a BlockHandle 43 | // EncodeTo()函数编码BlockHandle类实例时，需要编码两个64位成员， 44 | // 采用varint64编码方式时最多需要20个字节。 45 | enum { kMaxEncodedLength = 10 + 10 }; 46 | 47 | private: 48 | uint64_t offset_; 49 | uint64_t size_; 50 | }; 51 | 52 | // Footer encapsulates the fixed information stored at the tail 53 | // end of every table file. 54 | class Footer { 55 | public: 56 | Footer() { } 57 | 58 | // The block handle for the metaindex block of the table 59 | const BlockHandle& metaindex_handle() const { return metaindex_handle_; } 60 | void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } 61 | 62 | // The block handle for the index block of the table 63 | const BlockHandle& index_handle() const { 64 | return index_handle_; 65 | } 66 | void set_index_handle(const BlockHandle& h) { 67 | index_handle_ = h; 68 | } 69 | 70 | void EncodeTo(std::string* dst) const; 71 | Status DecodeFrom(Slice* input); 72 | 73 | // Encoded length of a Footer. Note that the serialization of a 74 | // Footer will always occupy exactly this many bytes. It consists 75 | // of two block handles and a magic number. 76 | // EncodeTo()方法在编码Footer类实例的时候，需要编码两个BlockHandle类实例， 77 | // 以及一个64位的魔术字，这个魔术字的编码方式没有采用varint64的格式，而 78 | // 是用普通的编码方式进行编码，需要8个字节。所以编码Footer类实例所需要的 79 | // 最大内存就是kEncodedLength字节。 80 | enum { 81 | kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8 82 | }; 83 | 84 | private: 85 | BlockHandle metaindex_handle_; 86 | BlockHandle index_handle_; 87 | }; 88 | 89 | // kTableMagicNumber was picked by running 90 | // echo http://code.google.com/p/leveldb/ | sha1sum 91 | // and taking the leading 64 bits. 92 | static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; 93 | 94 | // 1-byte type + 32-bit crc 95 | // ssttable文件的读写也是以一个块为单位的，一个块中分为三部分 96 | // block + type + crc，所以这里的kBlockTrailerSize就包括了 97 | // type和crc的大小。 98 | static const size_t kBlockTrailerSize = 5; 99 | 100 | // BlockContents用来存放Block相关的一些信息，在ReadBlock()时使用。 101 | struct BlockContents { 102 | Slice data; // Actual contents of data 103 | bool cachable; // True iff data can be cached 104 | bool heap_allocated; // True iff caller should delete[] data.data() 105 | }; 106 | 107 | // Read the block identified by "handle" from "file". On failure 108 | // return non-OK. On success fill *result and return OK. 109 | extern Status ReadBlock(RandomAccessFile* file, 110 | const ReadOptions& options, 111 | const BlockHandle& handle, 112 | BlockContents* result); 113 | 114 | // Implementation details follow. Clients should ignore, 115 | 116 | inline BlockHandle::BlockHandle() 117 | : offset_(~static_cast(0)), 118 | size_(~static_cast(0)) { 119 | } 120 | 121 | } // namespace leveldb 122 | 123 | #endif // STORAGE_LEVELDB_TABLE_FORMAT_H_ 124 | -------------------------------------------------------------------------------- /leveldb-master/table/iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/iterator.h" 6 | 7 | namespace leveldb { 8 | 9 | Iterator::Iterator() { 10 | cleanup_.function = NULL; 11 | cleanup_.next = NULL; 12 | } 13 | 14 | Iterator::~Iterator() { 15 | if (cleanup_.function != NULL) { 16 | (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); 17 | for (Cleanup* c = cleanup_.next; c != NULL; ) { 18 | (*c->function)(c->arg1, c->arg2); 19 | Cleanup* next = c->next; 20 | delete c; 21 | c = next; 22 | } 23 | } 24 | } 25 | 26 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 27 | assert(func != NULL); 28 | Cleanup* c; 29 | if (cleanup_.function == NULL) { 30 | c = &cleanup_; 31 | } else { 32 | c = new Cleanup; 33 | c->next = cleanup_.next; 34 | cleanup_.next = c; 35 | } 36 | c->function = func; 37 | c->arg1 = arg1; 38 | c->arg2 = arg2; 39 | } 40 | 41 | namespace { 42 | class EmptyIterator : public Iterator { 43 | public: 44 | EmptyIterator(const Status& s) : status_(s) { } 45 | virtual bool Valid() const { return false; } 46 | virtual void Seek(const Slice& target) { } 47 | virtual void SeekToFirst() { } 48 | virtual void SeekToLast() { } 49 | virtual void Next() { assert(false); } 50 | virtual void Prev() { assert(false); } 51 | Slice key() const { assert(false); return Slice(); } 52 | Slice value() const { assert(false); return Slice(); } 53 | virtual Status status() const { return status_; } 54 | private: 55 | Status status_; 56 | }; 57 | } // namespace 58 | 59 | Iterator* NewEmptyIterator() { 60 | return new EmptyIterator(Status::OK()); 61 | } 62 | 63 | Iterator* NewErrorIterator(const Status& status) { 64 | return new EmptyIterator(status); 65 | } 66 | 67 | } // namespace leveldb 68 | -------------------------------------------------------------------------------- /leveldb-master/table/iterator_wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 6 | #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | #include "leveldb/slice.h" 10 | 11 | namespace leveldb { 12 | 13 | // A internal wrapper class with an interface similar to Iterator that 14 | // caches the valid() and key() results for an underlying iterator. 15 | // This can help avoid virtual function calls and also gives better 16 | // cache locality. 17 | class IteratorWrapper { 18 | public: 19 | IteratorWrapper(): iter_(NULL), valid_(false) { } 20 | explicit IteratorWrapper(Iterator* iter): iter_(NULL) { 21 | Set(iter); 22 | } 23 | ~IteratorWrapper() { delete iter_; } 24 | Iterator* iter() const { return iter_; } 25 | 26 | // Takes ownership of "iter" and will delete it when destroyed, or 27 | // when Set() is invoked again. 28 | void Set(Iterator* iter) { 29 | delete iter_; 30 | iter_ = iter; 31 | if (iter_ == NULL) { 32 | valid_ = false; 33 | } else { 34 | Update(); 35 | } 36 | } 37 | 38 | 39 | // Iterator interface methods 40 | bool Valid() const { return valid_; } 41 | Slice key() const { assert(Valid()); return key_; } 42 | Slice value() const { assert(Valid()); return iter_->value(); } 43 | // Methods below require iter() != NULL 44 | Status status() const { assert(iter_); return iter_->status(); } 45 | void Next() { assert(iter_); iter_->Next(); Update(); } 46 | void Prev() { assert(iter_); iter_->Prev(); Update(); } 47 | void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); } 48 | void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); } 49 | void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); } 50 | 51 | private: 52 | void Update() { 53 | valid_ = iter_->Valid(); 54 | if (valid_) { 55 | key_ = iter_->key(); 56 | } 57 | } 58 | 59 | Iterator* iter_; 60 | bool valid_; 61 | Slice key_; 62 | }; 63 | 64 | } // namespace leveldb 65 | 66 | #endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 67 | -------------------------------------------------------------------------------- /leveldb-master/table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 6 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Comparator; 11 | class Iterator; 12 | 13 | // Return an iterator that provided the union of the data in 14 | // children[0,n-1]. Takes ownership of the child iterators and 15 | // will delete them when the result iterator is deleted. 16 | // 17 | // The result does no duplicate suppression. I.e., if a particular 18 | // key is present in K child iterators, it will be yielded K times. 19 | // 20 | // REQUIRES: n >= 0 21 | extern Iterator* NewMergingIterator( 22 | const Comparator* comparator, Iterator** children, int n); 23 | 24 | } // namespace leveldb 25 | 26 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 27 | -------------------------------------------------------------------------------- /leveldb-master/table/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | 10 | namespace leveldb { 11 | 12 | struct ReadOptions; 13 | 14 | // Return a new two level iterator. A two-level iterator contains an 15 | // index iterator whose values point to a sequence of blocks where 16 | // each block is itself a sequence of key,value pairs. The returned 17 | // two-level iterator yields the concatenation of all key/value pairs 18 | // in the sequence of blocks. Takes ownership of "index_iter" and 19 | // will delete it when no longer needed. 20 | // 21 | // Uses a supplied function to convert an index_iter value into 22 | // an iterator over the contents of the corresponding block. 23 | extern Iterator* NewTwoLevelIterator( 24 | Iterator* index_iter, 25 | Iterator* (*block_function)( 26 | void* arg, 27 | const ReadOptions& options, 28 | const Slice& index_value), 29 | void* arg, 30 | const ReadOptions& options); 31 | 32 | } // namespace leveldb 33 | 34 | #endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 35 | -------------------------------------------------------------------------------- /leveldb-master/util/arena.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | #include 7 | 8 | namespace leveldb { 9 | 10 | // leveldb的Arena类用于向操作系统一次性申请的内存大小，即4K字节。 11 | static const int kBlockSize = 4096; 12 | 13 | // 构造函数将memory_usage_的内部指针初始化为0，指向地址为0的内存处，表明此时没有内存使用 14 | Arena::Arena() : memory_usage_(0) { 15 | alloc_ptr_ = NULL; // First allocation will allocate a block 16 | alloc_bytes_remaining_ = 0; 17 | } 18 | 19 | // Arena类的析构函数，将blocks_维护的向操作系统申请的内存统一进行释放。这样的话，在 20 | // 内存使用过程中就可以不用考虑内存释放问题，因为内存会在Arena类实例析构的时候统一 21 | // 进行释放操作。 22 | Arena::~Arena() { 23 | for (size_t i = 0; i < blocks_.size(); i++) { 24 | delete[] blocks_[i]; 25 | } 26 | } 27 | 28 | // Arena::AllocateFallback()函数根据bytes大小给上层调用者返回可用内存，如果bytes大于 29 | // kBlockSize / 4的话，则申请一块bytes大小内存直接返回给上层调用者；如果bytes小于等于 30 | // kBlockSize的话，则先申请一块kBlockSize大小内存，然后从这块新的内存中分配bytes大小 31 | // 内存给上层调用者。同时更新本次申请之后内部可用内存首地址alloc_ptr_和可用大小 32 | // alloc_bytes_remaining_的值。 33 | char* Arena::AllocateFallback(size_t bytes) { 34 | if (bytes > kBlockSize / 4) { 35 | // Object is more than a quarter of our block size. Allocate it separately 36 | // to avoid wasting too much space in leftover bytes. 37 | char* result = AllocateNewBlock(bytes); 38 | return result; 39 | } 40 | 41 | // We waste the remaining space in the current block. 42 | // 这个地方可能会导致部分内存的浪费。举个例子，假设之前的内存分配使得alloc_bytes_remaining_ 43 | // 为256B，但是本次上层调用者需要申请的内存大小为512B，那么alloc_ptr_指向的内存不够本次 44 | // 分配，那么这个时候leveldb就会重新申请一块大小为kBlockSize的内存，并让alloc_ptr_指向这块 45 | // 新内存，那alloc_ptr_原来指向的那块大小为256B的内存就没有被使用了，由于alloc_ptr_被重新 46 | // 赋值，所以原有那块256B内存就找不到了。 47 | // 个人觉得如果需要将内存分配控制得更精细一些的话，可以在内部用一个数组来维护内部可用内存 48 | // 以上面那种情况为例，如果alloc_ptr_指向的内存不够用了，那么就用数组元素保存其地址，后续 49 | // 当上层调用者需要的内存小于256B时，则可以使用其进行分配。 50 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 51 | alloc_bytes_remaining_ = kBlockSize; 52 | 53 | // 更新alloc_ptr_和alloc_bytes_remaining_的值，用于后续内存分配。 54 | char* result = alloc_ptr_; 55 | alloc_ptr_ += bytes; 56 | alloc_bytes_remaining_ -= bytes; 57 | return result; 58 | } 59 | 60 | // 函数Arena::AllocateAligned()用于给上层调用者返回一块字节对齐的内存。该函数相对于 61 | // 函数Arena::Allocate()来说的主要区别再于会返回一个内存首地址是字节对齐的内存。 62 | char* Arena::AllocateAligned(size_t bytes) { 63 | 64 | // 首先先计算出当前操作系统的字节对齐方式，即计算出操作系统内存地址是几字节对齐的。 65 | const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; 66 | assert((align & (align-1)) == 0); // Pointer size should be a power of 2 67 | 68 | // 这一步是计算alloc_ptr_指向的内存首地址相对于操作系统字节对齐的偏移量。 69 | // 以64位机器为例，上面计算可以得出align为8，假设经过之前的一些内存分配， 70 | // alloc_ptr_指向的内存地址的低四位为1011，那么下面计算出来的current_mod为0011， 71 | // 也就是说如果上层调用者需要的内存地址是字节对齐的话，那么alloc_ptr_就需要向 72 | // 后移动(align - current_mod)字节大小使其指向的内存地址是字节对齐的。也就是 73 | // 说要让alloc_ptr_指向的内存的地址是8的倍数。 74 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); 75 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 76 | 77 | // needed是总共需要的内存大小，包括上层调用者需要的大小以及因为地址对齐而浪费的大小。 78 | size_t needed = bytes + slop; 79 | char* result; 80 | 81 | // 下面的实现步骤就和Arena::Allocate()函数的一样了。 82 | if (needed <= alloc_bytes_remaining_) { 83 | result = alloc_ptr_ + slop; 84 | alloc_ptr_ += needed; 85 | alloc_bytes_remaining_ -= needed; 86 | } else { 87 | // AllocateFallback always returned aligned memory 88 | result = AllocateFallback(bytes); 89 | } 90 | assert((reinterpret_cast(result) & (align-1)) == 0); 91 | return result; 92 | } 93 | 94 | // Arena::AllocateNewBlock()函数用于向操作系统申请一块大小为block_bytes的内存， 95 | // 并将这块内存追加到blocks_这个vector中，后续归还给操作系统的时候需要使用到。 96 | // 另外，leveldb会将本次申请的内存大小记录起来，追加到memory_usage_成员中，用于 97 | // 统计总的内存使用量。 98 | char* Arena::AllocateNewBlock(size_t block_bytes) { 99 | char* result = new char[block_bytes]; 100 | blocks_.push_back(result); 101 | 102 | // memory_usage_成员统计总的内存使用量是如何实现的呢?可以看到，memory_usage_ 103 | // 成员是一个原子变量，其类型为void *，其用于统计总的内存使用量的原理是这样的： 104 | // 在Arena类初始化的时候，memory_usage_成员内部指针在构造函数内被初始化为0， 105 | // 也就是指向了地址为0的内存处，相当于目前没有内存申请，即使用量为0。当上层调用者 106 | // 申请一块内存时（以大于可BlockSize/4为例），最终会调用到Arena::AllocateNewBlock()函数 107 | // 那么leveldb先将类实例memory_usage_的内部指针强转为uintptr_t类型，然后在这个 108 | // 值基础上加上本次申请的大小，然后再强转会void *，那么memory_usage_的内部指针 109 | // 就指向了地址为block_bytes的内存，相当于目前使用了block_bytes的内存。 110 | memory_usage_.NoBarrier_Store( 111 | reinterpret_cast(MemoryUsage() + block_bytes + sizeof(char*))); 112 | return result; 113 | } 114 | 115 | } // namespace leveldb 116 | -------------------------------------------------------------------------------- /leveldb-master/util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "port/port.h" 13 | 14 | namespace leveldb { 15 | 16 | // 内存管理类 17 | class Arena { 18 | public: 19 | Arena(); 20 | ~Arena(); 21 | 22 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 23 | char* Allocate(size_t bytes); 24 | 25 | // Allocate memory with the normal alignment guarantees provided by malloc 26 | char* AllocateAligned(size_t bytes); 27 | 28 | // Returns an estimate of the total memory usage of data allocated 29 | // by the arena. 30 | // 统计内存使用量 31 | size_t MemoryUsage() const { 32 | return reinterpret_cast(memory_usage_.NoBarrier_Load()); 33 | } 34 | 35 | private: 36 | char* AllocateFallback(size_t bytes); 37 | char* AllocateNewBlock(size_t block_bytes); 38 | 39 | // Allocation state 40 | // alloc_ptr_成员指向的是当前可提供给上层调用者申请的内存的首地址。在leveldb中 41 | // 为了避免频繁向操作系统申请内存，首次上层调用者申请内存时，leveldb会申请一个大小 42 | // 为kBlockSize的内存，将操作系统返回的地址赋值给alloc_ptr_。以上层申请大小为size的 43 | // 内存为例，当size < kBlockSize时，leveldb会从alloc_ptr_指向的内存块分配一块上层指定 44 | // 大小的内存，然后alloc_ptr_则会指向alloc_ptr_ + size的地址，alloc_bytes_remaining_ 45 | // 则为kBlockSize - size。直到alloc_ptr_指向的可用内存不足以满足上层调用者需求时， 46 | // leveldb才会又申请一块大小为BlockSize的内存用于后续内存分配。如果size大于kBlockSize 47 | // 则leveldb会直接向操作系统申请对应大小的内存，并返回给上层调用者。 48 | char* alloc_ptr_; 49 | size_t alloc_bytes_remaining_; 50 | 51 | // Array of new[] allocated memory blocks 52 | // 在Arena类中，使用一个vector来管理申请的内存块，当Arena类析构时，统一将之前申请的所有 53 | // 内存返还给操作系统。 54 | std::vector blocks_; 55 | 56 | // Total memory usage of the arena. 57 | // 该私有变量用于统计内存使用量 58 | // memory_usage_成员统计总的内存使用量是如何实现的呢?可以看到，memory_usage_成员内部 59 | // 指针是一个原子变量，其内部指针类型为void *，其用于统计总的内存使用量的原理是这样的： 60 | // 在Arena类初始化的时候，memory_usage_成员内部指针在构造函数内被初始化为0， 61 | // 也就是指向了地址为0的内存处，相当于目前没有内存申请，即使用量为0。当上层调用者 62 | // 申请一块内存时（以大于可BlockSize/4为例），最终会调用到Arena::AllocateNewBlock()函数 63 | // 那么leveldb先将类实例memory_usage_的内部指针强转为uintptr_t类型，然后在这个 64 | // 值基础上加上本次申请的大小，然后再强转会void *，那么memory_usage_的内部指针 65 | // 就指向了地址为block_bytes的内存，相当于目前使用了block_bytes的内存。当然这个内存使用 66 | // 量统计是相对于操作系统而言的。 67 | port::AtomicPointer memory_usage_; 68 | 69 | // No copying allowed 70 | Arena(const Arena&); 71 | void operator=(const Arena&); 72 | }; 73 | 74 | // Arena::Allocate()函数用于提供给上层调用者申请一块大小为bytes的内存， 75 | // 该函数不保证给上层调用者的内存首地址是字节对齐的。如果上层调用者对 76 | // 内存的字节对齐有要求，则可以使用Arena::AllocateAligned()函数。 77 | inline char* Arena::Allocate(size_t bytes) { 78 | // The semantics of what to return are a bit messy if we allow 79 | // 0-byte allocations, so we disallow them here (we don't need 80 | // them for our internal use). 81 | assert(bytes > 0); 82 | 83 | // 如果上层调用者申请的内存小于当前leveldb内部可用内存时，则直接使用 84 | // 内部可用内存给上层调用者分配内存，并同时更新本次申请之后内部可用内存 85 | // 首地址和剩余大小。否则的话，调用AllocateFallback()函数根据bytes大小 86 | // 或申请一块bytes大小内存直接返回给上层调用者，或申请一块kBlockSize大小 87 | // 内存，然后从这块新的内存中分配bytes大小内存给上层调用者。 88 | if (bytes <= alloc_bytes_remaining_) { 89 | char* result = alloc_ptr_; 90 | alloc_ptr_ += bytes; 91 | alloc_bytes_remaining_ -= bytes; 92 | return result; 93 | } 94 | return AllocateFallback(bytes); 95 | } 96 | 97 | } // namespace leveldb 98 | 99 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 100 | -------------------------------------------------------------------------------- /leveldb-master/util/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | #include "util/random.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | class ArenaTest { }; 13 | 14 | TEST(ArenaTest, Empty) { 15 | Arena arena; 16 | } 17 | 18 | TEST(ArenaTest, Simple) { 19 | std::vector > allocated; 20 | Arena arena; 21 | const int N = 100000; 22 | size_t bytes = 0; 23 | Random rnd(301); 24 | for (int i = 0; i < N; i++) { 25 | size_t s; 26 | if (i % (N / 10) == 0) { 27 | s = i; 28 | } else { 29 | s = rnd.OneIn(4000) ? rnd.Uniform(6000) : 30 | (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 31 | } 32 | if (s == 0) { 33 | // Our arena disallows size 0 allocations. 34 | s = 1; 35 | } 36 | char* r; 37 | if (rnd.OneIn(10)) { 38 | r = arena.AllocateAligned(s); 39 | } else { 40 | r = arena.Allocate(s); 41 | } 42 | 43 | for (size_t b = 0; b < s; b++) { 44 | // Fill the "i"th allocation with a known bit pattern 45 | r[b] = i % 256; 46 | } 47 | bytes += s; 48 | allocated.push_back(std::make_pair(s, r)); 49 | ASSERT_GE(arena.MemoryUsage(), bytes); 50 | if (i > N/10) { 51 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 52 | } 53 | } 54 | for (size_t i = 0; i < allocated.size(); i++) { 55 | size_t num_bytes = allocated[i].first; 56 | const char* p = allocated[i].second; 57 | for (size_t b = 0; b < num_bytes; b++) { 58 | // Check the "i"th allocation for the known bit pattern 59 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 60 | } 61 | } 62 | } 63 | 64 | } // namespace leveldb 65 | 66 | int main(int argc, char** argv) { 67 | return leveldb::test::RunAllTests(); 68 | } 69 | -------------------------------------------------------------------------------- /leveldb-master/util/bloom.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | #include "leveldb/slice.h" 8 | #include "util/hash.h" 9 | 10 | namespace leveldb { 11 | 12 | namespace { 13 | static uint32_t BloomHash(const Slice& key) { 14 | return Hash(key.data(), key.size(), 0xbc9f1d34); 15 | } 16 | 17 | class BloomFilterPolicy : public FilterPolicy { 18 | private: 19 | 20 | // bits_per_key_存储的是每个key加入到Bloom过滤器时在位缓冲区占的位数 21 | size_t bits_per_key_; 22 | 23 | // k_存储的是key值加入到Bloom过滤器时需要执行的hash次数。 24 | size_t k_; 25 | 26 | public: 27 | explicit BloomFilterPolicy(int bits_per_key) 28 | : bits_per_key_(bits_per_key) { 29 | // We intentionally round down to reduce probing cost a little bit 30 | k_ = static_cast(bits_per_key * 0.69); // 0.69 =~ ln(2) 31 | if (k_ < 1) k_ = 1; 32 | if (k_ > 30) k_ = 30; 33 | } 34 | 35 | //Name()方法返回Bloom过滤器的名字。 36 | virtual const char* Name() const { 37 | return "leveldb.BuiltinBloomFilter2"; 38 | } 39 | 40 | // CreateFilter()用于创建Bloom过滤器 41 | virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { 42 | // Compute bloom filter size (in both bits and bytes) 43 | // 计算n个key总共会占用的位数 44 | size_t bits = n * bits_per_key_; 45 | 46 | // For small n, we can see a very high false positive rate. Fix it 47 | // by enforcing a minimum bloom filter length. 48 | if (bits < 64) bits = 64; 49 | 50 | // 计算总位数向上规整后所需要的字节数，并得出最后需要的位数 51 | size_t bytes = (bits + 7) / 8; 52 | bits = bytes * 8; 53 | 54 | // 对dst进行精准扩容，接着在原有内容后面追加每个key加入到Bloom过滤器时 55 | // 需要进行hash的次数。然后遍历所有的key值，将每个key值进行hash，得到的 56 | // hash值对总位数求余，然后利用余数将array数组中对应的二进制位置位。 57 | // 循环k_次之后array数组中key对应的位就都置位了。 58 | const size_t init_size = dst->size(); 59 | dst->resize(init_size + bytes, 0); 60 | dst->push_back(static_cast(k_)); // Remember # of probes in filter 61 | char* array = &(*dst)[init_size]; 62 | for (int i = 0; i < n; i++) { 63 | // Use double-hashing to generate a sequence of hash values. 64 | // See analysis in [Kirsch,Mitzenmacher 2006]. 65 | uint32_t h = BloomHash(keys[i]); 66 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 67 | for (size_t j = 0; j < k_; j++) { 68 | const uint32_t bitpos = h % bits; 69 | array[bitpos/8] |= (1 << (bitpos % 8)); 70 | h += delta; 71 | } 72 | } 73 | } 74 | 75 | // KeyMayMatch()函数用于判断key是否在bloom过滤器中，其判断逻辑为： 76 | // 1. 从bloom_filter中取出创建过滤器时存入到其中的key需要hash的次数。 77 | // 2. 计算key对应的bloom hash值。并计算出delta值。 78 | // 3. 然后循环判断每一次hash的结果在bloom filter中对应的位是否被置位，如果某次 79 | // 循环计算得到的hash值在bloom filter中对应的位没有被置位，说明key一定不在 80 | // bloom filter中，如果所有的hash值在bloom filter中的位都处于被置位状态，那么 81 | // key值很有可能在bloom filter中。 82 | virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const { 83 | const size_t len = bloom_filter.size(); 84 | if (len < 2) return false; 85 | 86 | const char* array = bloom_filter.data(); 87 | const size_t bits = (len - 1) * 8; 88 | 89 | // Use the encoded k so that we can read filters generated by 90 | // bloom filters created using different parameters. 91 | const size_t k = array[len-1]; 92 | if (k > 30) { 93 | // Reserved for potentially new encodings for short bloom filters. 94 | // Consider it a match. 95 | return true; 96 | } 97 | 98 | uint32_t h = BloomHash(key); 99 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits 100 | for (size_t j = 0; j < k; j++) { 101 | // 用hash值对总位数求余，得到本次hash值在bloom filter中对应的bit位。 102 | const uint32_t bitpos = h % bits; 103 | if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false; 104 | h += delta; 105 | } 106 | return true; 107 | } 108 | }; 109 | } 110 | 111 | // NewBloomFilterPolicy()函数用于创建一个BloomFilterPolicy类实例。 112 | const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) { 113 | return new BloomFilterPolicy(bits_per_key); 114 | } 115 | 116 | } // namespace leveldb 117 | -------------------------------------------------------------------------------- /leveldb-master/util/bloom_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | #include "util/coding.h" 8 | #include "util/logging.h" 9 | #include "util/testharness.h" 10 | #include "util/testutil.h" 11 | 12 | namespace leveldb { 13 | 14 | static const int kVerbose = 1; 15 | 16 | static Slice Key(int i, char* buffer) { 17 | EncodeFixed32(buffer, i); 18 | return Slice(buffer, sizeof(uint32_t)); 19 | } 20 | 21 | class BloomTest { 22 | private: 23 | const FilterPolicy* policy_; 24 | std::string filter_; 25 | std::vector keys_; 26 | 27 | public: 28 | BloomTest() : policy_(NewBloomFilterPolicy(10)) { } 29 | 30 | ~BloomTest() { 31 | delete policy_; 32 | } 33 | 34 | void Reset() { 35 | keys_.clear(); 36 | filter_.clear(); 37 | } 38 | 39 | void Add(const Slice& s) { 40 | keys_.push_back(s.ToString()); 41 | } 42 | 43 | void Build() { 44 | std::vector key_slices; 45 | for (size_t i = 0; i < keys_.size(); i++) { 46 | key_slices.push_back(Slice(keys_[i])); 47 | } 48 | filter_.clear(); 49 | policy_->CreateFilter(&key_slices[0], static_cast(key_slices.size()), 50 | &filter_); 51 | keys_.clear(); 52 | if (kVerbose >= 2) DumpFilter(); 53 | } 54 | 55 | size_t FilterSize() const { 56 | return filter_.size(); 57 | } 58 | 59 | void DumpFilter() { 60 | fprintf(stderr, "F("); 61 | for (size_t i = 0; i+1 < filter_.size(); i++) { 62 | const unsigned int c = static_cast(filter_[i]); 63 | for (int j = 0; j < 8; j++) { 64 | fprintf(stderr, "%c", (c & (1 <KeyMayMatch(s, filter_); 75 | } 76 | 77 | double FalsePositiveRate() { 78 | char buffer[sizeof(int)]; 79 | int result = 0; 80 | for (int i = 0; i < 10000; i++) { 81 | if (Matches(Key(i + 1000000000, buffer))) { 82 | result++; 83 | } 84 | } 85 | return result / 10000.0; 86 | } 87 | }; 88 | 89 | TEST(BloomTest, EmptyFilter) { 90 | ASSERT_TRUE(! Matches("hello")); 91 | ASSERT_TRUE(! Matches("world")); 92 | } 93 | 94 | TEST(BloomTest, Small) { 95 | Add("hello"); 96 | Add("world"); 97 | ASSERT_TRUE(Matches("hello")); 98 | ASSERT_TRUE(Matches("world")); 99 | ASSERT_TRUE(! Matches("x")); 100 | ASSERT_TRUE(! Matches("foo")); 101 | } 102 | 103 | static int NextLength(int length) { 104 | if (length < 10) { 105 | length += 1; 106 | } else if (length < 100) { 107 | length += 10; 108 | } else if (length < 1000) { 109 | length += 100; 110 | } else { 111 | length += 1000; 112 | } 113 | return length; 114 | } 115 | 116 | TEST(BloomTest, VaryingLengths) { 117 | char buffer[sizeof(int)]; 118 | 119 | // Count number of filters that significantly exceed the false positive rate 120 | int mediocre_filters = 0; 121 | int good_filters = 0; 122 | 123 | for (int length = 1; length <= 10000; length = NextLength(length)) { 124 | Reset(); 125 | for (int i = 0; i < length; i++) { 126 | Add(Key(i, buffer)); 127 | } 128 | Build(); 129 | 130 | ASSERT_LE(FilterSize(), static_cast((length * 10 / 8) + 40)) 131 | << length; 132 | 133 | // All added keys must match 134 | for (int i = 0; i < length; i++) { 135 | ASSERT_TRUE(Matches(Key(i, buffer))) 136 | << "Length " << length << "; key " << i; 137 | } 138 | 139 | // Check false positive rate 140 | double rate = FalsePositiveRate(); 141 | if (kVerbose >= 1) { 142 | fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n", 143 | rate*100.0, length, static_cast(FilterSize())); 144 | } 145 | ASSERT_LE(rate, 0.02); // Must not be over 2% 146 | if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often 147 | else good_filters++; 148 | } 149 | if (kVerbose >= 1) { 150 | fprintf(stderr, "Filters: %d good, %d mediocre\n", 151 | good_filters, mediocre_filters); 152 | } 153 | ASSERT_LE(mediocre_filters, good_filters/5); 154 | } 155 | 156 | // Different bits-per-byte 157 | 158 | } // namespace leveldb 159 | 160 | int main(int argc, char** argv) { 161 | return leveldb::test::RunAllTests(); 162 | } 163 | -------------------------------------------------------------------------------- /leveldb-master/util/comparator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/slice.h" 9 | #include "port/port.h" 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | 14 | Comparator::~Comparator() { } 15 | 16 | namespace { 17 | class BytewiseComparatorImpl : public Comparator { 18 | public: 19 | BytewiseComparatorImpl() { } 20 | 21 | // Name()返回当前比较器实现的名字。 22 | virtual const char* Name() const { 23 | return "leveldb.BytewiseComparator"; 24 | } 25 | 26 | // Compare()函数比较两个slice的大小，从实现可以看出其是利用了Slice对象 27 | // 的比较方法。 28 | virtual int Compare(const Slice& a, const Slice& b) const { 29 | return a.compare(b); 30 | } 31 | 32 | // If *start < limit, changes *start to a short string in [start,limit)。 33 | virtual void FindShortestSeparator( 34 | std::string* start, 35 | const Slice& limit) const { 36 | // Find length of common prefix 37 | // 1. 找到start和limit的共同前缀，记录下两者不同部分的起始索引 38 | size_t min_length = std::min(start->size(), limit.size()); 39 | size_t diff_index = 0; 40 | while ((diff_index < min_length) && 41 | ((*start)[diff_index] == limit[diff_index])) { 42 | diff_index++; 43 | } 44 | 45 | // 经过上面的处理，diff_index就是start和limit不同部分的起始索引。 46 | // 如果diff_index大于等于min_length，说明start和limit中有一个是另外 47 | // 一个的前缀，在这种情况下无论谁是谁的前缀，我们都不做任何操作， 48 | // 因为如果start是limit的前缀，那么start天然满足[start, limit); 49 | // 如果limit是start的前缀，那么前提条件都不满足，所以也不做任何操作。 50 | if (diff_index >= min_length) { 51 | // Do not shorten if one string is a prefix of the other 52 | } else { 53 | // 如果diff_index小于min_length，说明start和limit还存在一部分不相同的。 54 | // 这种情况下，我们只判断不同部分的第一个字节，如果start中与limit 55 | // 不同部分的第一个字节内容不等于255，也比limit中对应字节至少小2，那么就 56 | // 将这个start中的这个字节加1，并修改start的长度，使得修改后的start动态字符串 57 | // 比原start动态字符串大，这样的start就满足了[start, limit)；如果start比 58 | // limit大的话，那前提条件不满足就不用做任何处理了。 59 | uint8_t diff_byte = static_cast((*start)[diff_index]); 60 | if (diff_byte < static_cast(0xff) && 61 | diff_byte + 1 < static_cast(limit[diff_index])) { 62 | (*start)[diff_index]++; 63 | start->resize(diff_index + 1); 64 | assert(Compare(*start, limit) < 0); 65 | } 66 | } 67 | } 68 | 69 | // Changes *key to a short string >= *key，该函数的作用就是将key设置为一个 70 | // 比原来key大，但是比原来key短的动态字符串。其实现方法就是： 71 | // 遍历动态字符窜key，找到其中第一个值不为0xff的字节，然后将该字节的值 72 | // 在原来基础上加1，这样就使得新的动态字符串key就大于旧的动态字符串key了。 73 | // 然后修改新的动态字符串的大小。 74 | virtual void FindShortSuccessor(std::string* key) const { 75 | // Find first character that can be incremented 76 | size_t n = key->size(); 77 | for (size_t i = 0; i < n; i++) { 78 | const uint8_t byte = (*key)[i]; 79 | if (byte != static_cast(0xff)) { 80 | (*key)[i] = byte + 1; 81 | key->resize(i+1); 82 | return; 83 | } 84 | } 85 | // *key is a run of 0xffs. Leave it alone. 86 | } 87 | }; 88 | } // namespace 89 | 90 | static port::OnceType once = LEVELDB_ONCE_INIT; 91 | static const Comparator* bytewise; 92 | 93 | static void InitModule() { 94 | bytewise = new BytewiseComparatorImpl; 95 | } 96 | 97 | // 创建并返回一个bytewise比较器。 98 | const Comparator* BytewiseComparator() { 99 | port::InitOnce(&once, InitModule); 100 | return bytewise; 101 | } 102 | 103 | } // namespace leveldb 104 | -------------------------------------------------------------------------------- /leveldb-master/util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ 6 | #define STORAGE_LEVELDB_UTIL_CRC32C_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace crc32c { 13 | 14 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 15 | // crc32c of some string A. Extend() is often used to maintain the 16 | // crc32c of a stream of data. 17 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 18 | 19 | // Return the crc32c of data[0,n-1] 20 | inline uint32_t Value(const char* data, size_t n) { 21 | return Extend(0, data, n); 22 | } 23 | 24 | static const uint32_t kMaskDelta = 0xa282ead8ul; 25 | 26 | // Return a masked representation of crc. 27 | // 28 | // Motivation: it is problematic to compute the CRC of a string that 29 | // contains embedded CRCs. Therefore we recommend that CRCs stored 30 | // somewhere (e.g., in files) should be masked before being stored. 31 | inline uint32_t Mask(uint32_t crc) { 32 | // Rotate right by 15 bits and add a constant. 33 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 34 | } 35 | 36 | // Return the crc whose masked representation is masked_crc. 37 | inline uint32_t Unmask(uint32_t masked_crc) { 38 | uint32_t rot = masked_crc - kMaskDelta; 39 | return ((rot >> 17) | (rot << 15)); 40 | } 41 | 42 | } // namespace crc32c 43 | } // namespace leveldb 44 | 45 | #endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ 46 | -------------------------------------------------------------------------------- /leveldb-master/util/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/crc32c.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | namespace crc32c { 10 | 11 | class CRC { }; 12 | 13 | TEST(CRC, StandardResults) { 14 | // From rfc3720 section B.4. 15 | char buf[32]; 16 | 17 | memset(buf, 0, sizeof(buf)); 18 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 19 | 20 | memset(buf, 0xff, sizeof(buf)); 21 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 22 | 23 | for (int i = 0; i < 32; i++) { 24 | buf[i] = i; 25 | } 26 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 27 | 28 | for (int i = 0; i < 32; i++) { 29 | buf[i] = 31 - i; 30 | } 31 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 32 | 33 | unsigned char data[48] = { 34 | 0x01, 0xc0, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x00, 0x00, 37 | 0x00, 0x00, 0x00, 0x00, 38 | 0x14, 0x00, 0x00, 0x00, 39 | 0x00, 0x00, 0x04, 0x00, 40 | 0x00, 0x00, 0x00, 0x14, 41 | 0x00, 0x00, 0x00, 0x18, 42 | 0x28, 0x00, 0x00, 0x00, 43 | 0x00, 0x00, 0x00, 0x00, 44 | 0x02, 0x00, 0x00, 0x00, 45 | 0x00, 0x00, 0x00, 0x00, 46 | }; 47 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 48 | } 49 | 50 | TEST(CRC, Values) { 51 | ASSERT_NE(Value("a", 1), Value("foo", 3)); 52 | } 53 | 54 | TEST(CRC, Extend) { 55 | ASSERT_EQ(Value("hello world", 11), 56 | Extend(Value("hello ", 6), "world", 5)); 57 | } 58 | 59 | TEST(CRC, Mask) { 60 | uint32_t crc = Value("foo", 3); 61 | ASSERT_NE(crc, Mask(crc)); 62 | ASSERT_NE(crc, Mask(Mask(crc))); 63 | ASSERT_EQ(crc, Unmask(Mask(crc))); 64 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 65 | } 66 | 67 | } // namespace crc32c 68 | } // namespace leveldb 69 | 70 | int main(int argc, char** argv) { 71 | return leveldb::test::RunAllTests(); 72 | } 73 | -------------------------------------------------------------------------------- /leveldb-master/util/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/env.h" 6 | 7 | namespace leveldb { 8 | 9 | Env::~Env() { 10 | } 11 | 12 | Status Env::NewAppendableFile(const std::string& fname, WritableFile** result) { 13 | return Status::NotSupported("NewAppendableFile", fname); 14 | } 15 | 16 | SequentialFile::~SequentialFile() { 17 | } 18 | 19 | RandomAccessFile::~RandomAccessFile() { 20 | } 21 | 22 | WritableFile::~WritableFile() { 23 | } 24 | 25 | Logger::~Logger() { 26 | } 27 | 28 | FileLock::~FileLock() { 29 | } 30 | 31 | void Log(Logger* info_log, const char* format, ...) { 32 | if (info_log != NULL) { 33 | va_list ap; 34 | va_start(ap, format); 35 | info_log->Logv(format, ap); 36 | va_end(ap); 37 | } 38 | } 39 | 40 | static Status DoWriteStringToFile(Env* env, const Slice& data, 41 | const std::string& fname, 42 | bool should_sync) { 43 | WritableFile* file; 44 | Status s = env->NewWritableFile(fname, &file); 45 | if (!s.ok()) { 46 | return s; 47 | } 48 | s = file->Append(data); 49 | if (s.ok() && should_sync) { 50 | s = file->Sync(); 51 | } 52 | if (s.ok()) { 53 | s = file->Close(); 54 | } 55 | delete file; // Will auto-close if we did not close above 56 | if (!s.ok()) { 57 | env->DeleteFile(fname); 58 | } 59 | return s; 60 | } 61 | 62 | Status WriteStringToFile(Env* env, const Slice& data, 63 | const std::string& fname) { 64 | return DoWriteStringToFile(env, data, fname, false); 65 | } 66 | 67 | Status WriteStringToFileSync(Env* env, const Slice& data, 68 | const std::string& fname) { 69 | return DoWriteStringToFile(env, data, fname, true); 70 | } 71 | 72 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { 73 | data->clear(); 74 | SequentialFile* file; 75 | Status s = env->NewSequentialFile(fname, &file); 76 | if (!s.ok()) { 77 | return s; 78 | } 79 | static const int kBufferSize = 8192; 80 | char* space = new char[kBufferSize]; 81 | while (true) { 82 | Slice fragment; 83 | s = file->Read(kBufferSize, &fragment, space); 84 | if (!s.ok()) { 85 | break; 86 | } 87 | data->append(fragment.data(), fragment.size()); 88 | if (fragment.empty()) { 89 | break; 90 | } 91 | } 92 | delete[] space; 93 | delete file; 94 | return s; 95 | } 96 | 97 | EnvWrapper::~EnvWrapper() { 98 | } 99 | 100 | } // namespace leveldb 101 | -------------------------------------------------------------------------------- /leveldb-master/util/env_posix_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/env.h" 6 | 7 | #include "port/port.h" 8 | #include "util/testharness.h" 9 | #include "util/env_posix_test_helper.h" 10 | 11 | namespace leveldb { 12 | 13 | static const int kDelayMicros = 100000; 14 | static const int kReadOnlyFileLimit = 4; 15 | static const int kMMapLimit = 4; 16 | 17 | class EnvPosixTest { 18 | public: 19 | Env* env_; 20 | EnvPosixTest() : env_(Env::Default()) { } 21 | 22 | static void SetFileLimits(int read_only_file_limit, int mmap_limit) { 23 | EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit); 24 | EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit); 25 | } 26 | }; 27 | 28 | TEST(EnvPosixTest, TestOpenOnRead) { 29 | // Write some test data to a single file that will be opened |n| times. 30 | std::string test_dir; 31 | ASSERT_OK(env_->GetTestDirectory(&test_dir)); 32 | std::string test_file = test_dir + "/open_on_read.txt"; 33 | 34 | FILE* f = fopen(test_file.c_str(), "w"); 35 | ASSERT_TRUE(f != NULL); 36 | const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; 37 | fputs(kFileData, f); 38 | fclose(f); 39 | 40 | // Open test file some number above the sum of the two limits to force 41 | // open-on-read behavior of POSIX Env leveldb::RandomAccessFile. 42 | const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5; 43 | leveldb::RandomAccessFile* files[kNumFiles] = {0}; 44 | for (int i = 0; i < kNumFiles; i++) { 45 | ASSERT_OK(env_->NewRandomAccessFile(test_file, &files[i])); 46 | } 47 | char scratch; 48 | Slice read_result; 49 | for (int i = 0; i < kNumFiles; i++) { 50 | ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch)); 51 | ASSERT_EQ(kFileData[i], read_result[0]); 52 | } 53 | for (int i = 0; i < kNumFiles; i++) { 54 | delete files[i]; 55 | } 56 | ASSERT_OK(env_->DeleteFile(test_file)); 57 | } 58 | 59 | } // namespace leveldb 60 | 61 | int main(int argc, char** argv) { 62 | // All tests currently run with the same read-only file limits. 63 | leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit, 64 | leveldb::kMMapLimit); 65 | return leveldb::test::RunAllTests(); 66 | } 67 | -------------------------------------------------------------------------------- /leveldb-master/util/env_posix_test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 6 | #define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class EnvPosixTest; 11 | 12 | // A helper for the POSIX Env to facilitate testing. 13 | class EnvPosixTestHelper { 14 | private: 15 | friend class EnvPosixTest; 16 | 17 | // Set the maximum number of read-only files that will be opened. 18 | // Must be called before creating an Env. 19 | static void SetReadOnlyFDLimit(int limit); 20 | 21 | // Set the maximum number of read-only files that will be mapped via mmap. 22 | // Must be called before creating an Env. 23 | static void SetReadOnlyMMapLimit(int limit); 24 | }; 25 | 26 | } // namespace leveldb 27 | 28 | #endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ 29 | -------------------------------------------------------------------------------- /leveldb-master/util/filter_policy.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/filter_policy.h" 6 | 7 | namespace leveldb { 8 | 9 | FilterPolicy::~FilterPolicy() { } 10 | 11 | } // namespace leveldb 12 | -------------------------------------------------------------------------------- /leveldb-master/util/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "util/coding.h" 7 | #include "util/hash.h" 8 | 9 | // The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through 10 | // between switch labels. The real definition should be provided externally. 11 | // This one is a fallback version for unsupported compilers. 12 | #ifndef FALLTHROUGH_INTENDED 13 | #define FALLTHROUGH_INTENDED do { } while (0) 14 | #endif 15 | 16 | namespace leveldb { 17 | 18 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 19 | // Similar to murmur hash 20 | const uint32_t m = 0xc6a4a793; 21 | const uint32_t r = 24; 22 | const char* limit = data + n; 23 | uint32_t h = seed ^ (n * m); 24 | 25 | // Pick up four bytes at a time 26 | while (data + 4 <= limit) { 27 | uint32_t w = DecodeFixed32(data); 28 | data += 4; 29 | h += w; 30 | h *= m; 31 | h ^= (h >> 16); 32 | } 33 | 34 | // Pick up remaining bytes 35 | switch (limit - data) { 36 | case 3: 37 | h += static_cast(data[2]) << 16; 38 | FALLTHROUGH_INTENDED; 39 | case 2: 40 | h += static_cast(data[1]) << 8; 41 | FALLTHROUGH_INTENDED; 42 | case 1: 43 | h += static_cast(data[0]); 44 | h *= m; 45 | h ^= (h >> r); 46 | break; 47 | } 48 | return h; 49 | } 50 | 51 | 52 | } // namespace leveldb 53 | -------------------------------------------------------------------------------- /leveldb-master/util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Simple hash function used for internal data structures 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 8 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 9 | 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 16 | 17 | } 18 | 19 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 20 | -------------------------------------------------------------------------------- /leveldb-master/util/hash_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/hash.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | class HASH { }; 11 | 12 | TEST(HASH, SignedUnsignedIssue) { 13 | const unsigned char data1[1] = {0x62}; 14 | const unsigned char data2[2] = {0xc3, 0x97}; 15 | const unsigned char data3[3] = {0xe2, 0x99, 0xa5}; 16 | const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32}; 17 | const unsigned char data5[48] = { 18 | 0x01, 0xc0, 0x00, 0x00, 19 | 0x00, 0x00, 0x00, 0x00, 20 | 0x00, 0x00, 0x00, 0x00, 21 | 0x00, 0x00, 0x00, 0x00, 22 | 0x14, 0x00, 0x00, 0x00, 23 | 0x00, 0x00, 0x04, 0x00, 24 | 0x00, 0x00, 0x00, 0x14, 25 | 0x00, 0x00, 0x00, 0x18, 26 | 0x28, 0x00, 0x00, 0x00, 27 | 0x00, 0x00, 0x00, 0x00, 28 | 0x02, 0x00, 0x00, 0x00, 29 | 0x00, 0x00, 0x00, 0x00, 30 | }; 31 | 32 | ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34); 33 | ASSERT_EQ( 34 | Hash(reinterpret_cast(data1), sizeof(data1), 0xbc9f1d34), 35 | 0xef1345c4); 36 | ASSERT_EQ( 37 | Hash(reinterpret_cast(data2), sizeof(data2), 0xbc9f1d34), 38 | 0x5b663814); 39 | ASSERT_EQ( 40 | Hash(reinterpret_cast(data3), sizeof(data3), 0xbc9f1d34), 41 | 0x323c078f); 42 | ASSERT_EQ( 43 | Hash(reinterpret_cast(data4), sizeof(data4), 0xbc9f1d34), 44 | 0xed21633a); 45 | ASSERT_EQ( 46 | Hash(reinterpret_cast(data5), sizeof(data5), 0x12345678), 47 | 0xf333dabb); 48 | } 49 | 50 | } // namespace leveldb 51 | 52 | int main(int argc, char** argv) { 53 | return leveldb::test::RunAllTests(); 54 | } 55 | -------------------------------------------------------------------------------- /leveldb-master/util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 6 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Histogram { 13 | public: 14 | Histogram() { } 15 | ~Histogram() { } 16 | 17 | void Clear(); 18 | void Add(double value); 19 | void Merge(const Histogram& other); 20 | 21 | std::string ToString() const; 22 | 23 | private: 24 | double min_; 25 | double max_; 26 | double num_; 27 | double sum_; 28 | double sum_squares_; 29 | 30 | enum { kNumBuckets = 154 }; 31 | static const double kBucketLimit[kNumBuckets]; 32 | double buckets_[kNumBuckets]; 33 | 34 | double Median() const; 35 | double Percentile(double p) const; 36 | double Average() const; 37 | double StandardDeviation() const; 38 | }; 39 | 40 | } // namespace leveldb 41 | 42 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 43 | -------------------------------------------------------------------------------- /leveldb-master/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "leveldb/env.h" 12 | #include "leveldb/slice.h" 13 | 14 | namespace leveldb { 15 | 16 | // AppendNumberTo()函数用于将number转换成一个字符串，并将结果追加到str中。 17 | void AppendNumberTo(std::string* str, uint64_t num) { 18 | char buf[30]; 19 | snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); 20 | str->append(buf); 21 | } 22 | 23 | // 将Slice对象中的内容编码到str中，其中对于Slice对象value中的可打印字符， 24 | // 那么直接将其追加到str中，而对于其中的不可打印字符，则以十六进制 25 | // 形式将不可见字符转换成对应的整形数字，取最低字节内容，拼接在"\\x"之后，然后追加到str中。 26 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 27 | for (size_t i = 0; i < value.size(); i++) { 28 | char c = value[i]; 29 | if (c >= ' ' && c <= '~') { 30 | str->push_back(c); 31 | } else { 32 | char buf[10]; 33 | snprintf(buf, sizeof(buf), "\\x%02x", 34 | static_cast(c) & 0xff); 35 | str->append(buf); 36 | } 37 | } 38 | } 39 | 40 | // NumberToString()函数会将number转换成一个string对象 41 | std::string NumberToString(uint64_t num) { 42 | std::string r; 43 | AppendNumberTo(&r, num); 44 | return r; 45 | } 46 | 47 | // EscapeString()函数将Slice对象内容转换成字符串。 48 | std::string EscapeString(const Slice& value) { 49 | std::string r; 50 | AppendEscapedStringTo(&r, value); 51 | return r; 52 | } 53 | 54 | // ConsumeDecimalNumber()函数用于将Slice对象in中的数字部分转换为对应的数字，但受以下条件制约： 55 | // 1. 如果Slice对象in不是以数字字符开始，那么将直接返回，将0赋值给*val，并返回false，表示*val值无效。 56 | // 2. 遇到Slice对象in中第一个非数字字符后，就不再处理Slice对象in的后续内容，而直接将前面计算 57 | // 得到的数字作为最终结果直接赋值给*val，返回true，表示值有效。 58 | // 3. 如果Slice对象in中已经处理完的数字字符解析结果超过了64位无符号整形最大值，那么将认为值无效， 59 | // 返回false，*val将会是0。 60 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 61 | uint64_t v = 0; 62 | int digits = 0; 63 | while (!in->empty()) { 64 | char c = (*in)[0]; 65 | if (c >= '0' && c <= '9') { 66 | ++digits; 67 | // |delta| intentionally unit64_t to avoid Android crash (see log). 68 | const uint64_t delta = (c - '0'); 69 | // 如果 70 | static const uint64_t kMaxUint64 = ~static_cast(0); 71 | if (v > kMaxUint64/10 || 72 | (v == kMaxUint64/10 && delta > kMaxUint64%10)) { 73 | // Overflow 74 | return false; 75 | } 76 | v = (v * 10) + delta; 77 | in->remove_prefix(1); 78 | } else { 79 | break; 80 | } 81 | } 82 | *val = v; 83 | return (digits > 0); 84 | } 85 | 86 | } // namespace leveldb 87 | -------------------------------------------------------------------------------- /leveldb-master/util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Slice; 19 | class WritableFile; 20 | 21 | // Append a human-readable printout of "num" to *str 22 | extern void AppendNumberTo(std::string* str, uint64_t num); 23 | 24 | // Append a human-readable printout of "value" to *str. 25 | // Escapes any non-printable characters found in "value". 26 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 27 | 28 | // Return a human-readable printout of "num" 29 | extern std::string NumberToString(uint64_t num); 30 | 31 | // Return a human-readable version of "value". 32 | // Escapes any non-printable characters found in "value". 33 | extern std::string EscapeString(const Slice& value); 34 | 35 | // Parse a human-readable number from "*in" into *value. On success, 36 | // advances "*in" past the consumed number and sets "*val" to the 37 | // numeric value. Otherwise, returns false and leaves *in in an 38 | // unspecified state. 39 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 40 | 41 | } // namespace leveldb 42 | 43 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 44 | -------------------------------------------------------------------------------- /leveldb-master/util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | #include "port/thread_annotations.h" 10 | 11 | namespace leveldb { 12 | 13 | // Helper class that locks a mutex on construction and unlocks the mutex when 14 | // the destructor of the MutexLock object is invoked. 15 | // 16 | // Typical usage: 17 | // 18 | // void MyClass::MyMethod() { 19 | // MutexLock l(&mu_); // mu_ is an instance variable 20 | // ... some complex code, possibly with multiple return paths ... 21 | // } 22 | 23 | class SCOPED_LOCKABLE MutexLock { 24 | public: 25 | explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) 26 | : mu_(mu) { 27 | this->mu_->Lock(); 28 | } 29 | ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 30 | 31 | private: 32 | port::Mutex *const mu_; 33 | // No copying allowed 34 | MutexLock(const MutexLock&); 35 | void operator=(const MutexLock&); 36 | }; 37 | 38 | } // namespace leveldb 39 | 40 | 41 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 42 | -------------------------------------------------------------------------------- /leveldb-master/util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/options.h" 6 | 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/env.h" 9 | 10 | namespace leveldb { 11 | 12 | Options::Options() 13 | : comparator(BytewiseComparator()), 14 | create_if_missing(false), 15 | error_if_exists(false), 16 | paranoid_checks(false), 17 | env(Env::Default()), 18 | info_log(NULL), 19 | write_buffer_size(4<<20), 20 | max_open_files(1000), 21 | block_cache(NULL), 22 | block_size(4096), 23 | block_restart_interval(16), 24 | max_file_size(2<<20), 25 | compression(kSnappyCompression), 26 | reuse_logs(false), 27 | filter_policy(NULL) { 28 | } 29 | 30 | } // namespace leveldb 31 | -------------------------------------------------------------------------------- /leveldb-master/util/posix_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Logger implementation that can be shared by all environments 6 | // where enough posix functionality is available. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 9 | #define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "leveldb/env.h" 16 | 17 | namespace leveldb { 18 | 19 | class PosixLogger : public Logger { 20 | private: 21 | FILE* file_; 22 | uint64_t (*gettid_)(); // Return the thread id for the current thread 23 | public: 24 | PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { } 25 | virtual ~PosixLogger() { 26 | fclose(file_); 27 | } 28 | virtual void Logv(const char* format, va_list ap) { 29 | const uint64_t thread_id = (*gettid_)(); 30 | 31 | // We try twice: the first time with a fixed-size stack allocated buffer, 32 | // and the second time with a much larger dynamically allocated buffer. 33 | char buffer[500]; 34 | for (int iter = 0; iter < 2; iter++) { 35 | char* base; 36 | int bufsize; 37 | if (iter == 0) { 38 | bufsize = sizeof(buffer); 39 | base = buffer; 40 | } else { 41 | bufsize = 30000; 42 | base = new char[bufsize]; 43 | } 44 | char* p = base; 45 | char* limit = base + bufsize; 46 | 47 | struct timeval now_tv; 48 | gettimeofday(&now_tv, NULL); 49 | const time_t seconds = now_tv.tv_sec; 50 | struct tm t; 51 | localtime_r(&seconds, &t); 52 | p += snprintf(p, limit - p, 53 | "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", 54 | t.tm_year + 1900, 55 | t.tm_mon + 1, 56 | t.tm_mday, 57 | t.tm_hour, 58 | t.tm_min, 59 | t.tm_sec, 60 | static_cast(now_tv.tv_usec), 61 | static_cast(thread_id)); 62 | 63 | // Print the message 64 | if (p < limit) { 65 | va_list backup_ap; 66 | va_copy(backup_ap, ap); 67 | p += vsnprintf(p, limit - p, format, backup_ap); 68 | va_end(backup_ap); 69 | } 70 | 71 | // Truncate to available space if necessary 72 | if (p >= limit) { 73 | if (iter == 0) { 74 | continue; // Try again with larger buffer 75 | } else { 76 | p = limit - 1; 77 | } 78 | } 79 | 80 | // Add newline if necessary 81 | if (p == base || p[-1] != '\n') { 82 | *p++ = '\n'; 83 | } 84 | 85 | assert(p <= limit); 86 | fwrite(base, 1, p - base, file_); 87 | fflush(file_); 88 | if (base != buffer) { 89 | delete[] base; 90 | } 91 | break; 92 | } 93 | } 94 | }; 95 | 96 | } // namespace leveldb 97 | 98 | #endif // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ 99 | -------------------------------------------------------------------------------- /leveldb-master/util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ 6 | #define STORAGE_LEVELDB_UTIL_RANDOM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | // A very simple random number generator. Not especially good at 13 | // generating truly random bits, but good enough for our needs in this 14 | // package. 15 | class Random { 16 | private: 17 | uint32_t seed_; 18 | public: 19 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { 20 | // Avoid bad seeds. 21 | if (seed_ == 0 || seed_ == 2147483647L) { 22 | seed_ = 1; 23 | } 24 | } 25 | uint32_t Next() { 26 | static const uint32_t M = 2147483647L; // 2^31-1 27 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 28 | // We are computing 29 | // seed_ = (seed_ * A) % M, where M = 2^31-1 30 | // 31 | // seed_ must not be zero or M, or else all subsequent computed values 32 | // will be zero or M respectively. For all other values, seed_ will end 33 | // up cycling through every number in [1,M-1] 34 | uint64_t product = seed_ * A; 35 | 36 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 37 | seed_ = static_cast((product >> 31) + (product & M)); 38 | // The first reduction may overflow by 1 bit, so we may need to 39 | // repeat. mod == M is not possible; using > allows the faster 40 | // sign-bit-based test. 41 | if (seed_ > M) { 42 | seed_ -= M; 43 | } 44 | return seed_; 45 | } 46 | // Returns a uniformly distributed value in the range [0..n-1] 47 | // REQUIRES: n > 0 48 | uint32_t Uniform(int n) { return Next() % n; } 49 | 50 | // Randomly returns true ~"1/n" of the time, and false otherwise. 51 | // REQUIRES: n > 0 52 | bool OneIn(int n) { return (Next() % n) == 0; } 53 | 54 | // Skewed: pick "base" uniformly from range [0,max_log] and then 55 | // return "base" random bits. The effect is to pick a number in the 56 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 57 | uint32_t Skewed(int max_log) { 58 | return Uniform(1 << Uniform(max_log + 1)); 59 | } 60 | }; 61 | 62 | } // namespace leveldb 63 | 64 | #endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ 65 | -------------------------------------------------------------------------------- /leveldb-master/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "port/port.h" 7 | #include "leveldb/status.h" 8 | 9 | namespace leveldb { 10 | 11 | // 申请一块内存，然后将state中的内容拷贝到新申请的内存中，然后将新申请内存作为返回值。 12 | const char* Status::CopyState(const char* state) { 13 | uint32_t size; 14 | memcpy(&size, state, sizeof(size)); 15 | char* result = new char[size + 5]; 16 | memcpy(result, state, size + 5); 17 | return result; 18 | } 19 | 20 | // Status::Status()是类Status的构造函数，利用code和msg构造出一个状态信息，状态信息的 21 | // 编码格式为： 22 | // state_[0..3] == length of message 23 | // state_[4] == code 24 | // state_[5..] == message 25 | 26 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 27 | assert(code != kOk); 28 | const uint32_t len1 = msg.size(); 29 | const uint32_t len2 = msg2.size(); 30 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 31 | 32 | // 多申请5个字节的内存，用于存放信息内容和状态码。 33 | char* result = new char[size + 5]; 34 | 35 | // 按照上述格式构造状态信息。 36 | memcpy(result, &size, sizeof(size)); 37 | result[4] = static_cast(code); 38 | memcpy(result + 5, msg.data(), len1); 39 | if (len2) { 40 | result[5 + len1] = ':'; 41 | result[6 + len1] = ' '; 42 | memcpy(result + 7 + len1, msg2.data(), len2); 43 | } 44 | 45 | // 状态信息构造好之后存放到state_这个私有成员中。 46 | state_ = result; 47 | } 48 | 49 | std::string Status::ToString() const { 50 | 51 | // 获取状态码对应的字符串 52 | if (state_ == NULL) { 53 | return "OK"; 54 | } else { 55 | char tmp[30]; 56 | const char* type; 57 | switch (code()) { 58 | case kOk: 59 | type = "OK"; 60 | break; 61 | case kNotFound: 62 | type = "NotFound: "; 63 | break; 64 | case kCorruption: 65 | type = "Corruption: "; 66 | break; 67 | case kNotSupported: 68 | type = "Not implemented: "; 69 | break; 70 | case kInvalidArgument: 71 | type = "Invalid argument: "; 72 | break; 73 | case kIOError: 74 | type = "IO error: "; 75 | break; 76 | default: 77 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 78 | static_cast(code())); 79 | type = tmp; 80 | break; 81 | } 82 | 83 | // 将状态码对应的字符串用来初始化结果字符串。 84 | std::string result(type); 85 | uint32_t length; 86 | 87 | // state_中存放的第一部分是状态信息的长度部分。 88 | memcpy(&length, state_, sizeof(length)); 89 | 90 | // 将状态信息内容追加到结果字符串中。到最后结果字符串中的字符串编码格式为： 91 | // ------------------------------- 92 | // | code string | state content | 93 | // ------------------------------- 94 | result.append(state_ + 5, length); 95 | return result; 96 | } 97 | } 98 | 99 | } // namespace leveldb 100 | -------------------------------------------------------------------------------- /leveldb-master/util/testharness.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testharness.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | namespace { 16 | struct Test { 17 | const char* base; 18 | const char* name; 19 | void (*func)(); 20 | }; 21 | std::vector* tests; 22 | } 23 | 24 | bool RegisterTest(const char* base, const char* name, void (*func)()) { 25 | if (tests == NULL) { 26 | tests = new std::vector; 27 | } 28 | Test t; 29 | t.base = base; 30 | t.name = name; 31 | t.func = func; 32 | tests->push_back(t); 33 | return true; 34 | } 35 | 36 | int RunAllTests() { 37 | const char* matcher = getenv("LEVELDB_TESTS"); 38 | 39 | int num = 0; 40 | if (tests != NULL) { 41 | for (size_t i = 0; i < tests->size(); i++) { 42 | const Test& t = (*tests)[i]; 43 | if (matcher != NULL) { 44 | std::string name = t.base; 45 | name.push_back('.'); 46 | name.append(t.name); 47 | if (strstr(name.c_str(), matcher) == NULL) { 48 | continue; 49 | } 50 | } 51 | fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); 52 | (*t.func)(); 53 | ++num; 54 | } 55 | } 56 | fprintf(stderr, "==== PASSED %d tests\n", num); 57 | return 0; 58 | } 59 | 60 | std::string TmpDir() { 61 | std::string dir; 62 | Status s = Env::Default()->GetTestDirectory(&dir); 63 | ASSERT_TRUE(s.ok()) << s.ToString(); 64 | return dir; 65 | } 66 | 67 | int RandomSeed() { 68 | const char* env = getenv("TEST_RANDOM_SEED"); 69 | int result = (env != NULL ? atoi(env) : 301); 70 | if (result <= 0) { 71 | result = 301; 72 | } 73 | return result; 74 | } 75 | 76 | } // namespace test 77 | } // namespace leveldb 78 | -------------------------------------------------------------------------------- /leveldb-master/util/testutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testutil.h" 6 | 7 | #include "util/random.h" 8 | 9 | namespace leveldb { 10 | namespace test { 11 | 12 | Slice RandomString(Random* rnd, int len, std::string* dst) { 13 | dst->resize(len); 14 | for (int i = 0; i < len; i++) { 15 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 16 | } 17 | return Slice(*dst); 18 | } 19 | 20 | std::string RandomKey(Random* rnd, int len) { 21 | // Make sure to generate a wide variety of characters so we 22 | // test the boundary conditions for short-key optimizations. 23 | static const char kTestChars[] = { 24 | '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' 25 | }; 26 | std::string result; 27 | for (int i = 0; i < len; i++) { 28 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 29 | } 30 | return result; 31 | } 32 | 33 | 34 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 35 | size_t len, std::string* dst) { 36 | int raw = static_cast(len * compressed_fraction); 37 | if (raw < 1) raw = 1; 38 | std::string raw_data; 39 | RandomString(rnd, raw, &raw_data); 40 | 41 | // Duplicate the random data until we have filled "len" bytes 42 | dst->clear(); 43 | while (dst->size() < len) { 44 | dst->append(raw_data); 45 | } 46 | dst->resize(len); 47 | return Slice(*dst); 48 | } 49 | 50 | } // namespace test 51 | } // namespace leveldb 52 | -------------------------------------------------------------------------------- /leveldb-master/util/testutil.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 6 | #define STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 7 | 8 | #include "leveldb/env.h" 9 | #include "leveldb/slice.h" 10 | #include "util/random.h" 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | // Store in *dst a random string of length "len" and return a Slice that 16 | // references the generated data. 17 | extern Slice RandomString(Random* rnd, int len, std::string* dst); 18 | 19 | // Return a random key with the specified length that may contain interesting 20 | // characters (e.g. \x00, \xff, etc.). 21 | extern std::string RandomKey(Random* rnd, int len); 22 | 23 | // Store in *dst a string of length "len" that will compress to 24 | // "N*compressed_fraction" bytes and return a Slice that references 25 | // the generated data. 26 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 27 | size_t len, std::string* dst); 28 | 29 | // A wrapper that allows injection of errors. 30 | class ErrorEnv : public EnvWrapper { 31 | public: 32 | bool writable_file_error_; 33 | int num_writable_file_errors_; 34 | 35 | ErrorEnv() : EnvWrapper(Env::Default()), 36 | writable_file_error_(false), 37 | num_writable_file_errors_(0) { } 38 | 39 | virtual Status NewWritableFile(const std::string& fname, 40 | WritableFile** result) { 41 | if (writable_file_error_) { 42 | ++num_writable_file_errors_; 43 | *result = NULL; 44 | return Status::IOError(fname, "fake error"); 45 | } 46 | return target()->NewWritableFile(fname, result); 47 | } 48 | 49 | virtual Status NewAppendableFile(const std::string& fname, 50 | WritableFile** result) { 51 | if (writable_file_error_) { 52 | ++num_writable_file_errors_; 53 | *result = NULL; 54 | return Status::IOError(fname, "fake error"); 55 | } 56 | return target()->NewAppendableFile(fname, result); 57 | } 58 | }; 59 | 60 | } // namespace test 61 | } // namespace leveldb 62 | 63 | #endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 64 | --------------------------------------------------------------------------------