├── RELEASE-SCRIPT ├── .gitignore ├── leveldb ├── AUTHORS ├── TODO ├── port │ ├── README │ ├── port.h │ ├── win │ │ └── stdint.h │ ├── sha1_portable.h │ ├── sha1_test.cc │ ├── port_osx.cc │ ├── port_posix.cc │ ├── port_android.cc │ ├── port_chromium.cc │ ├── port_chromium.h │ ├── port_posix.h │ ├── port_osx.h │ ├── port_android.h │ └── port_example.h ├── NEWS ├── util │ ├── hash.h │ ├── options.cc │ ├── histogram.h │ ├── hash.cc │ ├── mutexlock.h │ ├── crc32c.h │ ├── testharness.cc │ ├── testutil.cc │ ├── logging.h │ ├── env.cc │ ├── testutil.h │ ├── arena_test.cc │ ├── crc32c_test.cc │ ├── arena.h │ ├── logging.cc │ ├── status.cc │ ├── arena.cc │ ├── comparator.cc │ ├── random.h │ ├── env_test.cc │ ├── coding.h │ ├── testharness.h │ ├── histogram.cc │ ├── cache_test.cc │ ├── coding_test.cc │ └── coding.cc ├── db │ ├── db_iter.h │ ├── log_format.h │ ├── builder.h │ ├── log_writer.h │ ├── version_edit_test.cc │ ├── write_batch_internal.h │ ├── table_cache.h │ ├── snapshot.h │ ├── builder.cc │ ├── memtable.h │ ├── write_batch_test.cc │ ├── table_cache.cc │ ├── filename.h │ ├── dbformat.cc │ ├── log_writer.cc │ ├── version_edit.h │ ├── filename_test.cc │ ├── memtable.cc │ ├── log_reader.h │ ├── write_batch.cc │ ├── dbformat_test.cc │ └── filename.cc ├── table │ ├── merger.h │ ├── block.h │ ├── two_level_iterator.h │ ├── iterator.cc │ ├── block_builder.h │ ├── iterator_wrapper.h │ ├── format.h │ ├── block_builder.cc │ ├── format.cc │ └── two_level_iterator.cc ├── doc │ ├── doc.css │ ├── table_format.txt │ └── log_format.txt ├── LICENSE ├── README └── include │ └── leveldb │ ├── write_batch.h │ ├── comparator.h │ ├── table.h │ ├── status.h │ ├── table_builder.h │ ├── slice.h │ ├── iterator.h │ └── cache.h ├── ext └── leveldb │ └── extconf.rb ├── README ├── lib └── leveldb.rb └── Rakefile /RELEASE-SCRIPT: -------------------------------------------------------------------------------- 1 | rake gem 2 | gem push pkg/leveldb-ruby-$VERSION.gem 3 | git archive --format=tar --prefix=leveldb-ruby-$VERSION/ HEAD | gzip -9 > leveldb-ruby-$VERSION.tar.gz 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *.o 3 | ext/leveldb/Makefile 4 | ext/leveldb/extconf.h 5 | ext/leveldb/leveldb.so 6 | ext/leveldb/libleveldb.a 7 | leveldb/libleveldb.a 8 | leveldb/leveldb.gyp 9 | pkg 10 | -------------------------------------------------------------------------------- /leveldb/AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | Google Inc. 5 | 6 | # Initial version authors: 7 | Jeffrey Dean 8 | Sanjay Ghemawat 9 | -------------------------------------------------------------------------------- /ext/leveldb/extconf.rb: -------------------------------------------------------------------------------- 1 | require 'mkmf' 2 | require 'fileutils' 3 | 4 | Dir.chdir "../../leveldb" 5 | system "make libleveldb.a" or abort 6 | Dir.chdir "../ext/leveldb" 7 | 8 | CONFIG['LDSHARED'] = "$(CXX) -shared" 9 | 10 | $CFLAGS << " -I../../leveldb/include" 11 | $LIBS << " -L../../leveldb -lleveldb" 12 | create_makefile "leveldb/leveldb" 13 | -------------------------------------------------------------------------------- /leveldb/TODO: -------------------------------------------------------------------------------- 1 | ss 2 | - Stats 3 | 4 | db 5 | - Maybe implement DB::BulkDeleteForRange(start_key, end_key) 6 | that would blow away files whose ranges are entirely contained 7 | within [start_key..end_key]? For Chrome, deletion of obsolete 8 | object stores, etc. can be done in the background anyway, so 9 | probably not that important. 10 | 11 | api changes: 12 | - Make it wrappable 13 | 14 | Faster Get implementation 15 | -------------------------------------------------------------------------------- /leveldb/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /leveldb/NEWS: -------------------------------------------------------------------------------- 1 | Release 1.2 2011-05-16 2 | ---------------------- 3 | 4 | Fixes for larger databases (tested up to one billion 100-byte entries, 5 | i.e., ~100GB). 6 | 7 | (1) Place hard limit on number of level-0 files. This fixes errors 8 | of the form "too many open files". 9 | 10 | (2) Fixed memtable management. Before the fix, a heavy write burst 11 | could cause unbounded memory usage. 12 | 13 | A fix for a logging bug where the reader would incorrectly complain 14 | about corruption. 15 | 16 | Allow public access to WriteBatch contents so that users can easily 17 | wrap a DB. 18 | -------------------------------------------------------------------------------- /leveldb/util/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Simple hash function used for internal data structures 6 | 7 | #ifndef STORAGE_LEVELDB_UTIL_HASH_H_ 8 | #define STORAGE_LEVELDB_UTIL_HASH_H_ 9 | 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | extern uint32_t Hash(const char* data, size_t n, uint32_t seed); 16 | 17 | } 18 | 19 | #endif // STORAGE_LEVELDB_UTIL_HASH_H_ 20 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | LevelDB Ruby bindings. 2 | 3 | Do this: 4 | 5 | gem install leveldb-ruby 6 | 7 | 8 | And then run this: 9 | 10 | require 'rubygems' # on ruby < 1.9 11 | require 'leveldb' 12 | 13 | db = LevelDB::DB.new "/tmp/asdf" 14 | db.put "it", "works" 15 | db.get "it" 16 | 17 | Only a very few operations are supported right now: 18 | 19 | DB.new # creates or loads as necessary 20 | DB.create # dies if it already exists 21 | DB.load # dies unless it alread exists 22 | 23 | DB#get # alias: DB#[] 24 | DB#put # alias: DB#[]= 25 | DB#delete 26 | 27 | DB#each 28 | ... and all the enumerable methods 29 | -------------------------------------------------------------------------------- /leveldb/util/options.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/options.h" 6 | 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/env.h" 9 | 10 | namespace leveldb { 11 | 12 | Options::Options() 13 | : comparator(BytewiseComparator()), 14 | create_if_missing(false), 15 | error_if_exists(false), 16 | paranoid_checks(false), 17 | env(Env::Default()), 18 | info_log(NULL), 19 | write_buffer_size(4<<20), 20 | max_open_files(1000), 21 | block_cache(NULL), 22 | block_size(4096), 23 | block_restart_interval(16), 24 | compression(kSnappyCompression) { 25 | } 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /leveldb/db/db_iter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ 6 | #define STORAGE_LEVELDB_DB_DB_ITER_H_ 7 | 8 | #include 9 | #include "leveldb/db.h" 10 | #include "db/dbformat.h" 11 | 12 | namespace leveldb { 13 | 14 | // Return a new iterator that converts internal keys (yielded by 15 | // "*internal_iter") that were live at the specified "sequence" number 16 | // into appropriate user keys. 17 | extern Iterator* NewDBIterator( 18 | const std::string* dbname, 19 | Env* env, 20 | const Comparator* user_key_comparator, 21 | Iterator* internal_iter, 22 | const SequenceNumber& sequence); 23 | 24 | } 25 | 26 | #endif // STORAGE_LEVELDB_DB_DB_ITER_H_ 27 | -------------------------------------------------------------------------------- /leveldb/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_posix.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #elif defined(LEVELDB_PLATFORM_ANDROID) 18 | # include "port/port_android.h" 19 | #elif defined(LEVELDB_PLATFORM_OSX) 20 | # include "port/port_osx.h" 21 | #endif 22 | 23 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 24 | -------------------------------------------------------------------------------- /leveldb/table/merger.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_MERGER_H_ 6 | #define STORAGE_LEVELDB_TABLE_MERGER_H_ 7 | 8 | namespace leveldb { 9 | 10 | class Comparator; 11 | class Iterator; 12 | 13 | // Return an iterator that provided the union of the data in 14 | // children[0,n-1]. Takes ownership of the child iterators and 15 | // will delete them when the result iterator is deleted. 16 | // 17 | // The result does no duplicate suppression. I.e., if a particular 18 | // key is present in K child iterators, it will be yielded K times. 19 | // 20 | // REQUIRES: n >= 0 21 | extern Iterator* NewMergingIterator( 22 | const Comparator* comparator, Iterator** children, int n); 23 | 24 | } 25 | 26 | #endif // STORAGE_LEVELDB_TABLE_MERGER_H_ 27 | -------------------------------------------------------------------------------- /leveldb/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /leveldb/port/sha1_portable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ 6 | #define STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | namespace port { 12 | 13 | // Compute the SHA1 hash value of "data[0..len-1]" and store it in 14 | // "hash_array[0..19]". hash_array must have 20 bytes of space available. 15 | // 16 | // This function is portable but may not be as fast as a version 17 | // optimized for your platform. It is provided as a default method 18 | // that can be used when porting leveldb to a new platform if no 19 | // better SHA1 hash implementation is available. 20 | void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array); 21 | 22 | } 23 | } 24 | 25 | #endif // STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_ 26 | -------------------------------------------------------------------------------- /lib/leveldb.rb: -------------------------------------------------------------------------------- 1 | require 'leveldb/leveldb' # the c extension 2 | 3 | module LevelDB 4 | class DB 5 | include Enumerable 6 | class << self 7 | 8 | ## Loads or creates a LevelDB database as necessary, stored on disk at 9 | ## +pathname+. 10 | def new pathname 11 | make pathname, true, false 12 | end 13 | 14 | ## Creates a new LevelDB database stored on disk at +pathname+. Throws an 15 | ## exception if the database already exists. 16 | def create pathname 17 | make pathname, true, true 18 | end 19 | 20 | ## Loads a LevelDB database stored on disk at +pathname+. Throws an 21 | ## exception unless the database already exists. 22 | def load pathname 23 | make pathname, false, false 24 | end 25 | end 26 | 27 | alias :includes? :exists? 28 | alias :contains? :exists? 29 | alias :member? :exists? 30 | alias :[] :get 31 | alias :[]= :put 32 | 33 | def keys; map { |k, v| k } end 34 | def values; map { |k, v| v } end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /leveldb/db/log_format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Log format information shared by reader and writer. 6 | // See ../doc/log_format.txt for more detail. 7 | 8 | #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 9 | #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 10 | 11 | namespace leveldb { 12 | namespace log { 13 | 14 | enum RecordType { 15 | // Zero is reserved for preallocated files 16 | kZeroType = 0, 17 | 18 | kFullType = 1, 19 | 20 | // For fragments 21 | kFirstType = 2, 22 | kMiddleType = 3, 23 | kLastType = 4, 24 | }; 25 | static const int kMaxRecordType = kLastType; 26 | 27 | static const int kBlockSize = 32768; 28 | 29 | // Header is checksum (4 bytes), type (1 byte), length (2 bytes). 30 | static const int kHeaderSize = 4 + 1 + 2; 31 | 32 | } 33 | } 34 | 35 | #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ 36 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rubygems/package_task' 3 | require 'find' 4 | 5 | spec = Gem::Specification.new do |s| 6 | s.name = "leveldb-ruby" 7 | s.version = "0.7" 8 | s.date = Time.now.to_s 9 | s.email = "wmorgan-leveldb-ruby-gemspec@masanjin.net" 10 | s.authors = ["William Morgan"] 11 | s.summary = "a Ruby binding to LevelDB" 12 | s.homepage = "http://github.com/wmorgan/leveldb-ruby" 13 | s.files = %w(README ext/leveldb/extconf.rb lib/leveldb.rb ext/leveldb/leveldb.cc leveldb/Makefile) 14 | Find.find("leveldb") { |x| s.files << x if x =~ /\.(cc|h)$/} 15 | s.extensions = %w(ext/leveldb/extconf.rb) 16 | s.executables = [] 17 | s.extra_rdoc_files = %w(README ext/leveldb/leveldb.cc) 18 | s.rdoc_options = %w(-c utf8 --main README --title LevelDB) 19 | s.description = "LevelDB-Ruby is a Ruby binding to LevelDB." 20 | end 21 | 22 | task :rdoc do |t| 23 | sh "rm -rf doc; rdoc #{spec.rdoc_options.join(' ')} #{spec.extra_rdoc_files.join(' ')} lib/leveldb.rb" 24 | end 25 | 26 | Gem::PackageTask.new spec do 27 | end 28 | 29 | # vim: syntax=ruby 30 | -------------------------------------------------------------------------------- /leveldb/util/histogram.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 6 | #define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Histogram { 13 | public: 14 | Histogram() { } 15 | ~Histogram() { } 16 | 17 | void Clear(); 18 | void Add(double value); 19 | 20 | std::string ToString() const; 21 | 22 | private: 23 | double min_; 24 | double max_; 25 | double num_; 26 | double sum_; 27 | double sum_squares_; 28 | 29 | enum { kNumBuckets = 154 }; 30 | static const double kBucketLimit[kNumBuckets]; 31 | double buckets_[kNumBuckets]; 32 | 33 | double Median() const; 34 | double Percentile(double p) const; 35 | double Average() const; 36 | double StandardDeviation() const; 37 | }; 38 | 39 | } 40 | 41 | #endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_ 42 | -------------------------------------------------------------------------------- /leveldb/util/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "util/coding.h" 7 | #include "util/hash.h" 8 | 9 | namespace leveldb { 10 | 11 | uint32_t Hash(const char* data, size_t n, uint32_t seed) { 12 | // Similar to murmur hash 13 | const uint32_t m = 0xc6a4a793; 14 | const uint32_t r = 24; 15 | const char* limit = data + n; 16 | uint32_t h = seed ^ (n * m); 17 | 18 | // Pick up four bytes at a time 19 | while (data + 4 <= limit) { 20 | uint32_t w = DecodeFixed32(data); 21 | data += 4; 22 | h += w; 23 | h *= m; 24 | h ^= (h >> 16); 25 | } 26 | 27 | // Pick up remaining bytes 28 | switch (limit - data) { 29 | case 3: 30 | h += data[2] << 16; 31 | // fall through 32 | case 2: 33 | h += data[1] << 8; 34 | // fall through 35 | case 1: 36 | h += data[0]; 37 | h *= m; 38 | h ^= (h >> r); 39 | break; 40 | } 41 | return h; 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /leveldb/util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | 10 | namespace leveldb { 11 | 12 | // Helper class that locks a mutex on construction and unlocks the mutex when 13 | // the destructor of the MutexLock object is invoked. 14 | // 15 | // Typical usage: 16 | // 17 | // void MyClass::MyMethod() { 18 | // MutexLock l(&mu_); // mu_ is an instance variable 19 | // ... some complex code, possibly with multiple return paths ... 20 | // } 21 | 22 | class MutexLock { 23 | public: 24 | explicit MutexLock(port::Mutex *mu) : mu_(mu) { 25 | this->mu_->Lock(); 26 | } 27 | ~MutexLock() { this->mu_->Unlock(); } 28 | 29 | private: 30 | port::Mutex *const mu_; 31 | // No copying allowed 32 | MutexLock(const MutexLock&); 33 | void operator=(const MutexLock&); 34 | }; 35 | 36 | } 37 | 38 | 39 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 40 | -------------------------------------------------------------------------------- /leveldb/table/block.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_H_ 7 | 8 | #include 9 | #include 10 | #include "leveldb/iterator.h" 11 | 12 | namespace leveldb { 13 | 14 | class Comparator; 15 | 16 | class Block { 17 | public: 18 | // Initialize the block with the specified contents. 19 | // Takes ownership of data[] and will delete[] it when done. 20 | Block(const char* data, size_t size); 21 | 22 | ~Block(); 23 | 24 | size_t size() const { return size_; } 25 | Iterator* NewIterator(const Comparator* comparator); 26 | 27 | private: 28 | uint32_t NumRestarts() const; 29 | 30 | const char* data_; 31 | size_t size_; 32 | uint32_t restart_offset_; // Offset in data_ of restart array 33 | 34 | // No copying allowed 35 | Block(const Block&); 36 | void operator=(const Block&); 37 | 38 | class Iter; 39 | }; 40 | 41 | } 42 | 43 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_H_ 44 | -------------------------------------------------------------------------------- /leveldb/port/sha1_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | namespace port { 10 | 11 | class SHA1 { }; 12 | 13 | static std::string TestSHA1(const char* data, size_t len) { 14 | char hash_val[20]; 15 | SHA1_Hash(data, len, hash_val); 16 | char buf[41]; 17 | for (int i = 0; i < 20; i++) { 18 | snprintf(buf + i * 2, 41 - i * 2, 19 | "%02x", 20 | static_cast(static_cast( 21 | hash_val[i]))); 22 | } 23 | return std::string(buf, 40); 24 | } 25 | 26 | TEST(SHA1, Simple) { 27 | ASSERT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", TestSHA1("", 0)); 28 | ASSERT_EQ("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", TestSHA1("hello", 5)); 29 | std::string x(10000, 'x'); 30 | ASSERT_EQ("f8c5cde791c5056cf515881e701c8a9ecb439a75", 31 | TestSHA1(x.data(), x.size())); 32 | } 33 | 34 | } 35 | } 36 | 37 | int main(int argc, char** argv) { 38 | return leveldb::test::RunAllTests(); 39 | } 40 | -------------------------------------------------------------------------------- /leveldb/db/builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_BUILDER_H_ 6 | #define STORAGE_LEVELDB_DB_BUILDER_H_ 7 | 8 | #include "leveldb/status.h" 9 | 10 | namespace leveldb { 11 | 12 | struct Options; 13 | struct FileMetaData; 14 | 15 | class Env; 16 | class Iterator; 17 | class TableCache; 18 | class VersionEdit; 19 | 20 | // Build a Table file from the contents of *iter. The generated file 21 | // will be named according to meta->number. On success, the rest of 22 | // *meta will be filled with metadata about the generated table, and 23 | // the file information will be added to *edit. If no data is present 24 | // in *iter, meta->file_size will be set to zero, and no Table file 25 | // will be produced. 26 | extern Status BuildTable(const std::string& dbname, 27 | Env* env, 28 | const Options& options, 29 | TableCache* table_cache, 30 | Iterator* iter, 31 | FileMetaData* meta, 32 | VersionEdit* edit); 33 | 34 | } 35 | 36 | #endif // STORAGE_LEVELDB_DB_BUILDER_H_ 37 | -------------------------------------------------------------------------------- /leveldb/table/two_level_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 6 | #define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 7 | 8 | #include "leveldb/iterator.h" 9 | 10 | namespace leveldb { 11 | 12 | struct ReadOptions; 13 | 14 | // Return a new two level iterator. A two-level iterator contains an 15 | // index iterator whose values point to a sequence of blocks where 16 | // each block is itself a sequence of key,value pairs. The returned 17 | // two-level iterator yields the concatenation of all key/value pairs 18 | // in the sequence of blocks. Takes ownership of "index_iter" and 19 | // will delete it when no longer needed. 20 | // 21 | // Uses a supplied function to convert an index_iter value into 22 | // an iterator over the contents of the corresponding block. 23 | extern Iterator* NewTwoLevelIterator( 24 | Iterator* index_iter, 25 | Iterator* (*block_function)( 26 | void* arg, 27 | const ReadOptions& options, 28 | const Slice& index_value), 29 | void* arg, 30 | const ReadOptions& options); 31 | 32 | } 33 | 34 | #endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_ 35 | -------------------------------------------------------------------------------- /leveldb/db/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ 7 | 8 | #include 9 | #include "db/log_format.h" 10 | #include "leveldb/slice.h" 11 | #include "leveldb/status.h" 12 | 13 | namespace leveldb { 14 | 15 | class WritableFile; 16 | 17 | namespace log { 18 | 19 | class Writer { 20 | public: 21 | // Create a writer that will append data to "*dest". 22 | // "*dest" must be initially empty. 23 | // "*dest" must remain live while this Writer is in use. 24 | explicit Writer(WritableFile* dest); 25 | ~Writer(); 26 | 27 | Status AddRecord(const Slice& slice); 28 | 29 | private: 30 | WritableFile* dest_; 31 | int block_offset_; // Current offset in block 32 | 33 | // crc32c values for all supported record types. These are 34 | // pre-computed to reduce the overhead of computing the crc of the 35 | // record type stored in the header. 36 | uint32_t type_crc_[kMaxRecordType + 1]; 37 | 38 | Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); 39 | 40 | // No copying allowed 41 | Writer(const Writer&); 42 | void operator=(const Writer&); 43 | }; 44 | 45 | } 46 | } 47 | 48 | #endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_ 49 | -------------------------------------------------------------------------------- /leveldb/port/port_osx.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port_osx.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | namespace port { 14 | 15 | static void PthreadCall(const char* label, int result) { 16 | if (result != 0) { 17 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 18 | abort(); 19 | } 20 | } 21 | 22 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 23 | 24 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 25 | 26 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 27 | 28 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 29 | 30 | CondVar::CondVar(Mutex* mu) 31 | : mu_(mu) { 32 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 33 | } 34 | 35 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 36 | 37 | void CondVar::Wait() { 38 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 39 | } 40 | 41 | void CondVar::Signal() { 42 | PthreadCall("signal", pthread_cond_signal(&cv_)); 43 | } 44 | 45 | void CondVar::SignalAll() { 46 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /leveldb/port/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_posix.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | namespace port { 14 | 15 | static void PthreadCall(const char* label, int result) { 16 | if (result != 0) { 17 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 18 | abort(); 19 | } 20 | } 21 | 22 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 23 | 24 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 25 | 26 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 27 | 28 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 29 | 30 | CondVar::CondVar(Mutex* mu) 31 | : mu_(mu) { 32 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 33 | } 34 | 35 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 36 | 37 | void CondVar::Wait() { 38 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 39 | } 40 | 41 | void CondVar::Signal() { 42 | PthreadCall("signal", pthread_cond_signal(&cv_)); 43 | } 44 | 45 | void CondVar::SignalAll() { 46 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /leveldb/db/version_edit_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/version_edit.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | 10 | static void TestEncodeDecode(const VersionEdit& edit) { 11 | std::string encoded, encoded2; 12 | edit.EncodeTo(&encoded); 13 | VersionEdit parsed; 14 | Status s = parsed.DecodeFrom(encoded); 15 | ASSERT_TRUE(s.ok()) << s.ToString(); 16 | parsed.EncodeTo(&encoded2); 17 | ASSERT_EQ(encoded, encoded2); 18 | } 19 | 20 | class VersionEditTest { }; 21 | 22 | TEST(VersionEditTest, EncodeDecode) { 23 | static const uint64_t kBig = 1ull << 50; 24 | 25 | VersionEdit edit; 26 | for (int i = 0; i < 4; i++) { 27 | TestEncodeDecode(edit); 28 | edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 29 | InternalKey("foo", kBig + 500 + i, kTypeValue), 30 | InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); 31 | edit.DeleteFile(4, kBig + 700 + i); 32 | edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); 33 | } 34 | 35 | edit.SetComparatorName("foo"); 36 | edit.SetLogNumber(kBig + 100); 37 | edit.SetNextFile(kBig + 200); 38 | edit.SetLastSequence(kBig + 1000); 39 | TestEncodeDecode(edit); 40 | } 41 | 42 | } 43 | 44 | int main(int argc, char** argv) { 45 | return leveldb::test::RunAllTests(); 46 | } 47 | -------------------------------------------------------------------------------- /leveldb/doc/doc.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin-left: 0.5in; 3 | margin-right: 0.5in; 4 | background: white; 5 | color: black; 6 | } 7 | 8 | h1 { 9 | margin-left: -0.2in; 10 | font-size: 14pt; 11 | } 12 | h2 { 13 | margin-left: -0in; 14 | font-size: 12pt; 15 | } 16 | h3 { 17 | margin-left: -0in; 18 | } 19 | h4 { 20 | margin-left: -0in; 21 | } 22 | hr { 23 | margin-left: -0in; 24 | } 25 | 26 | /* Definition lists: definition term bold */ 27 | dt { 28 | font-weight: bold; 29 | } 30 | 31 | address { 32 | text-align: center; 33 | } 34 | code,samp,var { 35 | color: blue; 36 | } 37 | kbd { 38 | color: #600000; 39 | } 40 | div.note p { 41 | float: right; 42 | width: 3in; 43 | margin-right: 0%; 44 | padding: 1px; 45 | border: 2px solid #6060a0; 46 | background-color: #fffff0; 47 | } 48 | 49 | ul { 50 | margin-top: -0em; 51 | margin-bottom: -0em; 52 | } 53 | 54 | ol { 55 | margin-top: -0em; 56 | margin-bottom: -0em; 57 | } 58 | 59 | UL.nobullets { 60 | list-style-type: none; 61 | list-style-image: none; 62 | margin-left: -1em; 63 | } 64 | 65 | p { 66 | margin: 1em 0 1em 0; 67 | padding: 0 0 0 0; 68 | } 69 | 70 | pre { 71 | line-height: 1.3em; 72 | padding: 0.4em 0 0.8em 0; 73 | margin: 0 0 0 0; 74 | border: 0 0 0 0; 75 | color: blue; 76 | } 77 | 78 | .datatable { 79 | margin-left: auto; 80 | margin-right: auto; 81 | margin-top: 2em; 82 | margin-bottom: 2em; 83 | border: 1px solid; 84 | } 85 | 86 | .datatable td,th { 87 | padding: 0 0.5em 0 0.5em; 88 | text-align: right; 89 | } 90 | -------------------------------------------------------------------------------- /leveldb/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /leveldb/util/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_ 6 | #define STORAGE_LEVELDB_UTIL_CRC32C_H_ 7 | 8 | #include 9 | #include 10 | 11 | namespace leveldb { 12 | namespace crc32c { 13 | 14 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 15 | // crc32c of some string A. Extend() is often used to maintain the 16 | // crc32c of a stream of data. 17 | extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n); 18 | 19 | // Return the crc32c of data[0,n-1] 20 | inline uint32_t Value(const char* data, size_t n) { 21 | return Extend(0, data, n); 22 | } 23 | 24 | static const uint32_t kMaskDelta = 0xa282ead8ul; 25 | 26 | // Return a masked representation of crc. 27 | // 28 | // Motivation: it is problematic to compute the CRC of a string that 29 | // contains embedded CRCs. Therefore we recommend that CRCs stored 30 | // somewhere (e.g., in files) should be masked before being stored. 31 | inline uint32_t Mask(uint32_t crc) { 32 | // Rotate right by 15 bits and add a constant. 33 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 34 | } 35 | 36 | // Return the crc whose masked representation is masked_crc. 37 | inline uint32_t Unmask(uint32_t masked_crc) { 38 | uint32_t rot = masked_crc - kMaskDelta; 39 | return ((rot >> 17) | (rot << 15)); 40 | } 41 | 42 | } 43 | } 44 | 45 | #endif // STORAGE_LEVELDB_UTIL_CRC32C_H_ 46 | -------------------------------------------------------------------------------- /leveldb/db/write_batch_internal.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 6 | #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 7 | 8 | #include "leveldb/write_batch.h" 9 | 10 | namespace leveldb { 11 | 12 | // WriteBatchInternal provides static methods for manipulating a 13 | // WriteBatch that we don't want in the public WriteBatch interface. 14 | class WriteBatchInternal { 15 | public: 16 | // Return the number of entries in the batch. 17 | static int Count(const WriteBatch* batch); 18 | 19 | // Set the count for the number of entries in the batch. 20 | static void SetCount(WriteBatch* batch, int n); 21 | 22 | // Return the seqeunce number for the start of this batch. 23 | static SequenceNumber Sequence(const WriteBatch* batch); 24 | 25 | // Store the specified number as the seqeunce number for the start of 26 | // this batch. 27 | static void SetSequence(WriteBatch* batch, SequenceNumber seq); 28 | 29 | static Slice Contents(const WriteBatch* batch) { 30 | return Slice(batch->rep_); 31 | } 32 | 33 | static size_t ByteSize(const WriteBatch* batch) { 34 | return batch->rep_.size(); 35 | } 36 | 37 | static void SetContents(WriteBatch* batch, const Slice& contents); 38 | 39 | static Status InsertInto(const WriteBatch* batch, MemTable* memtable); 40 | }; 41 | 42 | } 43 | 44 | 45 | #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ 46 | -------------------------------------------------------------------------------- /leveldb/util/testharness.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testharness.h" 6 | 7 | #include 8 | #include 9 | 10 | namespace leveldb { 11 | namespace test { 12 | 13 | namespace { 14 | struct Test { 15 | const char* base; 16 | const char* name; 17 | void (*func)(); 18 | }; 19 | std::vector* tests; 20 | } 21 | 22 | bool RegisterTest(const char* base, const char* name, void (*func)()) { 23 | if (tests == NULL) { 24 | tests = new std::vector; 25 | } 26 | Test t; 27 | t.base = base; 28 | t.name = name; 29 | t.func = func; 30 | tests->push_back(t); 31 | return true; 32 | } 33 | 34 | int RunAllTests() { 35 | int num = 0; 36 | if (tests != NULL) { 37 | for (int i = 0; i < tests->size(); i++) { 38 | const Test& t = (*tests)[i]; 39 | fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); 40 | (*t.func)(); 41 | ++num; 42 | } 43 | } 44 | fprintf(stderr, "==== PASSED %d tests\n", num); 45 | return 0; 46 | } 47 | 48 | std::string TmpDir() { 49 | std::string dir; 50 | Status s = Env::Default()->GetTestDirectory(&dir); 51 | ASSERT_TRUE(s.ok()) << s.ToString(); 52 | return dir; 53 | } 54 | 55 | int RandomSeed() { 56 | const char* env = getenv("TEST_RANDOM_SEED"); 57 | int result = (env != NULL ? atoi(env) : 301); 58 | if (result <= 0) { 59 | result = 301; 60 | } 61 | return result; 62 | } 63 | 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /leveldb/util/testutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/testutil.h" 6 | 7 | #include "util/random.h" 8 | 9 | namespace leveldb { 10 | namespace test { 11 | 12 | Slice RandomString(Random* rnd, int len, std::string* dst) { 13 | dst->resize(len); 14 | for (int i = 0; i < len; i++) { 15 | (*dst)[i] = static_cast(' ' + rnd->Uniform(95)); // ' ' .. '~' 16 | } 17 | return Slice(*dst); 18 | } 19 | 20 | std::string RandomKey(Random* rnd, int len) { 21 | // Make sure to generate a wide variety of characters so we 22 | // test the boundary conditions for short-key optimizations. 23 | static const char kTestChars[] = { 24 | '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff' 25 | }; 26 | std::string result; 27 | for (int i = 0; i < len; i++) { 28 | result += kTestChars[rnd->Uniform(sizeof(kTestChars))]; 29 | } 30 | return result; 31 | } 32 | 33 | 34 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 35 | int len, std::string* dst) { 36 | int raw = static_cast(len * compressed_fraction); 37 | if (raw < 1) raw = 1; 38 | std::string raw_data; 39 | RandomString(rnd, raw, &raw_data); 40 | 41 | // Duplicate the random data until we have filled "len" bytes 42 | dst->clear(); 43 | while (dst->size() < len) { 44 | dst->append(raw_data); 45 | } 46 | dst->resize(len); 47 | return Slice(*dst); 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /leveldb/db/table_cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Thread-safe (provides internal synchronization) 6 | 7 | #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 8 | #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 9 | 10 | #include 11 | #include 12 | #include "db/dbformat.h" 13 | #include "leveldb/cache.h" 14 | #include "leveldb/table.h" 15 | #include "port/port.h" 16 | 17 | namespace leveldb { 18 | 19 | class Env; 20 | 21 | class TableCache { 22 | public: 23 | TableCache(const std::string& dbname, const Options* options, int entries); 24 | ~TableCache(); 25 | 26 | // Return an iterator for the specified file number (the corresponding 27 | // file length must be exactly "file_size" bytes). If "tableptr" is 28 | // non-NULL, also sets "*tableptr" to point to the Table object 29 | // underlying the returned iterator, or NULL if no Table object underlies 30 | // the returned iterator. The returned "*tableptr" object is owned by 31 | // the cache and should not be deleted, and is valid for as long as the 32 | // returned iterator is live. 33 | Iterator* NewIterator(const ReadOptions& options, 34 | uint64_t file_number, 35 | uint64_t file_size, 36 | Table** tableptr = NULL); 37 | 38 | // Evict any entry for the specified file number 39 | void Evict(uint64_t file_number); 40 | 41 | private: 42 | Env* const env_; 43 | const std::string dbname_; 44 | const Options* options_; 45 | Cache* cache_; 46 | }; 47 | 48 | } 49 | 50 | #endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_ 51 | -------------------------------------------------------------------------------- /leveldb/util/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Must not be included from any .h files to avoid polluting the namespace 6 | // with macros. 7 | 8 | #ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_ 9 | #define STORAGE_LEVELDB_UTIL_LOGGING_H_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Slice; 19 | class WritableFile; 20 | 21 | // Append a human-readable printout of "num" to *str 22 | extern void AppendNumberTo(std::string* str, uint64_t num); 23 | 24 | // Append a human-readable printout of "value" to *str. 25 | // Escapes any non-printable characters found in "value". 26 | extern void AppendEscapedStringTo(std::string* str, const Slice& value); 27 | 28 | // Return a human-readable printout of "num" 29 | extern std::string NumberToString(uint64_t num); 30 | 31 | // Return a human-readable version of "value". 32 | // Escapes any non-printable characters found in "value". 33 | extern std::string EscapeString(const Slice& value); 34 | 35 | // If *in starts with "c", advances *in past the first character and 36 | // returns true. Otherwise, returns false. 37 | extern bool ConsumeChar(Slice* in, char c); 38 | 39 | // Parse a human-readable number from "*in" into *value. On success, 40 | // advances "*in" past the consumed number and sets "*val" to the 41 | // numeric value. Otherwise, returns false and leaves *in in an 42 | // unspecified state. 43 | extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val); 44 | 45 | } 46 | 47 | #endif // STORAGE_LEVELDB_UTIL_LOGGING_H_ 48 | -------------------------------------------------------------------------------- /leveldb/port/port_android.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_android.h" 6 | 7 | #include 8 | 9 | extern "C" { 10 | size_t fread_unlocked(void *a, size_t b, size_t c, FILE *d) { 11 | return fread(a, b, c, d); 12 | } 13 | 14 | size_t fwrite_unlocked(const void *a, size_t b, size_t c, FILE *d) { 15 | return fwrite(a, b, c, d); 16 | } 17 | 18 | int fflush_unlocked(FILE *f) { 19 | return fflush(f); 20 | } 21 | 22 | int fdatasync(int fd) { 23 | return fsync(fd); 24 | } 25 | } 26 | 27 | namespace leveldb { 28 | namespace port { 29 | 30 | static void PthreadCall(const char* label, int result) { 31 | if (result != 0) { 32 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 33 | abort(); 34 | } 35 | } 36 | 37 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 38 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 39 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 40 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 41 | 42 | CondVar::CondVar(Mutex* mu) 43 | : mu_(mu) { 44 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 45 | } 46 | 47 | CondVar::~CondVar() { 48 | PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); 49 | } 50 | 51 | void CondVar::Wait() { 52 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 53 | } 54 | 55 | void CondVar::Signal(){ 56 | PthreadCall("signal", pthread_cond_signal(&cv_)); 57 | } 58 | 59 | void CondVar::SignalAll() { 60 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 61 | } 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /leveldb/port/port_chromium.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_chromium.h" 6 | 7 | #include "util/logging.h" 8 | 9 | #if defined(USE_SNAPPY) 10 | # include "third_party/snappy/src/snappy.h" 11 | #endif 12 | 13 | namespace leveldb { 14 | namespace port { 15 | 16 | Mutex::Mutex() { 17 | } 18 | 19 | Mutex::~Mutex() { 20 | } 21 | 22 | void Mutex::Lock() { 23 | mu_.Acquire(); 24 | } 25 | 26 | void Mutex::Unlock() { 27 | mu_.Release(); 28 | } 29 | 30 | void Mutex::AssertHeld() { 31 | mu_.AssertAcquired(); 32 | } 33 | 34 | CondVar::CondVar(Mutex* mu) 35 | : cv_(&mu->mu_) { 36 | } 37 | 38 | CondVar::~CondVar() { } 39 | 40 | void CondVar::Wait() { 41 | cv_.Wait(); 42 | } 43 | 44 | void CondVar::Signal(){ 45 | cv_.Signal(); 46 | } 47 | 48 | void CondVar::SignalAll() { 49 | cv_.Broadcast(); 50 | } 51 | 52 | bool Snappy_Compress(const char* input, size_t input_length, 53 | std::string* output) { 54 | #if defined(USE_SNAPPY) 55 | output->resize(snappy::MaxCompressedLength(input_length)); 56 | size_t outlen; 57 | snappy::RawCompress(input, input_length, &(*output)[0], &outlen); 58 | output->resize(outlen); 59 | return true; 60 | #else 61 | return false; 62 | #endif 63 | } 64 | 65 | bool Snappy_Uncompress(const char* input_data, size_t input_length, 66 | std::string* output) { 67 | #if defined(USE_SNAPPY) 68 | size_t ulength; 69 | if (!snappy::GetUncompressedLength(input_data, input_length, &ulength)) { 70 | return false; 71 | } 72 | output->resize(ulength); 73 | return snappy::RawUncompress(input_data, input_length, &(*output)[0]); 74 | #else 75 | return false; 76 | #endif 77 | } 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /leveldb/README: -------------------------------------------------------------------------------- 1 | leveldb: A key-value store 2 | Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) 3 | 4 | The code under this directory implements a system for maintaining a 5 | persistent key/value store. 6 | 7 | See doc/index.html for more explanation. 8 | See doc/impl.html for a brief overview of the implementation. 9 | 10 | The public interface is in include/*.h. Callers should not include or 11 | rely on the details of any other header files in this package. Those 12 | internal APIs may be changed without warning. 13 | 14 | Guide to header files: 15 | 16 | include/db.h 17 | Main interface to the DB: Start here 18 | 19 | include/options.h 20 | Control over the behavior of an entire database, and also 21 | control over the behavior of individual reads and writes. 22 | 23 | include/comparator.h 24 | Abstraction for user-specified comparison function. If you want 25 | just bytewise comparison of keys, you can use the default comparator, 26 | but clients can write their own comparator implementations if they 27 | want custom ordering (e.g. to handle different character 28 | encodings, etc.) 29 | 30 | include/iterator.h 31 | Interface for iterating over data. You can get an iterator 32 | from a DB object. 33 | 34 | include/write_batch.h 35 | Interface for atomically applying multiple updates to a database. 36 | 37 | include/slice.h 38 | A simple module for maintaining a pointer and a length into some 39 | other byte array. 40 | 41 | include/status.h 42 | Status is returned from many of the public interfaces and is used 43 | to report success and various kinds of errors. 44 | 45 | include/env.h 46 | Abstraction of the OS environment. A posix implementation of 47 | this interface is in util/env_posix.cc 48 | 49 | include/table.h 50 | include/table_builder.h 51 | Lower-level modules that most clients probably won't use directly 52 | -------------------------------------------------------------------------------- /leveldb/util/env.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/env.h" 6 | 7 | namespace leveldb { 8 | 9 | Env::~Env() { 10 | } 11 | 12 | SequentialFile::~SequentialFile() { 13 | } 14 | 15 | RandomAccessFile::~RandomAccessFile() { 16 | } 17 | 18 | WritableFile::~WritableFile() { 19 | } 20 | 21 | FileLock::~FileLock() { 22 | } 23 | 24 | void Log(Env* env, WritableFile* info_log, const char* format, ...) { 25 | va_list ap; 26 | va_start(ap, format); 27 | env->Logv(info_log, format, ap); 28 | va_end(ap); 29 | } 30 | 31 | Status WriteStringToFile(Env* env, const Slice& data, 32 | const std::string& fname) { 33 | WritableFile* file; 34 | Status s = env->NewWritableFile(fname, &file); 35 | if (!s.ok()) { 36 | return s; 37 | } 38 | s = file->Append(data); 39 | if (s.ok()) { 40 | s = file->Close(); 41 | } 42 | delete file; // Will auto-close if we did not close above 43 | if (!s.ok()) { 44 | env->DeleteFile(fname); 45 | } 46 | return s; 47 | } 48 | 49 | Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { 50 | data->clear(); 51 | SequentialFile* file; 52 | Status s = env->NewSequentialFile(fname, &file); 53 | if (!s.ok()) { 54 | return s; 55 | } 56 | static const int kBufferSize = 8192; 57 | char* space = new char[kBufferSize]; 58 | while (true) { 59 | Slice fragment; 60 | s = file->Read(kBufferSize, &fragment, space); 61 | if (!s.ok()) { 62 | break; 63 | } 64 | data->append(fragment.data(), fragment.size()); 65 | if (fragment.empty()) { 66 | break; 67 | } 68 | } 69 | delete[] space; 70 | delete file; 71 | return s; 72 | } 73 | 74 | EnvWrapper::~EnvWrapper() { 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /leveldb/db/snapshot.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_ 6 | #define STORAGE_LEVELDB_DB_SNAPSHOT_H_ 7 | 8 | #include "leveldb/db.h" 9 | 10 | namespace leveldb { 11 | 12 | class SnapshotList; 13 | 14 | // Snapshots are kept in a doubly-linked list in the DB. 15 | // Each SnapshotImpl corresponds to a particular sequence number. 16 | class SnapshotImpl : public Snapshot { 17 | public: 18 | SequenceNumber number_; // const after creation 19 | 20 | private: 21 | friend class SnapshotList; 22 | 23 | // SnapshotImpl is kept in a doubly-linked circular list 24 | SnapshotImpl* prev_; 25 | SnapshotImpl* next_; 26 | 27 | SnapshotList* list_; // just for sanity checks 28 | }; 29 | 30 | class SnapshotList { 31 | public: 32 | SnapshotList() { 33 | list_.prev_ = &list_; 34 | list_.next_ = &list_; 35 | } 36 | 37 | bool empty() const { return list_.next_ == &list_; } 38 | SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; } 39 | SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; } 40 | 41 | const SnapshotImpl* New(SequenceNumber seq) { 42 | SnapshotImpl* s = new SnapshotImpl; 43 | s->number_ = seq; 44 | s->list_ = this; 45 | s->next_ = &list_; 46 | s->prev_ = list_.prev_; 47 | s->prev_->next_ = s; 48 | s->next_->prev_ = s; 49 | return s; 50 | } 51 | 52 | void Delete(const SnapshotImpl* s) { 53 | assert(s->list_ == this); 54 | s->prev_->next_ = s->next_; 55 | s->next_->prev_ = s->prev_; 56 | delete s; 57 | } 58 | 59 | private: 60 | // Dummy head of doubly-linked list of snapshots 61 | SnapshotImpl list_; 62 | }; 63 | 64 | } 65 | 66 | #endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_ 67 | -------------------------------------------------------------------------------- /leveldb/util/testutil.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 6 | #define STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 7 | 8 | #include "leveldb/env.h" 9 | #include "leveldb/slice.h" 10 | #include "util/random.h" 11 | 12 | namespace leveldb { 13 | namespace test { 14 | 15 | // Store in *dst a random string of length "len" and return a Slice that 16 | // references the generated data. 17 | extern Slice RandomString(Random* rnd, int len, std::string* dst); 18 | 19 | // Return a random key with the specified length that may contain interesting 20 | // characters (e.g. \x00, \xff, etc.). 21 | extern std::string RandomKey(Random* rnd, int len); 22 | 23 | // Store in *dst a string of length "len" that will compress to 24 | // "N*compressed_fraction" bytes and return a Slice that references 25 | // the generated data. 26 | extern Slice CompressibleString(Random* rnd, double compressed_fraction, 27 | int len, std::string* dst); 28 | 29 | // A wrapper that allows injection of errors. 30 | class ErrorEnv : public EnvWrapper { 31 | public: 32 | bool writable_file_error_; 33 | int num_writable_file_errors_; 34 | 35 | ErrorEnv() : EnvWrapper(Env::Default()), 36 | writable_file_error_(false), 37 | num_writable_file_errors_(0) { } 38 | 39 | virtual Status NewWritableFile(const std::string& fname, 40 | WritableFile** result) { 41 | if (writable_file_error_) { 42 | ++num_writable_file_errors_; 43 | *result = NULL; 44 | return Status::IOError(fname, "fake error"); 45 | } 46 | return target()->NewWritableFile(fname, result); 47 | } 48 | }; 49 | 50 | } 51 | } 52 | 53 | #endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_ 54 | -------------------------------------------------------------------------------- /leveldb/util/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | 7 | #include "util/random.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | class ArenaTest { }; 13 | 14 | TEST(ArenaTest, Empty) { 15 | Arena arena; 16 | } 17 | 18 | TEST(ArenaTest, Simple) { 19 | std::vector > allocated; 20 | Arena arena; 21 | const int N = 100000; 22 | size_t bytes = 0; 23 | Random rnd(301); 24 | for (int i = 0; i < N; i++) { 25 | size_t s; 26 | if (i % (N / 10) == 0) { 27 | s = i; 28 | } else { 29 | s = rnd.OneIn(4000) ? rnd.Uniform(6000) : 30 | (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); 31 | } 32 | if (s == 0) { 33 | // Our arena disallows size 0 allocations. 34 | s = 1; 35 | } 36 | char* r; 37 | if (rnd.OneIn(10)) { 38 | r = arena.AllocateAligned(s); 39 | } else { 40 | r = arena.Allocate(s); 41 | } 42 | 43 | for (int b = 0; b < s; b++) { 44 | // Fill the "i"th allocation with a known bit pattern 45 | r[b] = i % 256; 46 | } 47 | bytes += s; 48 | allocated.push_back(std::make_pair(s, r)); 49 | ASSERT_GE(arena.MemoryUsage(), bytes); 50 | if (i > N/10) { 51 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 52 | } 53 | } 54 | for (int i = 0; i < allocated.size(); i++) { 55 | size_t num_bytes = allocated[i].first; 56 | const char* p = allocated[i].second; 57 | for (int b = 0; b < num_bytes; b++) { 58 | // Check the "i"th allocation for the known bit pattern 59 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 60 | } 61 | } 62 | } 63 | 64 | } 65 | 66 | int main(int argc, char** argv) { 67 | return leveldb::test::RunAllTests(); 68 | } 69 | -------------------------------------------------------------------------------- /leveldb/table/iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/iterator.h" 6 | 7 | namespace leveldb { 8 | 9 | Iterator::Iterator() { 10 | cleanup_.function = NULL; 11 | cleanup_.next = NULL; 12 | } 13 | 14 | Iterator::~Iterator() { 15 | if (cleanup_.function != NULL) { 16 | (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2); 17 | for (Cleanup* c = cleanup_.next; c != NULL; ) { 18 | (*c->function)(c->arg1, c->arg2); 19 | Cleanup* next = c->next; 20 | delete c; 21 | c = next; 22 | } 23 | } 24 | } 25 | 26 | void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) { 27 | assert(func != NULL); 28 | Cleanup* c; 29 | if (cleanup_.function == NULL) { 30 | c = &cleanup_; 31 | } else { 32 | c = new Cleanup; 33 | c->next = cleanup_.next; 34 | cleanup_.next = c; 35 | } 36 | c->function = func; 37 | c->arg1 = arg1; 38 | c->arg2 = arg2; 39 | } 40 | 41 | namespace { 42 | class EmptyIterator : public Iterator { 43 | public: 44 | EmptyIterator(const Status& s) : status_(s) { } 45 | virtual bool Valid() const { return false; } 46 | virtual void Seek(const Slice& target) { } 47 | virtual void SeekToFirst() { } 48 | virtual void SeekToLast() { } 49 | virtual void Next() { assert(false); } 50 | virtual void Prev() { assert(false); } 51 | Slice key() const { assert(false); return Slice(); } 52 | Slice value() const { assert(false); return Slice(); } 53 | virtual Status status() const { return status_; } 54 | private: 55 | Status status_; 56 | }; 57 | } 58 | 59 | Iterator* NewEmptyIterator() { 60 | return new EmptyIterator(Status::OK()); 61 | } 62 | 63 | Iterator* NewErrorIterator(const Status& status) { 64 | return new EmptyIterator(status); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /leveldb/table/block_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 6 | #define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 7 | 8 | #include 9 | 10 | #include 11 | #include "leveldb/slice.h" 12 | 13 | namespace leveldb { 14 | 15 | struct Options; 16 | 17 | class BlockBuilder { 18 | public: 19 | explicit BlockBuilder(const Options* options); 20 | 21 | // Reset the contents as if the BlockBuilder was just constructed. 22 | void Reset(); 23 | 24 | // REQUIRES: Finish() has not been callled since the last call to Reset(). 25 | // REQUIRES: key is larger than any previously added key 26 | void Add(const Slice& key, const Slice& value); 27 | 28 | // Finish building the block and return a slice that refers to the 29 | // block contents. The returned slice will remain valid for the 30 | // lifetime of this builder or until Reset() is called. 31 | Slice Finish(); 32 | 33 | // Returns an estimate of the current (uncompressed) size of the block 34 | // we are building. 35 | size_t CurrentSizeEstimate() const; 36 | 37 | // Return true iff no entries have been added since the last Reset() 38 | bool empty() const { 39 | return buffer_.empty(); 40 | } 41 | 42 | private: 43 | const Options* options_; 44 | std::string buffer_; // Destination buffer 45 | std::vector restarts_; // Restart points 46 | int counter_; // Number of entries emitted since restart 47 | bool finished_; // Has Finish() been called? 48 | std::string last_key_; 49 | 50 | // No copying allowed 51 | BlockBuilder(const BlockBuilder&); 52 | void operator=(const BlockBuilder&); 53 | }; 54 | 55 | } 56 | 57 | #endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_ 58 | -------------------------------------------------------------------------------- /leveldb/util/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/crc32c.h" 6 | #include "util/testharness.h" 7 | 8 | namespace leveldb { 9 | namespace crc32c { 10 | 11 | class CRC { }; 12 | 13 | TEST(CRC, StandardResults) { 14 | // From rfc3720 section B.4. 15 | char buf[32]; 16 | 17 | memset(buf, 0, sizeof(buf)); 18 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 19 | 20 | memset(buf, 0xff, sizeof(buf)); 21 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 22 | 23 | for (int i = 0; i < 32; i++) { 24 | buf[i] = i; 25 | } 26 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 27 | 28 | for (int i = 0; i < 32; i++) { 29 | buf[i] = 31 - i; 30 | } 31 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 32 | 33 | unsigned char data[48] = { 34 | 0x01, 0xc0, 0x00, 0x00, 35 | 0x00, 0x00, 0x00, 0x00, 36 | 0x00, 0x00, 0x00, 0x00, 37 | 0x00, 0x00, 0x00, 0x00, 38 | 0x14, 0x00, 0x00, 0x00, 39 | 0x00, 0x00, 0x04, 0x00, 40 | 0x00, 0x00, 0x00, 0x14, 41 | 0x00, 0x00, 0x00, 0x18, 42 | 0x28, 0x00, 0x00, 0x00, 43 | 0x00, 0x00, 0x00, 0x00, 44 | 0x02, 0x00, 0x00, 0x00, 45 | 0x00, 0x00, 0x00, 0x00, 46 | }; 47 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 48 | } 49 | 50 | TEST(CRC, Values) { 51 | ASSERT_NE(Value("a", 1), Value("foo", 3)); 52 | } 53 | 54 | TEST(CRC, Extend) { 55 | ASSERT_EQ(Value("hello world", 11), 56 | Extend(Value("hello ", 6), "world", 5)); 57 | } 58 | 59 | TEST(CRC, Mask) { 60 | uint32_t crc = Value("foo", 3); 61 | ASSERT_NE(crc, Mask(crc)); 62 | ASSERT_NE(crc, Mask(Mask(crc))); 63 | ASSERT_EQ(crc, Unmask(Mask(crc))); 64 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 65 | } 66 | 67 | } 68 | } 69 | 70 | int main(int argc, char** argv) { 71 | return leveldb::test::RunAllTests(); 72 | } 73 | -------------------------------------------------------------------------------- /leveldb/util/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_ARENA_H_ 6 | #define STORAGE_LEVELDB_UTIL_ARENA_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace leveldb { 14 | 15 | class Arena { 16 | public: 17 | Arena(); 18 | ~Arena(); 19 | 20 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 21 | char* Allocate(size_t bytes); 22 | 23 | // Allocate memory with the normal alignment guarantees provided by malloc 24 | char* AllocateAligned(size_t bytes); 25 | 26 | // Returns an estimate of the total memory usage of data allocated 27 | // by the arena (including space allocated but not yet used for user 28 | // allocations). 29 | size_t MemoryUsage() const { 30 | return blocks_memory_ + blocks_.capacity() * sizeof(char*); 31 | } 32 | 33 | private: 34 | char* AllocateFallback(size_t bytes); 35 | char* AllocateNewBlock(size_t block_bytes); 36 | 37 | // Allocation state 38 | char* alloc_ptr_; 39 | size_t alloc_bytes_remaining_; 40 | 41 | // Array of new[] allocated memory blocks 42 | std::vector blocks_; 43 | 44 | // Bytes of memory in blocks allocated so far 45 | size_t blocks_memory_; 46 | 47 | // No copying allowed 48 | Arena(const Arena&); 49 | void operator=(const Arena&); 50 | }; 51 | 52 | inline char* Arena::Allocate(size_t bytes) { 53 | // The semantics of what to return are a bit messy if we allow 54 | // 0-byte allocations, so we disallow them here (we don't need 55 | // them for our internal use). 56 | assert(bytes > 0); 57 | if (bytes <= alloc_bytes_remaining_) { 58 | char* result = alloc_ptr_; 59 | alloc_ptr_ += bytes; 60 | alloc_bytes_remaining_ -= bytes; 61 | return result; 62 | } 63 | return AllocateFallback(bytes); 64 | } 65 | 66 | } 67 | 68 | #endif // STORAGE_LEVELDB_UTIL_ARENA_H_ 69 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/write_batch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch holds a collection of updates to apply atomically to a DB. 6 | // 7 | // The updates are applied in the order in which they are added 8 | // to the WriteBatch. For example, the value of "key" will be "v3" 9 | // after the following batch is written: 10 | // 11 | // batch.Put("key", "v1"); 12 | // batch.Delete("key"); 13 | // batch.Put("key", "v2"); 14 | // batch.Put("key", "v3"); 15 | // 16 | // Multiple threads can invoke const methods on a WriteBatch without 17 | // external synchronization, but if any of the threads may call a 18 | // non-const method, all threads accessing the same WriteBatch must use 19 | // external synchronization. 20 | 21 | #ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 22 | #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 23 | 24 | #include 25 | #include "leveldb/status.h" 26 | 27 | namespace leveldb { 28 | 29 | class Slice; 30 | 31 | class WriteBatch { 32 | public: 33 | WriteBatch(); 34 | ~WriteBatch(); 35 | 36 | // Store the mapping "key->value" in the database. 37 | void Put(const Slice& key, const Slice& value); 38 | 39 | // If the database contains a mapping for "key", erase it. Else do nothing. 40 | void Delete(const Slice& key); 41 | 42 | // Clear all updates buffered in this batch. 43 | void Clear(); 44 | 45 | // Support for iterating over the contents of a batch. 46 | class Handler { 47 | public: 48 | virtual ~Handler(); 49 | virtual void Put(const Slice& key, const Slice& value) = 0; 50 | virtual void Delete(const Slice& key) = 0; 51 | }; 52 | Status Iterate(Handler* handler) const; 53 | 54 | private: 55 | friend class WriteBatchInternal; 56 | 57 | std::string rep_; // See comment in write_batch.cc for the format of rep_ 58 | 59 | // Intentionally copyable 60 | }; 61 | 62 | } 63 | 64 | #endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ 65 | -------------------------------------------------------------------------------- /leveldb/util/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "leveldb/env.h" 12 | #include "leveldb/slice.h" 13 | 14 | namespace leveldb { 15 | 16 | void AppendNumberTo(std::string* str, uint64_t num) { 17 | char buf[30]; 18 | snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num); 19 | str->append(buf); 20 | } 21 | 22 | void AppendEscapedStringTo(std::string* str, const Slice& value) { 23 | for (size_t i = 0; i < value.size(); i++) { 24 | char c = value[i]; 25 | if (c >= ' ' && c <= '~') { 26 | str->push_back(c); 27 | } else { 28 | char buf[10]; 29 | snprintf(buf, sizeof(buf), "\\x%02x", 30 | static_cast(c) & 0xff); 31 | str->append(buf); 32 | } 33 | } 34 | } 35 | 36 | std::string NumberToString(uint64_t num) { 37 | std::string r; 38 | AppendNumberTo(&r, num); 39 | return r; 40 | } 41 | 42 | std::string EscapeString(const Slice& value) { 43 | std::string r; 44 | AppendEscapedStringTo(&r, value); 45 | return r; 46 | } 47 | 48 | bool ConsumeChar(Slice* in, char c) { 49 | if (!in->empty() && (*in)[0] == c) { 50 | in->remove_prefix(1); 51 | return true; 52 | } else { 53 | return false; 54 | } 55 | } 56 | 57 | bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { 58 | uint64_t v = 0; 59 | int digits = 0; 60 | while (!in->empty()) { 61 | char c = (*in)[0]; 62 | if (c >= '0' && c <= '9') { 63 | ++digits; 64 | const int delta = (c - '0'); 65 | static const uint64_t kMaxUint64 = ~static_cast(0); 66 | if (v > kMaxUint64/10 || 67 | (v == kMaxUint64/10 && delta > kMaxUint64%10)) { 68 | // Overflow 69 | return false; 70 | } 71 | v = (v * 10) + delta; 72 | in->remove_prefix(1); 73 | } else { 74 | break; 75 | } 76 | } 77 | *val = v; 78 | return (digits > 0); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /leveldb/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "port/port.h" 7 | #include "leveldb/status.h" 8 | 9 | namespace leveldb { 10 | 11 | const char* Status::CopyState(const char* state) { 12 | uint32_t size; 13 | memcpy(&size, state, sizeof(size)); 14 | char* result = new char[size + 5]; 15 | memcpy(result, state, size + 5); 16 | return result; 17 | } 18 | 19 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 20 | assert(code != kOk); 21 | const uint32_t len1 = msg.size(); 22 | const uint32_t len2 = msg2.size(); 23 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 24 | char* result = new char[size + 5]; 25 | memcpy(result, &size, sizeof(size)); 26 | result[4] = static_cast(code); 27 | memcpy(result + 5, msg.data(), len1); 28 | if (len2) { 29 | result[5 + len1] = ':'; 30 | result[6 + len1] = ' '; 31 | memcpy(result + 7 + len1, msg2.data(), len2); 32 | } 33 | state_ = result; 34 | } 35 | 36 | std::string Status::ToString() const { 37 | if (state_ == NULL) { 38 | return "OK"; 39 | } else { 40 | char tmp[30]; 41 | const char* type; 42 | switch (code()) { 43 | case kOk: 44 | type = "OK"; 45 | break; 46 | case kNotFound: 47 | type = "NotFound: "; 48 | break; 49 | case kCorruption: 50 | type = "Corruption: "; 51 | break; 52 | case kNotSupported: 53 | type = "Not implemented: "; 54 | break; 55 | case kInvalidArgument: 56 | type = "Invalid argument: "; 57 | break; 58 | case kIOError: 59 | type = "IO error: "; 60 | break; 61 | default: 62 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 63 | static_cast(code())); 64 | type = tmp; 65 | break; 66 | } 67 | std::string result(type); 68 | uint32_t length; 69 | memcpy(&length, state_, sizeof(length)); 70 | result.append(state_ + 5, length); 71 | return result; 72 | } 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /leveldb/util/arena.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/arena.h" 6 | #include 7 | 8 | namespace leveldb { 9 | 10 | static const int kBlockSize = 4096; 11 | 12 | Arena::Arena() { 13 | blocks_memory_ = 0; 14 | alloc_ptr_ = NULL; // First allocation will allocate a block 15 | alloc_bytes_remaining_ = 0; 16 | } 17 | 18 | Arena::~Arena() { 19 | for (size_t i = 0; i < blocks_.size(); i++) { 20 | delete[] blocks_[i]; 21 | } 22 | } 23 | 24 | char* Arena::AllocateFallback(size_t bytes) { 25 | if (bytes > kBlockSize / 4) { 26 | // Object is more than a quarter of our block size. Allocate it separately 27 | // to avoid wasting too much space in leftover bytes. 28 | char* result = AllocateNewBlock(bytes); 29 | return result; 30 | } 31 | 32 | // We waste the remaining space in the current block. 33 | alloc_ptr_ = AllocateNewBlock(kBlockSize); 34 | alloc_bytes_remaining_ = kBlockSize; 35 | 36 | char* result = alloc_ptr_; 37 | alloc_ptr_ += bytes; 38 | alloc_bytes_remaining_ -= bytes; 39 | return result; 40 | } 41 | 42 | char* Arena::AllocateAligned(size_t bytes) { 43 | const int align = sizeof(void*); // We'll align to pointer size 44 | assert((align & (align-1)) == 0); // Pointer size should be a power of 2 45 | size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); 46 | size_t slop = (current_mod == 0 ? 0 : align - current_mod); 47 | size_t needed = bytes + slop; 48 | char* result; 49 | if (needed <= alloc_bytes_remaining_) { 50 | result = alloc_ptr_ + slop; 51 | alloc_ptr_ += needed; 52 | alloc_bytes_remaining_ -= needed; 53 | } else { 54 | // AllocateFallback always returned aligned memory 55 | result = AllocateFallback(bytes); 56 | } 57 | assert((reinterpret_cast(result) & (align-1)) == 0); 58 | return result; 59 | } 60 | 61 | char* Arena::AllocateNewBlock(size_t block_bytes) { 62 | char* result = new char[block_bytes]; 63 | blocks_memory_ += block_bytes; 64 | blocks_.push_back(result); 65 | return result; 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /leveldb/util/comparator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "leveldb/comparator.h" 7 | #include "leveldb/slice.h" 8 | #include "util/logging.h" 9 | 10 | namespace leveldb { 11 | 12 | Comparator::~Comparator() { } 13 | 14 | namespace { 15 | class BytewiseComparatorImpl : public Comparator { 16 | public: 17 | BytewiseComparatorImpl() { } 18 | 19 | virtual const char* Name() const { 20 | return "leveldb.BytewiseComparator"; 21 | } 22 | 23 | virtual int Compare(const Slice& a, const Slice& b) const { 24 | return a.compare(b); 25 | } 26 | 27 | virtual void FindShortestSeparator( 28 | std::string* start, 29 | const Slice& limit) const { 30 | // Find length of common prefix 31 | size_t min_length = std::min(start->size(), limit.size()); 32 | size_t diff_index = 0; 33 | while ((diff_index < min_length) && 34 | ((*start)[diff_index] == limit[diff_index])) { 35 | diff_index++; 36 | } 37 | 38 | if (diff_index >= min_length) { 39 | // Do not shorten if one string is a prefix of the other 40 | } else { 41 | uint8_t diff_byte = static_cast((*start)[diff_index]); 42 | if (diff_byte < static_cast(0xff) && 43 | diff_byte + 1 < static_cast(limit[diff_index])) { 44 | (*start)[diff_index]++; 45 | start->resize(diff_index + 1); 46 | assert(Compare(*start, limit) < 0); 47 | } 48 | } 49 | } 50 | 51 | virtual void FindShortSuccessor(std::string* key) const { 52 | // Find first character that can be incremented 53 | size_t n = key->size(); 54 | for (size_t i = 0; i < n; i++) { 55 | const uint8_t byte = (*key)[i]; 56 | if (byte != static_cast(0xff)) { 57 | (*key)[i] = byte + 1; 58 | key->resize(i+1); 59 | return; 60 | } 61 | } 62 | // *key is a run of 0xffs. Leave it alone. 63 | } 64 | }; 65 | } 66 | static const BytewiseComparatorImpl bytewise; 67 | 68 | const Comparator* BytewiseComparator() { 69 | return &bytewise; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /leveldb/util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ 6 | #define STORAGE_LEVELDB_UTIL_RANDOM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | // A very simple random number generator. Not especially good at 13 | // generating truly random bits, but good enough for our needs in this 14 | // package. 15 | class Random { 16 | private: 17 | uint32_t seed_; 18 | public: 19 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { } 20 | uint32_t Next() { 21 | static const uint32_t M = 2147483647L; // 2^31-1 22 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 23 | // We are computing 24 | // seed_ = (seed_ * A) % M, where M = 2^31-1 25 | // 26 | // seed_ must not be zero or M, or else all subsequent computed values 27 | // will be zero or M respectively. For all other values, seed_ will end 28 | // up cycling through every number in [1,M-1] 29 | uint64_t product = seed_ * A; 30 | 31 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 32 | seed_ = static_cast((product >> 31) + (product & M)); 33 | // The first reduction may overflow by 1 bit, so we may need to 34 | // repeat. mod == M is not possible; using > allows the faster 35 | // sign-bit-based test. 36 | if (seed_ > M) { 37 | seed_ -= M; 38 | } 39 | return seed_; 40 | } 41 | // Returns a uniformly distributed value in the range [0..n-1] 42 | // REQUIRES: n > 0 43 | uint32_t Uniform(int n) { return Next() % n; } 44 | 45 | // Randomly returns true ~"1/n" of the time, and false otherwise. 46 | // REQUIRES: n > 0 47 | bool OneIn(int n) { return (Next() % n) == 0; } 48 | 49 | // Skewed: pick "base" uniformly from range [0,max_log] and then 50 | // return "base" random bits. The effect is to pick a number in the 51 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 52 | uint32_t Skewed(int max_log) { 53 | return Uniform(1 << Uniform(max_log + 1)); 54 | } 55 | }; 56 | 57 | } 58 | 59 | #endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ 60 | -------------------------------------------------------------------------------- /leveldb/table/iterator_wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 6 | #define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 7 | 8 | namespace leveldb { 9 | 10 | // A internal wrapper class with an interface similar to Iterator that 11 | // caches the valid() and key() results for an underlying iterator. 12 | // This can help avoid virtual function calls and also gives better 13 | // cache locality. 14 | class IteratorWrapper { 15 | public: 16 | IteratorWrapper(): iter_(NULL), valid_(false) { } 17 | explicit IteratorWrapper(Iterator* iter): iter_(NULL) { 18 | Set(iter); 19 | } 20 | ~IteratorWrapper() { delete iter_; } 21 | Iterator* iter() const { return iter_; } 22 | 23 | // Takes ownership of "iter" and will delete it when destroyed, or 24 | // when Set() is invoked again. 25 | void Set(Iterator* iter) { 26 | delete iter_; 27 | iter_ = iter; 28 | if (iter_ == NULL) { 29 | valid_ = false; 30 | } else { 31 | Update(); 32 | } 33 | } 34 | 35 | 36 | // Iterator interface methods 37 | bool Valid() const { return valid_; } 38 | Slice key() const { assert(Valid()); return key_; } 39 | Slice value() const { assert(Valid()); return iter_->value(); } 40 | // Methods below require iter() != NULL 41 | Status status() const { assert(iter_); return iter_->status(); } 42 | void Next() { assert(iter_); iter_->Next(); Update(); } 43 | void Prev() { assert(iter_); iter_->Prev(); Update(); } 44 | void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); } 45 | void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); } 46 | void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); } 47 | 48 | private: 49 | void Update() { 50 | valid_ = iter_->Valid(); 51 | if (valid_) { 52 | key_ = iter_->key(); 53 | } 54 | } 55 | 56 | Iterator* iter_; 57 | bool valid_; 58 | Slice key_; 59 | }; 60 | 61 | } // namespace leveldb 62 | 63 | #endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_ 64 | -------------------------------------------------------------------------------- /leveldb/doc/table_format.txt: -------------------------------------------------------------------------------- 1 | File format 2 | =========== 3 | 4 | 5 | [data block 1] 6 | [data block 2] 7 | ... 8 | [data block N] 9 | [meta block 1] 10 | ... 11 | [meta block K] 12 | [metaindex block] 13 | [index block] 14 | [Footer] (fixed size; starts at file_size - sizeof(Footer)) 15 | 16 | 17 | The file contains internal pointers. Each such pointer is called 18 | a BlockHandle and contains the following information: 19 | offset: varint64 20 | size: varint64 21 | 22 | (1) The sequence of key/value pairs in the file are stored in sorted 23 | order and partitioned into a sequence of data blocks. These blocks 24 | come one after another at the beginning of the file. Each data block 25 | is formatted according to the code in block_builder.cc, and then 26 | optionally compressed. 27 | 28 | (2) After the data blocks we store a bunch of meta blocks. The 29 | supported meta block types are described below. More meta block types 30 | may be added in the future. Each meta block is again formatted using 31 | block_builder.cc and then optionally compressed. 32 | 33 | (3) A "metaindex" block. It contains one entry for every other meta 34 | block where the key is the name of the meta block and the value is a 35 | BlockHandle pointing to that meta block. 36 | 37 | (4) An "index" block. This block contains one entry per data block, 38 | where the key is a string >= last key in that data block and before 39 | the first key in the successive data block. The value is the 40 | BlockHandle for the data block. 41 | 42 | (6) At the very end of the file is a fixed length footer that contains 43 | the BlockHandle of the metaindex and index blocks as well as a magic number. 44 | metaindex_handle: char[p]; // Block handle for metaindex 45 | index_handle: char[q]; // Block handle for index 46 | padding: char[40-p-q]; // 0 bytes to make fixed length 47 | // (40==2*BlockHandle::kMaxEncodedLength) 48 | magic: fixed64; // == 0xdb4775248b80fb57 49 | 50 | "stats" Meta Block 51 | ------------------ 52 | 53 | This meta block contains a bunch of stats. The key is the name 54 | of the statistic. The value contains the statistic. 55 | TODO(postrelease): record following stats. 56 | data size 57 | index size 58 | key size (uncompressed) 59 | value size (uncompressed) 60 | number of entries 61 | number of data blocks 62 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/comparator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | class Slice; 13 | 14 | // A Comparator object provides a total order across slices that are 15 | // used as keys in an sstable or a database. A Comparator implementation 16 | // must be thread-safe since leveldb may invoke its methods concurrently 17 | // from multiple threads. 18 | class Comparator { 19 | public: 20 | virtual ~Comparator(); 21 | 22 | // Three-way comparison. Returns value: 23 | // < 0 iff "a" < "b", 24 | // == 0 iff "a" == "b", 25 | // > 0 iff "a" > "b" 26 | virtual int Compare(const Slice& a, const Slice& b) const = 0; 27 | 28 | // The name of the comparator. Used to check for comparator 29 | // mismatches (i.e., a DB created with one comparator is 30 | // accessed using a different comparator. 31 | // 32 | // The client of this package should switch to a new name whenever 33 | // the comparator implementation changes in a way that will cause 34 | // the relative ordering of any two keys to change. 35 | // 36 | // Names starting with "leveldb." are reserved and should not be used 37 | // by any clients of this package. 38 | virtual const char* Name() const = 0; 39 | 40 | // Advanced functions: these are used to reduce the space requirements 41 | // for internal data structures like index blocks. 42 | 43 | // If *start < limit, changes *start to a short string in [start,limit). 44 | // Simple comparator implementations may return with *start unchanged, 45 | // i.e., an implementation of this method that does nothing is correct. 46 | virtual void FindShortestSeparator( 47 | std::string* start, 48 | const Slice& limit) const = 0; 49 | 50 | // Changes *key to a short string >= *key. 51 | // Simple comparator implementations may return with *key unchanged, 52 | // i.e., an implementation of this method that does nothing is correct. 53 | virtual void FindShortSuccessor(std::string* key) const = 0; 54 | }; 55 | 56 | // Return a builtin comparator that uses lexicographic byte-wise 57 | // ordering. The result remains the property of this module and 58 | // must not be deleted. 59 | extern const Comparator* BytewiseComparator(); 60 | 61 | } 62 | 63 | #endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ 64 | -------------------------------------------------------------------------------- /leveldb/db/builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/builder.h" 6 | 7 | #include "db/filename.h" 8 | #include "db/dbformat.h" 9 | #include "db/table_cache.h" 10 | #include "db/version_edit.h" 11 | #include "leveldb/db.h" 12 | #include "leveldb/env.h" 13 | #include "leveldb/iterator.h" 14 | 15 | namespace leveldb { 16 | 17 | Status BuildTable(const std::string& dbname, 18 | Env* env, 19 | const Options& options, 20 | TableCache* table_cache, 21 | Iterator* iter, 22 | FileMetaData* meta, 23 | VersionEdit* edit) { 24 | Status s; 25 | meta->file_size = 0; 26 | iter->SeekToFirst(); 27 | 28 | std::string fname = TableFileName(dbname, meta->number); 29 | if (iter->Valid()) { 30 | WritableFile* file; 31 | s = env->NewWritableFile(fname, &file); 32 | if (!s.ok()) { 33 | return s; 34 | } 35 | 36 | TableBuilder* builder = new TableBuilder(options, file); 37 | meta->smallest.DecodeFrom(iter->key()); 38 | for (; iter->Valid(); iter->Next()) { 39 | Slice key = iter->key(); 40 | meta->largest.DecodeFrom(key); 41 | builder->Add(key, iter->value()); 42 | } 43 | 44 | // Finish and check for builder errors 45 | if (s.ok()) { 46 | s = builder->Finish(); 47 | if (s.ok()) { 48 | meta->file_size = builder->FileSize(); 49 | assert(meta->file_size > 0); 50 | } 51 | } else { 52 | builder->Abandon(); 53 | } 54 | delete builder; 55 | 56 | // Finish and check for file errors 57 | if (s.ok()) { 58 | s = file->Sync(); 59 | } 60 | if (s.ok()) { 61 | s = file->Close(); 62 | } 63 | delete file; 64 | file = NULL; 65 | 66 | if (s.ok()) { 67 | // Verify that the table is usable 68 | Iterator* it = table_cache->NewIterator(ReadOptions(), 69 | meta->number, 70 | meta->file_size); 71 | s = it->status(); 72 | delete it; 73 | } 74 | } 75 | 76 | // Check for input iterator errors 77 | if (!iter->status().ok()) { 78 | s = iter->status(); 79 | } 80 | 81 | if (s.ok() && meta->file_size > 0) { 82 | edit->AddFile(0, meta->number, meta->file_size, 83 | meta->smallest, meta->largest); 84 | } else { 85 | env->DeleteFile(fname); 86 | } 87 | return s; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/table.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_ 6 | #define STORAGE_LEVELDB_INCLUDE_TABLE_H_ 7 | 8 | #include 9 | #include "leveldb/iterator.h" 10 | 11 | namespace leveldb { 12 | 13 | class Block; 14 | class BlockHandle; 15 | struct Options; 16 | class RandomAccessFile; 17 | struct ReadOptions; 18 | 19 | // A Table is a sorted map from strings to strings. Tables are 20 | // immutable and persistent. A Table may be safely accessed from 21 | // multiple threads without external synchronization. 22 | class Table { 23 | public: 24 | // Attempt to open the table that is stored in bytes [0..file_size) 25 | // of "file", and read the metadata entries necessary to allow 26 | // retrieving data from the table. 27 | // 28 | // If successful, returns ok and sets "*table" to the newly opened 29 | // table. The client should delete "*table" when no longer needed. 30 | // If there was an error while initializing the table, sets "*table" 31 | // to NULL and returns a non-ok status. Does not take ownership of 32 | // "*source", but the client must ensure that "source" remains live 33 | // for the duration of the returned table's lifetime. 34 | // 35 | // *file must remain live while this Table is in use. 36 | static Status Open(const Options& options, 37 | RandomAccessFile* file, 38 | uint64_t file_size, 39 | Table** table); 40 | 41 | ~Table(); 42 | 43 | // Returns a new iterator over the table contents. 44 | // The result of NewIterator() is initially invalid (caller must 45 | // call one of the Seek methods on the iterator before using it). 46 | Iterator* NewIterator(const ReadOptions&) const; 47 | 48 | // Given a key, return an approximate byte offset in the file where 49 | // the data for that key begins (or would begin if the key were 50 | // present in the file). The returned value is in terms of file 51 | // bytes, and so includes effects like compression of the underlying data. 52 | // E.g., the approximate offset of the last key in the table will 53 | // be close to the file length. 54 | uint64_t ApproximateOffsetOf(const Slice& key) const; 55 | 56 | private: 57 | struct Rep; 58 | Rep* rep_; 59 | 60 | explicit Table(Rep* rep) { rep_ = rep; } 61 | static Iterator* BlockReader(void*, const ReadOptions&, const Slice&); 62 | 63 | // No copying allowed 64 | Table(const Table&); 65 | void operator=(const Table&); 66 | }; 67 | 68 | } 69 | 70 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_ 71 | -------------------------------------------------------------------------------- /leveldb/util/env_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/env.h" 6 | 7 | #include "port/port.h" 8 | #include "util/testharness.h" 9 | 10 | namespace leveldb { 11 | 12 | static const int kDelayMicros = 100000; 13 | 14 | class EnvPosixTest { 15 | private: 16 | port::Mutex mu_; 17 | std::string events_; 18 | 19 | public: 20 | Env* env_; 21 | EnvPosixTest() : env_(Env::Default()) { } 22 | }; 23 | 24 | static void SetBool(void* ptr) { 25 | *(reinterpret_cast(ptr)) = true; 26 | } 27 | 28 | TEST(EnvPosixTest, RunImmediately) { 29 | bool called = false; 30 | env_->Schedule(&SetBool, &called); 31 | Env::Default()->SleepForMicroseconds(kDelayMicros); 32 | ASSERT_TRUE(called); 33 | } 34 | 35 | TEST(EnvPosixTest, RunMany) { 36 | int last_id = 0; 37 | 38 | struct CB { 39 | int* last_id_ptr; // Pointer to shared slot 40 | int id; // Order# for the execution of this callback 41 | 42 | CB(int* p, int i) : last_id_ptr(p), id(i) { } 43 | 44 | static void Run(void* v) { 45 | CB* cb = reinterpret_cast(v); 46 | ASSERT_EQ(cb->id-1, *cb->last_id_ptr); 47 | *cb->last_id_ptr = cb->id; 48 | } 49 | }; 50 | 51 | // Schedule in different order than start time 52 | CB cb1(&last_id, 1); 53 | CB cb2(&last_id, 2); 54 | CB cb3(&last_id, 3); 55 | CB cb4(&last_id, 4); 56 | env_->Schedule(&CB::Run, &cb1); 57 | env_->Schedule(&CB::Run, &cb2); 58 | env_->Schedule(&CB::Run, &cb3); 59 | env_->Schedule(&CB::Run, &cb4); 60 | 61 | Env::Default()->SleepForMicroseconds(kDelayMicros); 62 | ASSERT_EQ(4, last_id); 63 | } 64 | 65 | struct State { 66 | port::Mutex mu; 67 | int val; 68 | int num_running; 69 | }; 70 | 71 | static void ThreadBody(void* arg) { 72 | State* s = reinterpret_cast(arg); 73 | s->mu.Lock(); 74 | s->val += 1; 75 | s->num_running -= 1; 76 | s->mu.Unlock(); 77 | } 78 | 79 | TEST(EnvPosixTest, StartThread) { 80 | State state; 81 | state.val = 0; 82 | state.num_running = 3; 83 | for (int i = 0; i < 3; i++) { 84 | env_->StartThread(&ThreadBody, &state); 85 | } 86 | while (true) { 87 | state.mu.Lock(); 88 | int num = state.num_running; 89 | state.mu.Unlock(); 90 | if (num == 0) { 91 | break; 92 | } 93 | Env::Default()->SleepForMicroseconds(kDelayMicros); 94 | } 95 | ASSERT_EQ(state.val, 3); 96 | } 97 | 98 | } 99 | 100 | int main(int argc, char** argv) { 101 | return leveldb::test::RunAllTests(); 102 | } 103 | -------------------------------------------------------------------------------- /leveldb/port/port_chromium.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // See port_example.h for documentation for the following types/functions. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_PORT_CHROMIUM_H_ 8 | #define STORAGE_LEVELDB_PORT_PORT_CHROMIUM_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include "base/atomicops.h" 14 | #include "base/basictypes.h" 15 | #include "base/logging.h" 16 | #include "base/synchronization/condition_variable.h" 17 | #include "base/synchronization/lock.h" 18 | 19 | // Linux's ThreadIdentifier() needs this. 20 | #if defined(OS_LINUX) 21 | # include 22 | #endif 23 | 24 | #if defined(OS_WIN) 25 | #define snprintf _snprintf 26 | #define va_copy(a, b) do { (a) = (b); } while (0) 27 | #endif 28 | 29 | namespace leveldb { 30 | namespace port { 31 | 32 | // Chromium only supports little endian. 33 | static const bool kLittleEndian = true; 34 | 35 | class Mutex { 36 | public: 37 | Mutex(); 38 | ~Mutex(); 39 | void Lock(); 40 | void Unlock(); 41 | void AssertHeld(); 42 | 43 | private: 44 | base::Lock mu_; 45 | 46 | friend class CondVar; 47 | DISALLOW_COPY_AND_ASSIGN(Mutex); 48 | }; 49 | 50 | class CondVar { 51 | public: 52 | explicit CondVar(Mutex* mu); 53 | ~CondVar(); 54 | void Wait(); 55 | void Signal(); 56 | void SignalAll(); 57 | 58 | private: 59 | base::ConditionVariable cv_; 60 | 61 | DISALLOW_COPY_AND_ASSIGN(CondVar); 62 | }; 63 | 64 | class AtomicPointer { 65 | private: 66 | typedef base::subtle::AtomicWord Rep; 67 | Rep rep_; 68 | public: 69 | AtomicPointer() { } 70 | explicit AtomicPointer(void* p) : rep_(reinterpret_cast(p)) {} 71 | inline void* Acquire_Load() const { 72 | return reinterpret_cast(::base::subtle::Acquire_Load(&rep_)); 73 | } 74 | inline void Release_Store(void* v) { 75 | ::base::subtle::Release_Store(&rep_, reinterpret_cast(v)); 76 | } 77 | inline void* NoBarrier_Load() const { 78 | return reinterpret_cast(::base::subtle::NoBarrier_Load(&rep_)); 79 | } 80 | inline void NoBarrier_Store(void* v) { 81 | ::base::subtle::NoBarrier_Store(&rep_, reinterpret_cast(v)); 82 | } 83 | }; 84 | 85 | bool Snappy_Compress(const char* input, size_t input_length, 86 | std::string* output); 87 | bool Snappy_Uncompress(const char* input_data, size_t input_length, 88 | std::string* output); 89 | 90 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 91 | return false; 92 | } 93 | 94 | } 95 | } 96 | 97 | #endif // STORAGE_LEVELDB_PORT_PORT_CHROMIUM_H_ 98 | -------------------------------------------------------------------------------- /leveldb/db/memtable.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_ 6 | #define STORAGE_LEVELDB_DB_MEMTABLE_H_ 7 | 8 | #include 9 | #include "leveldb/db.h" 10 | #include "db/dbformat.h" 11 | #include "db/skiplist.h" 12 | #include "util/arena.h" 13 | 14 | namespace leveldb { 15 | 16 | class InternalKeyComparator; 17 | class Mutex; 18 | class MemTableIterator; 19 | 20 | class MemTable { 21 | public: 22 | // MemTables are reference counted. The initial reference count 23 | // is zero and the caller must call Ref() at least once. 24 | explicit MemTable(const InternalKeyComparator& comparator); 25 | 26 | // Increase reference count. 27 | void Ref() { ++refs_; } 28 | 29 | // Drop reference count. Delete if no more references exist. 30 | void Unref() { 31 | --refs_; 32 | assert(refs_ >= 0); 33 | if (refs_ <= 0) { 34 | delete this; 35 | } 36 | } 37 | 38 | // Returns an estimate of the number of bytes of data in use by this 39 | // data structure. 40 | // 41 | // REQUIRES: external synchronization to prevent simultaneous 42 | // operations on the same MemTable. 43 | size_t ApproximateMemoryUsage(); 44 | 45 | // Return an iterator that yields the contents of the memtable. 46 | // 47 | // The caller must ensure that the underlying MemTable remains live 48 | // while the returned iterator is live. The keys returned by this 49 | // iterator are internal keys encoded by AppendInternalKey in the 50 | // db/format.{h,cc} module. 51 | Iterator* NewIterator(); 52 | 53 | // Add an entry into memtable that maps key to value at the 54 | // specified sequence number and with the specified type. 55 | // Typically value will be empty if type==kTypeDeletion. 56 | void Add(SequenceNumber seq, ValueType type, 57 | const Slice& key, 58 | const Slice& value); 59 | 60 | private: 61 | ~MemTable(); // Private since only Unref() should be used to delete it 62 | 63 | struct KeyComparator { 64 | const InternalKeyComparator comparator; 65 | explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } 66 | int operator()(const char* a, const char* b) const; 67 | }; 68 | friend class MemTableIterator; 69 | friend class MemTableBackwardIterator; 70 | 71 | typedef SkipList Table; 72 | 73 | KeyComparator comparator_; 74 | int refs_; 75 | Arena arena_; 76 | Table table_; 77 | 78 | // No copying allowed 79 | MemTable(const MemTable&); 80 | void operator=(const MemTable&); 81 | }; 82 | 83 | } 84 | 85 | #endif // STORAGE_LEVELDB_DB_MEMTABLE_H_ 86 | -------------------------------------------------------------------------------- /leveldb/port/port_posix.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // See port_example.h for documentation for the following types/functions. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 8 | #define STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 9 | 10 | #define GCC_VERSION (__GNUC__ * 10000 \ 11 | + __GNUC_MINOR__ * 100 \ 12 | + __GNUC_PATCHLEVEL__) 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #if GCC_VERSION > 40500 // check for GCC > 4.5 19 | #include 20 | #else 21 | #include 22 | #endif // if GCC_VERSION > 40500 23 | 24 | #include 25 | 26 | namespace leveldb { 27 | namespace port { 28 | 29 | static const bool kLittleEndian = (__BYTE_ORDER == __LITTLE_ENDIAN); 30 | 31 | class CondVar; 32 | 33 | class Mutex { 34 | public: 35 | Mutex(); 36 | ~Mutex(); 37 | 38 | void Lock(); 39 | void Unlock(); 40 | void AssertHeld() { } 41 | 42 | private: 43 | friend class CondVar; 44 | pthread_mutex_t mu_; 45 | 46 | // No copying 47 | Mutex(const Mutex&); 48 | void operator=(const Mutex&); 49 | }; 50 | 51 | class CondVar { 52 | public: 53 | explicit CondVar(Mutex* mu); 54 | ~CondVar(); 55 | void Wait(); 56 | void Signal(); 57 | void SignalAll(); 58 | private: 59 | pthread_cond_t cv_; 60 | Mutex* mu_; 61 | }; 62 | 63 | // Storage for a lock-free pointer 64 | class AtomicPointer { 65 | private: 66 | std::atomic rep_; 67 | public: 68 | AtomicPointer() { } 69 | explicit AtomicPointer(void* v) : rep_(v) { } 70 | inline void* Acquire_Load() const { 71 | return rep_.load(std::memory_order_acquire); 72 | } 73 | inline void Release_Store(void* v) { 74 | rep_.store(v, std::memory_order_release); 75 | } 76 | inline void* NoBarrier_Load() const { 77 | return rep_.load(std::memory_order_relaxed); 78 | } 79 | inline void NoBarrier_Store(void* v) { 80 | rep_.store(v, std::memory_order_relaxed); 81 | } 82 | }; 83 | 84 | // TODO(gabor): Implement actual compress 85 | inline bool Snappy_Compress(const char* input, size_t input_length, 86 | std::string* output) { 87 | return false; 88 | } 89 | 90 | // TODO(gabor): Implement actual uncompress 91 | inline bool Snappy_Uncompress(const char* input_data, size_t input_length, 92 | std::string* output) { 93 | return false; 94 | } 95 | 96 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 97 | return false; 98 | } 99 | 100 | } 101 | } 102 | 103 | #endif // STORAGE_LEVELDB_PORT_PORT_POSIX_H_ 104 | -------------------------------------------------------------------------------- /leveldb/db/write_batch_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/db.h" 6 | 7 | #include "db/memtable.h" 8 | #include "db/write_batch_internal.h" 9 | #include "leveldb/env.h" 10 | #include "util/logging.h" 11 | #include "util/testharness.h" 12 | 13 | namespace leveldb { 14 | 15 | static std::string PrintContents(WriteBatch* b) { 16 | InternalKeyComparator cmp(BytewiseComparator()); 17 | MemTable* mem = new MemTable(cmp); 18 | mem->Ref(); 19 | std::string state; 20 | Status s = WriteBatchInternal::InsertInto(b, mem); 21 | Iterator* iter = mem->NewIterator(); 22 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { 23 | ParsedInternalKey ikey; 24 | ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); 25 | switch (ikey.type) { 26 | case kTypeValue: 27 | state.append("Put("); 28 | state.append(ikey.user_key.ToString()); 29 | state.append(", "); 30 | state.append(iter->value().ToString()); 31 | state.append(")"); 32 | break; 33 | case kTypeDeletion: 34 | state.append("Delete("); 35 | state.append(ikey.user_key.ToString()); 36 | state.append(")"); 37 | break; 38 | } 39 | state.append("@"); 40 | state.append(NumberToString(ikey.sequence)); 41 | } 42 | delete iter; 43 | if (!s.ok()) { 44 | state.append("ParseError()"); 45 | } 46 | mem->Unref(); 47 | return state; 48 | } 49 | 50 | class WriteBatchTest { }; 51 | 52 | TEST(WriteBatchTest, Empty) { 53 | WriteBatch batch; 54 | ASSERT_EQ("", PrintContents(&batch)); 55 | ASSERT_EQ(0, WriteBatchInternal::Count(&batch)); 56 | } 57 | 58 | TEST(WriteBatchTest, Multiple) { 59 | WriteBatch batch; 60 | batch.Put(Slice("foo"), Slice("bar")); 61 | batch.Delete(Slice("box")); 62 | batch.Put(Slice("baz"), Slice("boo")); 63 | WriteBatchInternal::SetSequence(&batch, 100); 64 | ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); 65 | ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); 66 | ASSERT_EQ("Put(baz, boo)@102" 67 | "Delete(box)@101" 68 | "Put(foo, bar)@100", 69 | PrintContents(&batch)); 70 | } 71 | 72 | TEST(WriteBatchTest, Corruption) { 73 | WriteBatch batch; 74 | batch.Put(Slice("foo"), Slice("bar")); 75 | batch.Delete(Slice("box")); 76 | WriteBatchInternal::SetSequence(&batch, 200); 77 | Slice contents = WriteBatchInternal::Contents(&batch); 78 | WriteBatchInternal::SetContents(&batch, 79 | Slice(contents.data(),contents.size()-1)); 80 | ASSERT_EQ("Put(foo, bar)@200" 81 | "ParseError()", 82 | PrintContents(&batch)); 83 | } 84 | 85 | } 86 | 87 | int main(int argc, char** argv) { 88 | return leveldb::test::RunAllTests(); 89 | } 90 | -------------------------------------------------------------------------------- /leveldb/db/table_cache.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/table_cache.h" 6 | 7 | #include "db/filename.h" 8 | #include "leveldb/env.h" 9 | #include "leveldb/table.h" 10 | #include "util/coding.h" 11 | 12 | namespace leveldb { 13 | 14 | struct TableAndFile { 15 | RandomAccessFile* file; 16 | Table* table; 17 | }; 18 | 19 | static void DeleteEntry(const Slice& key, void* value) { 20 | TableAndFile* tf = reinterpret_cast(value); 21 | delete tf->table; 22 | delete tf->file; 23 | delete tf; 24 | } 25 | 26 | static void UnrefEntry(void* arg1, void* arg2) { 27 | Cache* cache = reinterpret_cast(arg1); 28 | Cache::Handle* h = reinterpret_cast(arg2); 29 | cache->Release(h); 30 | } 31 | 32 | TableCache::TableCache(const std::string& dbname, 33 | const Options* options, 34 | int entries) 35 | : env_(options->env), 36 | dbname_(dbname), 37 | options_(options), 38 | cache_(NewLRUCache(entries)) { 39 | } 40 | 41 | TableCache::~TableCache() { 42 | delete cache_; 43 | } 44 | 45 | Iterator* TableCache::NewIterator(const ReadOptions& options, 46 | uint64_t file_number, 47 | uint64_t file_size, 48 | Table** tableptr) { 49 | if (tableptr != NULL) { 50 | *tableptr = NULL; 51 | } 52 | 53 | char buf[sizeof(file_number)]; 54 | EncodeFixed64(buf, file_number); 55 | Slice key(buf, sizeof(buf)); 56 | Cache::Handle* handle = cache_->Lookup(key); 57 | if (handle == NULL) { 58 | std::string fname = TableFileName(dbname_, file_number); 59 | RandomAccessFile* file = NULL; 60 | Table* table = NULL; 61 | Status s = env_->NewRandomAccessFile(fname, &file); 62 | if (s.ok()) { 63 | s = Table::Open(*options_, file, file_size, &table); 64 | } 65 | 66 | if (!s.ok()) { 67 | assert(table == NULL); 68 | delete file; 69 | // We do not cache error results so that if the error is transient, 70 | // or somebody repairs the file, we recover automatically. 71 | return NewErrorIterator(s); 72 | } 73 | 74 | TableAndFile* tf = new TableAndFile; 75 | tf->file = file; 76 | tf->table = table; 77 | handle = cache_->Insert(key, tf, 1, &DeleteEntry); 78 | } 79 | 80 | Table* table = reinterpret_cast(cache_->Value(handle))->table; 81 | Iterator* result = table->NewIterator(options); 82 | result->RegisterCleanup(&UnrefEntry, cache_, handle); 83 | if (tableptr != NULL) { 84 | *tableptr = table; 85 | } 86 | return result; 87 | } 88 | 89 | void TableCache::Evict(uint64_t file_number) { 90 | char buf[sizeof(file_number)]; 91 | EncodeFixed64(buf, file_number); 92 | cache_->Erase(Slice(buf, sizeof(buf))); 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /leveldb/doc/log_format.txt: -------------------------------------------------------------------------------- 1 | The log file contents are a sequence of 32KB blocks. The only 2 | exception is that the tail of the file may contain a partial block. 3 | 4 | Each block consists of a sequence of records: 5 | block := record* trailer? 6 | record := 7 | checksum: uint32 // crc32c of type and data[] 8 | length: uint16 9 | type: uint8 // One of FULL, FIRST, MIDDLE, LAST 10 | data: uint8[length] 11 | 12 | A record never starts within the last six bytes of a block (since it 13 | won't fit). Any leftover bytes here form the trailer, which must 14 | consist entirely of zero bytes and must be skipped by readers. 15 | 16 | Aside: if exactly seven bytes are left in the current block, and a new 17 | non-zero length record is added, the writer must emit a FIRST record 18 | (which contains zero bytes of user data) to fill up the trailing seven 19 | bytes of the block and then emit all of the user data in subsequent 20 | blocks. 21 | 22 | More types may be added in the future. Some Readers may skip record 23 | types they do not understand, others may report that some data was 24 | skipped. 25 | 26 | FULL == 1 27 | FIRST == 2 28 | MIDDLE == 3 29 | LAST == 4 30 | 31 | The FULL record contains the contents of an entire user record. 32 | 33 | FIRST, MIDDLE, LAST are types used for user records that have been 34 | split into multiple fragments (typically because of block boundaries). 35 | FIRST is the type of the first fragment of a user record, LAST is the 36 | type of the last fragment of a user record, and MID is the type of all 37 | interior fragments of a user record. 38 | 39 | Example: consider a sequence of user records: 40 | A: length 1000 41 | B: length 97270 42 | C: length 8000 43 | A will be stored as a FULL record in the first block. 44 | 45 | B will be split into three fragments: first fragment occupies the rest 46 | of the first block, second fragment occupies the entirety of the 47 | second block, and the third fragment occupies a prefix of the third 48 | block. This will leave six bytes free in the third block, which will 49 | be left empty as the trailer. 50 | 51 | C will be stored as a FULL record in the fourth block. 52 | 53 | =================== 54 | 55 | Some benefits over the recordio format: 56 | 57 | (1) We do not need any heuristics for resyncing - just go to next 58 | block boundary and scan. If there is a corruption, skip to the next 59 | block. As a side-benefit, we do not get confused when part of the 60 | contents of one log file are embedded as a record inside another log 61 | file. 62 | 63 | (2) Splitting at approximate boundaries (e.g., for mapreduce) is 64 | simple: find the next block boundary and skip records until we 65 | hit a FULL or FIRST record. 66 | 67 | (3) We do not need extra buffering for large records. 68 | 69 | Some downsides compared to recordio format: 70 | 71 | (1) No packing of tiny records. This could be fixed by adding a new 72 | record type, so it is a shortcoming of the current implementation, 73 | not necessarily the format. 74 | 75 | (2) No compression. Again, this could be fixed by adding new record types. 76 | -------------------------------------------------------------------------------- /leveldb/db/filename.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // File names used by DB code 6 | 7 | #ifndef STORAGE_LEVELDB_DB_FILENAME_H_ 8 | #define STORAGE_LEVELDB_DB_FILENAME_H_ 9 | 10 | #include 11 | #include 12 | #include "leveldb/slice.h" 13 | #include "leveldb/status.h" 14 | #include "port/port.h" 15 | 16 | namespace leveldb { 17 | 18 | class Env; 19 | 20 | enum FileType { 21 | kLogFile, 22 | kDBLockFile, 23 | kTableFile, 24 | kDescriptorFile, 25 | kCurrentFile, 26 | kTempFile, 27 | kInfoLogFile, // Either the current one, or an old one 28 | }; 29 | 30 | // Return the name of the log file with the specified number 31 | // in the db named by "dbname". The result will be prefixed with 32 | // "dbname". 33 | extern std::string LogFileName(const std::string& dbname, uint64_t number); 34 | 35 | // Return the name of the sstable with the specified number 36 | // in the db named by "dbname". The result will be prefixed with 37 | // "dbname". 38 | extern std::string TableFileName(const std::string& dbname, uint64_t number); 39 | 40 | // Return the name of the descriptor file for the db named by 41 | // "dbname" and the specified incarnation number. The result will be 42 | // prefixed with "dbname". 43 | extern std::string DescriptorFileName(const std::string& dbname, 44 | uint64_t number); 45 | 46 | // Return the name of the current file. This file contains the name 47 | // of the current manifest file. The result will be prefixed with 48 | // "dbname". 49 | extern std::string CurrentFileName(const std::string& dbname); 50 | 51 | // Return the name of the lock file for the db named by 52 | // "dbname". The result will be prefixed with "dbname". 53 | extern std::string LockFileName(const std::string& dbname); 54 | 55 | // Return the name of a temporary file owned by the db named "dbname". 56 | // The result will be prefixed with "dbname". 57 | extern std::string TempFileName(const std::string& dbname, uint64_t number); 58 | 59 | // Return the name of the info log file for "dbname". 60 | extern std::string InfoLogFileName(const std::string& dbname); 61 | 62 | // Return the name of the old info log file for "dbname". 63 | extern std::string OldInfoLogFileName(const std::string& dbname); 64 | 65 | // If filename is a leveldb file, store the type of the file in *type. 66 | // The number encoded in the filename is stored in *number. If the 67 | // filename was successfully parsed, returns true. Else return false. 68 | extern bool ParseFileName(const std::string& filename, 69 | uint64_t* number, 70 | FileType* type); 71 | 72 | // Make the CURRENT file point to the descriptor file with the 73 | // specified number. 74 | extern Status SetCurrentFile(Env* env, const std::string& dbname, 75 | uint64_t descriptor_number); 76 | 77 | 78 | } 79 | 80 | #endif // STORAGE_LEVELDB_DB_FILENAME_H_ 81 | -------------------------------------------------------------------------------- /leveldb/db/dbformat.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "db/dbformat.h" 7 | #include "port/port.h" 8 | #include "util/coding.h" 9 | 10 | namespace leveldb { 11 | 12 | static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) { 13 | assert(seq <= kMaxSequenceNumber); 14 | assert(t <= kValueTypeForSeek); 15 | return (seq << 8) | t; 16 | } 17 | 18 | void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { 19 | result->append(key.user_key.data(), key.user_key.size()); 20 | PutFixed64(result, PackSequenceAndType(key.sequence, key.type)); 21 | } 22 | 23 | std::string ParsedInternalKey::DebugString() const { 24 | char buf[50]; 25 | snprintf(buf, sizeof(buf), "' @ %llu : %d", 26 | (unsigned long long) sequence, 27 | int(type)); 28 | std::string result = "'"; 29 | result += user_key.ToString(); 30 | result += buf; 31 | return result; 32 | } 33 | 34 | const char* InternalKeyComparator::Name() const { 35 | return "leveldb.InternalKeyComparator"; 36 | } 37 | 38 | int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const { 39 | // Order by: 40 | // increasing user key (according to user-supplied comparator) 41 | // decreasing sequence number 42 | // decreasing type (though sequence# should be enough to disambiguate) 43 | int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey)); 44 | if (r == 0) { 45 | const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8); 46 | const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8); 47 | if (anum > bnum) { 48 | r = -1; 49 | } else if (anum < bnum) { 50 | r = +1; 51 | } 52 | } 53 | return r; 54 | } 55 | 56 | void InternalKeyComparator::FindShortestSeparator( 57 | std::string* start, 58 | const Slice& limit) const { 59 | // Attempt to shorten the user portion of the key 60 | Slice user_start = ExtractUserKey(*start); 61 | Slice user_limit = ExtractUserKey(limit); 62 | std::string tmp(user_start.data(), user_start.size()); 63 | user_comparator_->FindShortestSeparator(&tmp, user_limit); 64 | if (user_comparator_->Compare(*start, tmp) < 0) { 65 | // User key has become larger. Tack on the earliest possible 66 | // number to the shortened user key. 67 | PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); 68 | assert(this->Compare(*start, tmp) < 0); 69 | assert(this->Compare(tmp, limit) < 0); 70 | start->swap(tmp); 71 | } 72 | } 73 | 74 | void InternalKeyComparator::FindShortSuccessor(std::string* key) const { 75 | Slice user_key = ExtractUserKey(*key); 76 | std::string tmp(user_key.data(), user_key.size()); 77 | user_comparator_->FindShortSuccessor(&tmp); 78 | if (user_comparator_->Compare(user_key, tmp) < 0) { 79 | // User key has become larger. Tack on the earliest possible 80 | // number to the shortened user key. 81 | PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); 82 | assert(this->Compare(*key, tmp) < 0); 83 | key->swap(tmp); 84 | } 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /leveldb/db/log_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/log_writer.h" 6 | 7 | #include 8 | #include "leveldb/env.h" 9 | #include "util/coding.h" 10 | #include "util/crc32c.h" 11 | 12 | namespace leveldb { 13 | namespace log { 14 | 15 | Writer::Writer(WritableFile* dest) 16 | : dest_(dest), 17 | block_offset_(0) { 18 | for (int i = 0; i <= kMaxRecordType; i++) { 19 | char t = static_cast(i); 20 | type_crc_[i] = crc32c::Value(&t, 1); 21 | } 22 | } 23 | 24 | Writer::~Writer() { 25 | } 26 | 27 | Status Writer::AddRecord(const Slice& slice) { 28 | const char* ptr = slice.data(); 29 | size_t left = slice.size(); 30 | 31 | // Fragment the record if necessary and emit it. Note that if slice 32 | // is empty, we still want to iterate once to emit a single 33 | // zero-length record 34 | Status s; 35 | bool begin = true; 36 | do { 37 | const int leftover = kBlockSize - block_offset_; 38 | assert(leftover >= 0); 39 | if (leftover < kHeaderSize) { 40 | // Switch to a new block 41 | if (leftover > 0) { 42 | // Fill the trailer (literal below relies on kHeaderSize being 7) 43 | assert(kHeaderSize == 7); 44 | dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); 45 | } 46 | block_offset_ = 0; 47 | } 48 | 49 | // Invariant: we never leave < kHeaderSize bytes in a block. 50 | assert(kBlockSize - block_offset_ - kHeaderSize >= 0); 51 | 52 | const size_t avail = kBlockSize - block_offset_ - kHeaderSize; 53 | const size_t fragment_length = (left < avail) ? left : avail; 54 | 55 | RecordType type; 56 | const bool end = (left == fragment_length); 57 | if (begin && end) { 58 | type = kFullType; 59 | } else if (begin) { 60 | type = kFirstType; 61 | } else if (end) { 62 | type = kLastType; 63 | } else { 64 | type = kMiddleType; 65 | } 66 | 67 | s = EmitPhysicalRecord(type, ptr, fragment_length); 68 | ptr += fragment_length; 69 | left -= fragment_length; 70 | begin = false; 71 | } while (s.ok() && left > 0); 72 | return s; 73 | } 74 | 75 | Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { 76 | assert(n <= 0xffff); // Must fit in two bytes 77 | assert(block_offset_ + kHeaderSize + n <= kBlockSize); 78 | 79 | // Format the header 80 | char buf[kHeaderSize]; 81 | buf[4] = static_cast(n & 0xff); 82 | buf[5] = static_cast(n >> 8); 83 | buf[6] = static_cast(t); 84 | 85 | // Compute the crc of the record type and the payload. 86 | uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); 87 | crc = crc32c::Mask(crc); // Adjust for storage 88 | EncodeFixed32(buf, crc); 89 | 90 | // Write the header and the payload 91 | Status s = dest_->Append(Slice(buf, kHeaderSize)); 92 | if (s.ok()) { 93 | s = dest_->Append(Slice(ptr, n)); 94 | if (s.ok()) { 95 | s = dest_->Flush(); 96 | } 97 | } 98 | block_offset_ += kHeaderSize + n; 99 | return s; 100 | } 101 | 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /leveldb/port/port_osx.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // See port_example.h for documentation for the following types/functions. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_PORT_OSX_H_ 8 | #define STORAGE_LEVELDB_PORT_PORT_OSX_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | namespace leveldb { 18 | 19 | // The following 4 methods implemented here for the benefit of env_posix.cc. 20 | inline size_t fread_unlocked(void *a, size_t b, size_t c, FILE *d) { 21 | return fread(a, b, c, d); 22 | } 23 | 24 | inline size_t fwrite_unlocked(const void *a, size_t b, size_t c, FILE *d) { 25 | return fwrite(a, b, c, d); 26 | } 27 | 28 | inline int fflush_unlocked(FILE *f) { 29 | return fflush(f); 30 | } 31 | 32 | inline int fdatasync(int fd) { 33 | return fsync(fd); 34 | } 35 | 36 | namespace port { 37 | 38 | static const bool kLittleEndian = (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN); 39 | 40 | // ------------------ Threading ------------------- 41 | 42 | // A Mutex represents an exclusive lock. 43 | class Mutex { 44 | public: 45 | Mutex(); 46 | ~Mutex(); 47 | 48 | void Lock(); 49 | void Unlock(); 50 | void AssertHeld() { } 51 | 52 | private: 53 | friend class CondVar; 54 | pthread_mutex_t mu_; 55 | 56 | // No copying 57 | Mutex(const Mutex&); 58 | void operator=(const Mutex&); 59 | }; 60 | 61 | class CondVar { 62 | public: 63 | explicit CondVar(Mutex* mu); 64 | ~CondVar(); 65 | 66 | void Wait(); 67 | void Signal(); 68 | void SignalAll(); 69 | 70 | private: 71 | pthread_cond_t cv_; 72 | Mutex* mu_; 73 | }; 74 | 75 | inline void MemoryBarrier() { 76 | #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) 77 | // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 78 | // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 79 | __asm__ __volatile__("" : : : "memory"); 80 | #else 81 | OSMemoryBarrier(); 82 | #endif 83 | } 84 | 85 | class AtomicPointer { 86 | private: 87 | void* ptr_; 88 | public: 89 | AtomicPointer() { } 90 | explicit AtomicPointer(void* p) : ptr_(p) {} 91 | inline void* Acquire_Load() const { 92 | void* ptr = ptr_; 93 | MemoryBarrier(); 94 | return ptr; 95 | } 96 | inline void Release_Store(void* v) { 97 | MemoryBarrier(); 98 | ptr_ = v; 99 | } 100 | inline void* NoBarrier_Load() const { 101 | return ptr_; 102 | } 103 | inline void NoBarrier_Store(void* v) { 104 | ptr_ = v; 105 | } 106 | }; 107 | 108 | inline bool Snappy_Compress(const char* input, size_t input_length, 109 | std::string* output) { 110 | return false; 111 | } 112 | 113 | inline bool Snappy_Uncompress(const char* input_data, size_t input_length, 114 | std::string* output) { 115 | return false; 116 | } 117 | 118 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 119 | return false; 120 | } 121 | 122 | } 123 | } 124 | 125 | #endif // STORAGE_LEVELDB_PORT_PORT_OSX_H_ 126 | -------------------------------------------------------------------------------- /leveldb/db/version_edit.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 6 | #define STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "db/dbformat.h" 12 | 13 | namespace leveldb { 14 | 15 | class VersionSet; 16 | 17 | struct FileMetaData { 18 | int refs; 19 | uint64_t number; 20 | uint64_t file_size; // File size in bytes 21 | InternalKey smallest; // Smallest internal key served by table 22 | InternalKey largest; // Largest internal key served by table 23 | 24 | FileMetaData() : refs(0), file_size(0) { } 25 | }; 26 | 27 | class VersionEdit { 28 | public: 29 | VersionEdit() { Clear(); } 30 | ~VersionEdit() { } 31 | 32 | void Clear(); 33 | 34 | void SetComparatorName(const Slice& name) { 35 | has_comparator_ = true; 36 | comparator_ = name.ToString(); 37 | } 38 | void SetLogNumber(uint64_t num) { 39 | has_log_number_ = true; 40 | log_number_ = num; 41 | } 42 | void SetPrevLogNumber(uint64_t num) { 43 | has_prev_log_number_ = true; 44 | prev_log_number_ = num; 45 | } 46 | void SetNextFile(uint64_t num) { 47 | has_next_file_number_ = true; 48 | next_file_number_ = num; 49 | } 50 | void SetLastSequence(SequenceNumber seq) { 51 | has_last_sequence_ = true; 52 | last_sequence_ = seq; 53 | } 54 | void SetCompactPointer(int level, const InternalKey& key) { 55 | compact_pointers_.push_back(std::make_pair(level, key)); 56 | } 57 | 58 | // Add the specified file at the specified number. 59 | // REQUIRES: This version has not been saved (see VersionSet::SaveTo) 60 | // REQUIRES: "smallest" and "largest" are smallest and largest keys in file 61 | void AddFile(int level, uint64_t file, 62 | uint64_t file_size, 63 | const InternalKey& smallest, 64 | const InternalKey& largest) { 65 | FileMetaData f; 66 | f.number = file; 67 | f.file_size = file_size; 68 | f.smallest = smallest; 69 | f.largest = largest; 70 | new_files_.push_back(std::make_pair(level, f)); 71 | } 72 | 73 | // Delete the specified "file" from the specified "level". 74 | void DeleteFile(int level, uint64_t file) { 75 | deleted_files_.insert(std::make_pair(level, file)); 76 | } 77 | 78 | void EncodeTo(std::string* dst) const; 79 | Status DecodeFrom(const Slice& src); 80 | 81 | std::string DebugString() const; 82 | 83 | private: 84 | friend class VersionSet; 85 | 86 | typedef std::set< std::pair > DeletedFileSet; 87 | 88 | std::string comparator_; 89 | uint64_t log_number_; 90 | uint64_t prev_log_number_; 91 | uint64_t next_file_number_; 92 | SequenceNumber last_sequence_; 93 | bool has_comparator_; 94 | bool has_log_number_; 95 | bool has_prev_log_number_; 96 | bool has_next_file_number_; 97 | bool has_last_sequence_; 98 | 99 | std::vector< std::pair > compact_pointers_; 100 | DeletedFileSet deleted_files_; 101 | std::vector< std::pair > new_files_; 102 | }; 103 | 104 | } 105 | 106 | #endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_ 107 | -------------------------------------------------------------------------------- /leveldb/table/format.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_ 6 | #define STORAGE_LEVELDB_TABLE_FORMAT_H_ 7 | 8 | #include 9 | #include 10 | #include "leveldb/slice.h" 11 | #include "leveldb/status.h" 12 | #include "leveldb/table_builder.h" 13 | 14 | namespace leveldb { 15 | 16 | class Block; 17 | class RandomAccessFile; 18 | struct ReadOptions; 19 | 20 | // BlockHandle is a pointer to the extent of a file that stores a data 21 | // block or a meta block. 22 | class BlockHandle { 23 | public: 24 | BlockHandle(); 25 | 26 | // The offset of the block in the file. 27 | uint64_t offset() const { return offset_; } 28 | void set_offset(uint64_t offset) { offset_ = offset; } 29 | 30 | // The size of the stored block 31 | uint64_t size() const { return size_; } 32 | void set_size(uint64_t size) { size_ = size; } 33 | 34 | void EncodeTo(std::string* dst) const; 35 | Status DecodeFrom(Slice* input); 36 | 37 | // Maximum encoding length of a BlockHandle 38 | enum { kMaxEncodedLength = 10 + 10 }; 39 | 40 | private: 41 | uint64_t offset_; 42 | uint64_t size_; 43 | }; 44 | 45 | // Footer encapsulates the fixed information stored at the tail 46 | // end of every table file. 47 | class Footer { 48 | public: 49 | Footer() { } 50 | 51 | // The block handle for the metaindex block of the table 52 | const BlockHandle& metaindex_handle() const { return metaindex_handle_; } 53 | void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; } 54 | 55 | // The block handle for the index block of the table 56 | const BlockHandle& index_handle() const { 57 | return index_handle_; 58 | } 59 | void set_index_handle(const BlockHandle& h) { 60 | index_handle_ = h; 61 | } 62 | 63 | void EncodeTo(std::string* dst) const; 64 | Status DecodeFrom(Slice* input); 65 | 66 | // Encoded length of a Footer. Note that the serialization of a 67 | // Footer will always occupy exactly this many bytes. It consists 68 | // of two block handles and a magic number. 69 | enum { 70 | kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8 71 | }; 72 | 73 | private: 74 | BlockHandle metaindex_handle_; 75 | BlockHandle index_handle_; 76 | }; 77 | 78 | // kTableMagicNumber was picked by running 79 | // echo http://code.google.com/p/leveldb/ | sha1sum 80 | // and taking the leading 64 bits. 81 | static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull; 82 | 83 | // 1-byte type + 32-bit crc 84 | static const size_t kBlockTrailerSize = 5; 85 | 86 | // Read the block identified by "handle" from "file". On success, 87 | // store a pointer to the heap-allocated result in *block and return 88 | // OK. On failure store NULL in *block and return non-OK. 89 | extern Status ReadBlock(RandomAccessFile* file, 90 | const ReadOptions& options, 91 | const BlockHandle& handle, 92 | Block** block); 93 | 94 | // Implementation details follow. Clients should ignore, 95 | 96 | inline BlockHandle::BlockHandle() 97 | : offset_(~static_cast(0)), 98 | size_(~static_cast(0)) { 99 | } 100 | 101 | } 102 | 103 | #endif // STORAGE_LEVELDB_TABLE_FORMAT_H_ 104 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/status.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Status encapsulates the result of an operation. It may indicate success, 6 | // or it may indicate an error with an associated error message. 7 | // 8 | // Multiple threads can invoke const methods on a Status without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same Status must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_STATUS_H_ 15 | 16 | #include 17 | #include "leveldb/slice.h" 18 | 19 | namespace leveldb { 20 | 21 | class Status { 22 | public: 23 | // Create a success status. 24 | Status() : state_(NULL) { } 25 | ~Status() { delete[] state_; } 26 | 27 | // Copy the specified status. 28 | Status(const Status& s); 29 | void operator=(const Status& s); 30 | 31 | // Return a success status. 32 | static Status OK() { return Status(); } 33 | 34 | // Return error status of an appropriate type. 35 | static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { 36 | return Status(kNotFound, msg, msg2); 37 | } 38 | static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { 39 | return Status(kCorruption, msg, msg2); 40 | } 41 | static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { 42 | return Status(kNotSupported, msg, msg2); 43 | } 44 | static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { 45 | return Status(kInvalidArgument, msg, msg2); 46 | } 47 | static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { 48 | return Status(kIOError, msg, msg2); 49 | } 50 | 51 | // Returns true iff the status indicates success. 52 | bool ok() const { return (state_ == NULL); } 53 | 54 | // Returns true iff the status indicates a NotFound error. 55 | bool IsNotFound() const { return code() == kNotFound; } 56 | 57 | // Return a string representation of this status suitable for printing. 58 | // Returns the string "OK" for success. 59 | std::string ToString() const; 60 | 61 | private: 62 | // OK status has a NULL state_. Otherwise, state_ is a new[] array 63 | // of the following form: 64 | // state_[0..3] == length of message 65 | // state_[4] == code 66 | // state_[5..] == message 67 | const char* state_; 68 | 69 | enum Code { 70 | kOk = 0, 71 | kNotFound = 1, 72 | kCorruption = 2, 73 | kNotSupported = 3, 74 | kInvalidArgument = 4, 75 | kIOError = 5, 76 | }; 77 | 78 | Code code() const { 79 | return (state_ == NULL) ? kOk : static_cast(state_[4]); 80 | } 81 | 82 | Status(Code code, const Slice& msg, const Slice& msg2); 83 | static const char* CopyState(const char* s); 84 | }; 85 | 86 | inline Status::Status(const Status& s) { 87 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 88 | } 89 | inline void Status::operator=(const Status& s) { 90 | // The following condition catches both aliasing (when this == &s), 91 | // and the common case where both s and *this are ok. 92 | if (state_ != s.state_) { 93 | delete[] state_; 94 | state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); 95 | } 96 | } 97 | 98 | } 99 | 100 | #endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_ 101 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/table_builder.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // TableBuilder provides the interface used to build a Table 6 | // (an immutable and sorted map from keys to values). 7 | // 8 | // Multiple threads can invoke const methods on a TableBuilder without 9 | // external synchronization, but if any of the threads may call a 10 | // non-const method, all threads accessing the same TableBuilder must use 11 | // external synchronization. 12 | 13 | #ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 14 | #define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 15 | 16 | #include 17 | #include "leveldb/options.h" 18 | #include "leveldb/status.h" 19 | 20 | namespace leveldb { 21 | 22 | class BlockBuilder; 23 | class BlockHandle; 24 | class WritableFile; 25 | 26 | class TableBuilder { 27 | public: 28 | // Create a builder that will store the contents of the table it is 29 | // building in *file. Does not close the file. It is up to the 30 | // caller to close the file after calling Finish(). 31 | TableBuilder(const Options& options, WritableFile* file); 32 | 33 | // REQUIRES: Either Finish() or Abandon() has been called. 34 | ~TableBuilder(); 35 | 36 | // Change the options used by this builder. Note: only some of the 37 | // option fields can be changed after construction. If a field is 38 | // not allowed to change dynamically and its value in the structure 39 | // passed to the constructor is different from its value in the 40 | // structure passed to this method, this method will return an error 41 | // without changing any fields. 42 | Status ChangeOptions(const Options& options); 43 | 44 | // Add key,value to the table being constructed. 45 | // REQUIRES: key is after any previously added key according to comparator. 46 | // REQUIRES: Finish(), Abandon() have not been called 47 | void Add(const Slice& key, const Slice& value); 48 | 49 | // Advanced operation: flush any buffered key/value pairs to file. 50 | // Can be used to ensure that two adjacent entries never live in 51 | // the same data block. Most clients should not need to use this method. 52 | // REQUIRES: Finish(), Abandon() have not been called 53 | void Flush(); 54 | 55 | // Return non-ok iff some error has been detected. 56 | Status status() const; 57 | 58 | // Finish building the table. Stops using the file passed to the 59 | // constructor after this function returns. 60 | // REQUIRES: Finish(), Abandon() have not been called 61 | Status Finish(); 62 | 63 | // Indicate that the contents of this builder should be abandoned. Stops 64 | // using the file passed to the constructor after this function returns. 65 | // If the caller is not going to call Finish(), it must call Abandon() 66 | // before destroying this builder. 67 | // REQUIRES: Finish(), Abandon() have not been called 68 | void Abandon(); 69 | 70 | // Number of calls to Add() so far. 71 | uint64_t NumEntries() const; 72 | 73 | // Size of the file generated so far. If invoked after a successful 74 | // Finish() call, returns the size of the final generated file. 75 | uint64_t FileSize() const; 76 | 77 | private: 78 | bool ok() const { return status().ok(); } 79 | void WriteBlock(BlockBuilder* block, BlockHandle* handle); 80 | 81 | struct Rep; 82 | Rep* rep_; 83 | 84 | // No copying allowed 85 | TableBuilder(const TableBuilder&); 86 | void operator=(const TableBuilder&); 87 | }; 88 | 89 | } 90 | 91 | #endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_ 92 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Slice is a simple structure containing a pointer into some external 6 | // storage and a size. The user of a Slice must ensure that the slice 7 | // is not used after the corresponding external storage has been 8 | // deallocated. 9 | // 10 | // Multiple threads can invoke const methods on a Slice without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Slice must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_SLICE_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace leveldb { 24 | 25 | class Slice { 26 | public: 27 | // Create an empty slice. 28 | Slice() : data_(""), size_(0) { } 29 | 30 | // Create a slice that refers to data[0,n-1]. 31 | Slice(const char* data, size_t n) : data_(data), size_(n) { } 32 | 33 | // Create a slice that refers to the contents of "s" 34 | Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } 35 | 36 | // Create a slice that refers to s[0,strlen(s)-1] 37 | Slice(const char* s) : data_(s), size_(strlen(s)) { } 38 | 39 | // Return a pointer to the beginning of the referenced data 40 | const char* data() const { return data_; } 41 | 42 | // Return the length (in bytes) of the referenced data 43 | size_t size() const { return size_; } 44 | 45 | // Return true iff the length of the referenced data is zero 46 | bool empty() const { return size_ == 0; } 47 | 48 | // Return the ith byte in the referenced data. 49 | // REQUIRES: n < size() 50 | char operator[](size_t n) const { 51 | assert(n < size()); 52 | return data_[n]; 53 | } 54 | 55 | // Change this slice to refer to an empty array 56 | void clear() { data_ = ""; size_ = 0; } 57 | 58 | // Drop the first "n" bytes from this slice. 59 | void remove_prefix(size_t n) { 60 | assert(n <= size()); 61 | data_ += n; 62 | size_ -= n; 63 | } 64 | 65 | // Return a string that contains the copy of the referenced data. 66 | std::string ToString() const { return std::string(data_, size_); } 67 | 68 | // Three-way comparison. Returns value: 69 | // < 0 iff "*this" < "b", 70 | // == 0 iff "*this" == "b", 71 | // > 0 iff "*this" > "b" 72 | int compare(const Slice& b) const; 73 | 74 | // Return true iff "x" is a prefix of "*this" 75 | bool starts_with(const Slice& x) const { 76 | return ((size_ >= x.size_) && 77 | (memcmp(data_, x.data_, x.size_) == 0)); 78 | } 79 | 80 | private: 81 | const char* data_; 82 | size_t size_; 83 | 84 | // Intentionally copyable 85 | }; 86 | 87 | inline bool operator==(const Slice& x, const Slice& y) { 88 | return ((x.size() == y.size()) && 89 | (memcmp(x.data(), y.data(), x.size()) == 0)); 90 | } 91 | 92 | inline bool operator!=(const Slice& x, const Slice& y) { 93 | return !(x == y); 94 | } 95 | 96 | inline int Slice::compare(const Slice& b) const { 97 | const int min_len = (size_ < b.size_) ? size_ : b.size_; 98 | int r = memcmp(data_, b.data_, min_len); 99 | if (r == 0) { 100 | if (size_ < b.size_) r = -1; 101 | else if (size_ > b.size_) r = +1; 102 | } 103 | return r; 104 | } 105 | 106 | } 107 | 108 | 109 | #endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_ 110 | -------------------------------------------------------------------------------- /leveldb/db/filename_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/filename.h" 6 | 7 | #include "db/dbformat.h" 8 | #include "port/port.h" 9 | #include "util/logging.h" 10 | #include "util/testharness.h" 11 | 12 | namespace leveldb { 13 | 14 | class FileNameTest { }; 15 | 16 | TEST(FileNameTest, Parse) { 17 | Slice db; 18 | FileType type; 19 | uint64_t number; 20 | 21 | // Successful parses 22 | static struct { 23 | const char* fname; 24 | uint64_t number; 25 | FileType type; 26 | } cases[] = { 27 | { "100.log", 100, kLogFile }, 28 | { "0.log", 0, kLogFile }, 29 | { "0.sst", 0, kTableFile }, 30 | { "CURRENT", 0, kCurrentFile }, 31 | { "LOCK", 0, kDBLockFile }, 32 | { "MANIFEST-2", 2, kDescriptorFile }, 33 | { "MANIFEST-7", 7, kDescriptorFile }, 34 | { "LOG", 0, kInfoLogFile }, 35 | { "LOG.old", 0, kInfoLogFile }, 36 | { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, 37 | }; 38 | for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { 39 | std::string f = cases[i].fname; 40 | ASSERT_TRUE(ParseFileName(f, &number, &type)) << f; 41 | ASSERT_EQ(cases[i].type, type) << f; 42 | ASSERT_EQ(cases[i].number, number) << f; 43 | } 44 | 45 | // Errors 46 | static const char* errors[] = { 47 | "", 48 | "foo", 49 | "foo-dx-100.log", 50 | ".log", 51 | "", 52 | "manifest", 53 | "CURREN", 54 | "CURRENTX", 55 | "MANIFES", 56 | "MANIFEST", 57 | "MANIFEST-", 58 | "XMANIFEST-3", 59 | "MANIFEST-3x", 60 | "LOC", 61 | "LOCKx", 62 | "LO", 63 | "LOGx", 64 | "18446744073709551616.log", 65 | "184467440737095516150.log", 66 | "100", 67 | "100.", 68 | "100.lop" 69 | }; 70 | for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { 71 | std::string f = errors[i]; 72 | ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; 73 | }; 74 | } 75 | 76 | TEST(FileNameTest, Construction) { 77 | uint64_t number; 78 | FileType type; 79 | std::string fname; 80 | 81 | fname = CurrentFileName("foo"); 82 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 83 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 84 | ASSERT_EQ(0, number); 85 | ASSERT_EQ(kCurrentFile, type); 86 | 87 | fname = LockFileName("foo"); 88 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 89 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 90 | ASSERT_EQ(0, number); 91 | ASSERT_EQ(kDBLockFile, type); 92 | 93 | fname = LogFileName("foo", 192); 94 | ASSERT_EQ("foo/", std::string(fname.data(), 4)); 95 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 96 | ASSERT_EQ(192, number); 97 | ASSERT_EQ(kLogFile, type); 98 | 99 | fname = TableFileName("bar", 200); 100 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 101 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 102 | ASSERT_EQ(200, number); 103 | ASSERT_EQ(kTableFile, type); 104 | 105 | fname = DescriptorFileName("bar", 100); 106 | ASSERT_EQ("bar/", std::string(fname.data(), 4)); 107 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 108 | ASSERT_EQ(100, number); 109 | ASSERT_EQ(kDescriptorFile, type); 110 | 111 | fname = TempFileName("tmp", 999); 112 | ASSERT_EQ("tmp/", std::string(fname.data(), 4)); 113 | ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type)); 114 | ASSERT_EQ(999, number); 115 | ASSERT_EQ(kTempFile, type); 116 | } 117 | 118 | } 119 | 120 | int main(int argc, char** argv) { 121 | return leveldb::test::RunAllTests(); 122 | } 123 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // An iterator yields a sequence of key/value pairs from a source. 6 | // The following class defines the interface. Multiple implementations 7 | // are provided by this library. In particular, iterators are provided 8 | // to access the contents of a Table or a DB. 9 | // 10 | // Multiple threads can invoke const methods on an Iterator without 11 | // external synchronization, but if any of the threads may call a 12 | // non-const method, all threads accessing the same Iterator must use 13 | // external synchronization. 14 | 15 | #ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 16 | #define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 17 | 18 | #include "leveldb/slice.h" 19 | #include "leveldb/status.h" 20 | 21 | namespace leveldb { 22 | 23 | class Iterator { 24 | public: 25 | Iterator(); 26 | virtual ~Iterator(); 27 | 28 | // An iterator is either positioned at a key/value pair, or 29 | // not valid. This method returns true iff the iterator is valid. 30 | virtual bool Valid() const = 0; 31 | 32 | // Position at the first key in the source. The iterator is Valid() 33 | // after this call iff the source is not empty. 34 | virtual void SeekToFirst() = 0; 35 | 36 | // Position at the last key in the source. The iterator is 37 | // Valid() after this call iff the source is not empty. 38 | virtual void SeekToLast() = 0; 39 | 40 | // Position at the first key in the source that at or past target 41 | // The iterator is Valid() after this call iff the source contains 42 | // an entry that comes at or past target. 43 | virtual void Seek(const Slice& target) = 0; 44 | 45 | // Moves to the next entry in the source. After this call, Valid() is 46 | // true iff the iterator was not positioned at the last entry in the source. 47 | // REQUIRES: Valid() 48 | virtual void Next() = 0; 49 | 50 | // Moves to the previous entry in the source. After this call, Valid() is 51 | // true iff the iterator was not positioned at the first entry in source. 52 | // REQUIRES: Valid() 53 | virtual void Prev() = 0; 54 | 55 | // Return the key for the current entry. The underlying storage for 56 | // the returned slice is valid only until the next modification of 57 | // the iterator. 58 | // REQUIRES: Valid() 59 | virtual Slice key() const = 0; 60 | 61 | // Return the value for the current entry. The underlying storage for 62 | // the returned slice is valid only until the next modification of 63 | // the iterator. 64 | // REQUIRES: !AtEnd() && !AtStart() 65 | virtual Slice value() const = 0; 66 | 67 | // If an error has occurred, return it. Else return an ok status. 68 | virtual Status status() const = 0; 69 | 70 | // Clients are allowed to register function/arg1/arg2 triples that 71 | // will be invoked when this iterator is destroyed. 72 | // 73 | // Note that unlike all of the preceding methods, this method is 74 | // not abstract and therefore clients should not override it. 75 | typedef void (*CleanupFunction)(void* arg1, void* arg2); 76 | void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); 77 | 78 | private: 79 | struct Cleanup { 80 | CleanupFunction function; 81 | void* arg1; 82 | void* arg2; 83 | Cleanup* next; 84 | }; 85 | Cleanup cleanup_; 86 | 87 | // No copying allowed 88 | Iterator(const Iterator&); 89 | void operator=(const Iterator&); 90 | }; 91 | 92 | // Return an empty iterator (yields nothing). 93 | extern Iterator* NewEmptyIterator(); 94 | 95 | // Return an empty iterator with the specified status. 96 | extern Iterator* NewErrorIterator(const Status& status); 97 | 98 | } 99 | 100 | #endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ 101 | -------------------------------------------------------------------------------- /leveldb/include/leveldb/cache.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // A Cache is an interface that maps keys to values. It has internal 6 | // synchronization and may be safely accessed concurrently from 7 | // multiple threads. It may automatically evict entries to make room 8 | // for new entries. Values have a specified charge against the cache 9 | // capacity. For example, a cache where the values are variable 10 | // length strings, may use the length of the string as the charge for 11 | // the string. 12 | // 13 | // A builtin cache implementation with a least-recently-used eviction 14 | // policy is provided. Clients may use their own implementations if 15 | // they want something more sophisticated (like scan-resistance, a 16 | // custom eviction policy, variable cache sizing, etc.) 17 | 18 | #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ 19 | #define STORAGE_LEVELDB_INCLUDE_CACHE_H_ 20 | 21 | #include 22 | #include "leveldb/slice.h" 23 | 24 | namespace leveldb { 25 | 26 | class Cache; 27 | 28 | // Create a new cache with a fixed size capacity. This implementation 29 | // of Cache uses a least-recently-used eviction policy. 30 | extern Cache* NewLRUCache(size_t capacity); 31 | 32 | class Cache { 33 | public: 34 | Cache() { } 35 | 36 | // Destroys all existing entries by calling the "deleter" 37 | // function that was passed to the constructor. 38 | virtual ~Cache(); 39 | 40 | // Opaque handle to an entry stored in the cache. 41 | struct Handle { }; 42 | 43 | // Insert a mapping from key->value into the cache and assign it 44 | // the specified charge against the total cache capacity. 45 | // 46 | // Returns a handle that corresponds to the mapping. The caller 47 | // must call this->Release(handle) when the returned mapping is no 48 | // longer needed. 49 | // 50 | // When the inserted entry is no longer needed, the key and 51 | // value will be passed to "deleter". 52 | virtual Handle* Insert(const Slice& key, void* value, size_t charge, 53 | void (*deleter)(const Slice& key, void* value)) = 0; 54 | 55 | // If the cache has no mapping for "key", returns NULL. 56 | // 57 | // Else return a handle that corresponds to the mapping. The caller 58 | // must call this->Release(handle) when the returned mapping is no 59 | // longer needed. 60 | virtual Handle* Lookup(const Slice& key) = 0; 61 | 62 | // Release a mapping returned by a previous Lookup(). 63 | // REQUIRES: handle must not have been released yet. 64 | // REQUIRES: handle must have been returned by a method on *this. 65 | virtual void Release(Handle* handle) = 0; 66 | 67 | // Return the value encapsulated in a handle returned by a 68 | // successful Lookup(). 69 | // REQUIRES: handle must not have been released yet. 70 | // REQUIRES: handle must have been returned by a method on *this. 71 | virtual void* Value(Handle* handle) = 0; 72 | 73 | // If the cache contains entry for key, erase it. Note that the 74 | // underlying entry will be kept around until all existing handles 75 | // to it have been released. 76 | virtual void Erase(const Slice& key) = 0; 77 | 78 | // Return a new numeric id. May be used by multiple clients who are 79 | // sharing the same cache to partition the key space. Typically the 80 | // client will allocate a new id at startup and prepend the id to 81 | // its cache keys. 82 | virtual uint64_t NewId() = 0; 83 | 84 | private: 85 | void LRU_Remove(Handle* e); 86 | void LRU_Append(Handle* e); 87 | void Unref(Handle* e); 88 | 89 | struct Rep; 90 | Rep* rep_; 91 | 92 | // No copying allowed 93 | Cache(const Cache&); 94 | void operator=(const Cache&); 95 | }; 96 | 97 | } 98 | 99 | #endif // STORAGE_LEVELDB_UTIL_CACHE_H_ 100 | -------------------------------------------------------------------------------- /leveldb/db/memtable.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/memtable.h" 6 | #include "db/dbformat.h" 7 | #include "leveldb/comparator.h" 8 | #include "leveldb/env.h" 9 | #include "leveldb/iterator.h" 10 | #include "util/coding.h" 11 | 12 | namespace leveldb { 13 | 14 | static Slice GetLengthPrefixedSlice(const char* data) { 15 | uint32_t len; 16 | const char* p = data; 17 | p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted 18 | return Slice(p, len); 19 | } 20 | 21 | MemTable::MemTable(const InternalKeyComparator& cmp) 22 | : comparator_(cmp), 23 | refs_(0), 24 | table_(comparator_, &arena_) { 25 | } 26 | 27 | MemTable::~MemTable() { 28 | assert(refs_ == 0); 29 | } 30 | 31 | size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } 32 | 33 | int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) 34 | const { 35 | // Internal keys are encoded as length-prefixed strings. 36 | Slice a = GetLengthPrefixedSlice(aptr); 37 | Slice b = GetLengthPrefixedSlice(bptr); 38 | return comparator.Compare(a, b); 39 | } 40 | 41 | // Encode a suitable internal key target for "target" and return it. 42 | // Uses *scratch as scratch space, and the returned pointer will point 43 | // into this scratch space. 44 | static const char* EncodeKey(std::string* scratch, const Slice& target) { 45 | scratch->clear(); 46 | PutVarint32(scratch, target.size()); 47 | scratch->append(target.data(), target.size()); 48 | return scratch->data(); 49 | } 50 | 51 | class MemTableIterator: public Iterator { 52 | public: 53 | explicit MemTableIterator(MemTable::Table* table) : iter_(table) { } 54 | 55 | virtual bool Valid() const { return iter_.Valid(); } 56 | virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } 57 | virtual void SeekToFirst() { iter_.SeekToFirst(); } 58 | virtual void SeekToLast() { iter_.SeekToLast(); } 59 | virtual void Next() { iter_.Next(); } 60 | virtual void Prev() { iter_.Prev(); } 61 | virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } 62 | virtual Slice value() const { 63 | Slice key_slice = GetLengthPrefixedSlice(iter_.key()); 64 | return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); 65 | } 66 | 67 | virtual Status status() const { return Status::OK(); } 68 | 69 | private: 70 | MemTable::Table::Iterator iter_; 71 | std::string tmp_; // For passing to EncodeKey 72 | 73 | // No copying allowed 74 | MemTableIterator(const MemTableIterator&); 75 | void operator=(const MemTableIterator&); 76 | }; 77 | 78 | Iterator* MemTable::NewIterator() { 79 | return new MemTableIterator(&table_); 80 | } 81 | 82 | void MemTable::Add(SequenceNumber s, ValueType type, 83 | const Slice& key, 84 | const Slice& value) { 85 | // Format of an entry is concatenation of: 86 | // key_size : varint32 of internal_key.size() 87 | // key bytes : char[internal_key.size()] 88 | // value_size : varint32 of value.size() 89 | // value bytes : char[value.size()] 90 | size_t key_size = key.size(); 91 | size_t val_size = value.size(); 92 | size_t internal_key_size = key_size + 8; 93 | const size_t encoded_len = 94 | VarintLength(internal_key_size) + internal_key_size + 95 | VarintLength(val_size) + val_size; 96 | char* buf = arena_.Allocate(encoded_len); 97 | char* p = EncodeVarint32(buf, internal_key_size); 98 | memcpy(p, key.data(), key_size); 99 | p += key_size; 100 | EncodeFixed64(p, (s << 8) | type); 101 | p += 8; 102 | p = EncodeVarint32(p, val_size); 103 | memcpy(p, value.data(), val_size); 104 | assert((p + val_size) - buf == encoded_len); 105 | table_.Insert(buf); 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /leveldb/db/log_reader.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ 6 | #define STORAGE_LEVELDB_DB_LOG_READER_H_ 7 | 8 | #include 9 | 10 | #include "db/log_format.h" 11 | #include "leveldb/slice.h" 12 | #include "leveldb/status.h" 13 | 14 | namespace leveldb { 15 | 16 | class SequentialFile; 17 | 18 | namespace log { 19 | 20 | class Reader { 21 | public: 22 | // Interface for reporting errors. 23 | class Reporter { 24 | public: 25 | virtual ~Reporter(); 26 | 27 | // Some corruption was detected. "size" is the approximate number 28 | // of bytes dropped due to the corruption. 29 | virtual void Corruption(size_t bytes, const Status& status) = 0; 30 | }; 31 | 32 | // Create a reader that will return log records from "*file". 33 | // "*file" must remain live while this Reader is in use. 34 | // 35 | // If "reporter" is non-NULL, it is notified whenever some data is 36 | // dropped due to a detected corruption. "*reporter" must remain 37 | // live while this Reader is in use. 38 | // 39 | // If "checksum" is true, verify checksums if available. 40 | // 41 | // The Reader will start reading at the first record located at physical 42 | // position >= initial_offset within the file. 43 | Reader(SequentialFile* file, Reporter* reporter, bool checksum, 44 | uint64_t initial_offset); 45 | 46 | ~Reader(); 47 | 48 | // Read the next record into *record. Returns true if read 49 | // successfully, false if we hit end of the input. May use 50 | // "*scratch" as temporary storage. The contents filled in *record 51 | // will only be valid until the next mutating operation on this 52 | // reader or the next mutation to *scratch. 53 | bool ReadRecord(Slice* record, std::string* scratch); 54 | 55 | // Returns the physical offset of the last record returned by ReadRecord. 56 | // 57 | // Undefined before the first call to ReadRecord. 58 | uint64_t LastRecordOffset(); 59 | 60 | private: 61 | SequentialFile* const file_; 62 | Reporter* const reporter_; 63 | bool const checksum_; 64 | char* const backing_store_; 65 | Slice buffer_; 66 | bool eof_; // Last Read() indicated EOF by returning < kBlockSize 67 | 68 | // Offset of the last record returned by ReadRecord. 69 | uint64_t last_record_offset_; 70 | // Offset of the first location past the end of buffer_. 71 | uint64_t end_of_buffer_offset_; 72 | 73 | // Offset at which to start looking for the first record to return 74 | uint64_t const initial_offset_; 75 | 76 | // Extend record types with the following special values 77 | enum { 78 | kEof = kMaxRecordType + 1, 79 | // Returned whenever we find an invalid physical record. 80 | // Currently there are three situations in which this happens: 81 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 82 | // * The record is a 0-length record (No drop is reported) 83 | // * The record is below constructor's initial_offset (No drop is reported) 84 | kBadRecord = kMaxRecordType + 2 85 | }; 86 | 87 | // Skips all blocks that are completely before "initial_offset_". 88 | // 89 | // Returns true on success. Handles reporting. 90 | bool SkipToInitialBlock(); 91 | 92 | // Return type, or one of the preceding special values 93 | unsigned int ReadPhysicalRecord(Slice* result); 94 | 95 | // Reports dropped bytes to the reporter. 96 | // buffer_ must be updated to remove the dropped bytes prior to invocation. 97 | void ReportCorruption(size_t bytes, const char* reason); 98 | void ReportDrop(size_t bytes, const Status& reason); 99 | 100 | // No copying allowed 101 | Reader(const Reader&); 102 | void operator=(const Reader&); 103 | }; 104 | 105 | } 106 | } 107 | 108 | #endif // STORAGE_LEVELDB_DB_LOG_READER_H_ 109 | -------------------------------------------------------------------------------- /leveldb/table/block_builder.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // BlockBuilder generates blocks where keys are prefix-compressed: 6 | // 7 | // When we store a key, we drop the prefix shared with the previous 8 | // string. This helps reduce the space requirement significantly. 9 | // Furthermore, once every K keys, we do not apply the prefix 10 | // compression and store the entire key. We call this a "restart 11 | // point". The tail end of the block stores the offsets of all of the 12 | // restart points, and can be used to do a binary search when looking 13 | // for a particular key. Values are stored as-is (without compression) 14 | // immediately following the corresponding key. 15 | // 16 | // An entry for a particular key-value pair has the form: 17 | // shared_bytes: varint32 18 | // unshared_bytes: varint32 19 | // value_length: varint32 20 | // key_delta: char[unshared_bytes] 21 | // value: char[value_length] 22 | // shared_bytes == 0 for restart points. 23 | // 24 | // The trailer of the block has the form: 25 | // restarts: uint32[num_restarts] 26 | // num_restarts: uint32 27 | // restarts[i] contains the offset within the block of the ith restart point. 28 | 29 | #include "table/block_builder.h" 30 | 31 | #include 32 | #include 33 | #include "leveldb/comparator.h" 34 | #include "leveldb/table_builder.h" 35 | #include "util/coding.h" 36 | 37 | namespace leveldb { 38 | 39 | BlockBuilder::BlockBuilder(const Options* options) 40 | : options_(options), 41 | restarts_(), 42 | counter_(0), 43 | finished_(false) { 44 | assert(options->block_restart_interval >= 1); 45 | restarts_.push_back(0); // First restart point is at offset 0 46 | } 47 | 48 | void BlockBuilder::Reset() { 49 | buffer_.clear(); 50 | restarts_.clear(); 51 | restarts_.push_back(0); // First restart point is at offset 0 52 | counter_ = 0; 53 | finished_ = false; 54 | last_key_.clear(); 55 | } 56 | 57 | size_t BlockBuilder::CurrentSizeEstimate() const { 58 | return (buffer_.size() + // Raw data buffer 59 | restarts_.size() * sizeof(uint32_t) + // Restart array 60 | sizeof(uint32_t)); // Restart array length 61 | } 62 | 63 | Slice BlockBuilder::Finish() { 64 | // Append restart array 65 | for (size_t i = 0; i < restarts_.size(); i++) { 66 | PutFixed32(&buffer_, restarts_[i]); 67 | } 68 | PutFixed32(&buffer_, restarts_.size()); 69 | finished_ = true; 70 | return Slice(buffer_); 71 | } 72 | 73 | void BlockBuilder::Add(const Slice& key, const Slice& value) { 74 | Slice last_key_piece(last_key_); 75 | assert(!finished_); 76 | assert(counter_ <= options_->block_restart_interval); 77 | assert(buffer_.empty() // No values yet? 78 | || options_->comparator->Compare(key, last_key_piece) > 0); 79 | size_t shared = 0; 80 | if (counter_ < options_->block_restart_interval) { 81 | // See how much sharing to do with previous string 82 | const size_t min_length = std::min(last_key_piece.size(), key.size()); 83 | while ((shared < min_length) && (last_key_piece[shared] == key[shared])) { 84 | shared++; 85 | } 86 | } else { 87 | // Restart compression 88 | restarts_.push_back(buffer_.size()); 89 | counter_ = 0; 90 | } 91 | const size_t non_shared = key.size() - shared; 92 | 93 | // Add "" to buffer_ 94 | PutVarint32(&buffer_, shared); 95 | PutVarint32(&buffer_, non_shared); 96 | PutVarint32(&buffer_, value.size()); 97 | 98 | // Add string delta to buffer_ followed by value 99 | buffer_.append(key.data() + shared, non_shared); 100 | buffer_.append(value.data(), value.size()); 101 | 102 | // Update state 103 | last_key_.resize(shared); 104 | last_key_.append(key.data() + shared, non_shared); 105 | assert(Slice(last_key_) == key); 106 | counter_++; 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /leveldb/port/port_android.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // See port_example.h for documentation for the following types/functions. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_PORT_ANDROID_H_ 8 | #define STORAGE_LEVELDB_PORT_PORT_ANDROID_H_ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | // Collapse the plethora of ARM flavors available to an easier to manage set 18 | // Defs reference is at https://wiki.edubuntu.org/ARM/Thumb2PortingHowto 19 | #if defined(__ARM_ARCH_6__) || \ 20 | defined(__ARM_ARCH_6J__) || \ 21 | defined(__ARM_ARCH_6K__) || \ 22 | defined(__ARM_ARCH_6Z__) || \ 23 | defined(__ARM_ARCH_6T2__) || \ 24 | defined(__ARM_ARCH_6ZK__) || \ 25 | defined(__ARM_ARCH_7__) || \ 26 | defined(__ARM_ARCH_7R__) || \ 27 | defined(__ARM_ARCH_7A__) 28 | #define ARMV6_OR_7 1 29 | #endif 30 | 31 | extern "C" { 32 | size_t fread_unlocked(void *a, size_t b, size_t c, FILE *d); 33 | size_t fwrite_unlocked(const void *a, size_t b, size_t c, FILE *d); 34 | int fflush_unlocked(FILE *f); 35 | int fdatasync (int fd); 36 | } 37 | 38 | namespace leveldb { 39 | namespace port { 40 | 41 | static const bool kLittleEndian = __BYTE_ORDER == __LITTLE_ENDIAN; 42 | 43 | class CondVar; 44 | 45 | class Mutex { 46 | public: 47 | Mutex(); 48 | ~Mutex(); 49 | 50 | void Lock(); 51 | void Unlock(); 52 | void AssertHeld() { 53 | //TODO(gabor): How can I implement this? 54 | } 55 | 56 | private: 57 | friend class CondVar; 58 | pthread_mutex_t mu_; 59 | 60 | // No copying 61 | Mutex(const Mutex&); 62 | void operator=(const Mutex&); 63 | }; 64 | 65 | class CondVar { 66 | public: 67 | explicit CondVar(Mutex* mu); 68 | ~CondVar(); 69 | void Wait(); 70 | void Signal(); 71 | void SignalAll(); 72 | private: 73 | Mutex* mu_; 74 | pthread_cond_t cv_; 75 | }; 76 | 77 | #ifndef ARMV6_OR_7 78 | // On ARM chipsets = V6 92 | #ifdef ARMV6_OR_7 93 | __asm__ __volatile__("dmb" : : : "memory"); 94 | #else 95 | pLinuxKernelMemoryBarrier(); 96 | #endif 97 | } 98 | 99 | public: 100 | AtomicPointer() { } 101 | explicit AtomicPointer(void* v) : rep_(v) { } 102 | inline void* Acquire_Load() const { 103 | void* r = rep_; 104 | MemoryBarrier(); 105 | return r; 106 | } 107 | inline void Release_Store(void* v) { 108 | MemoryBarrier(); 109 | rep_ = v; 110 | } 111 | inline void* NoBarrier_Load() const { 112 | void* r = rep_; 113 | return r; 114 | } 115 | inline void NoBarrier_Store(void* v) { 116 | rep_ = v; 117 | } 118 | }; 119 | 120 | // TODO(gabor): Implement compress 121 | inline bool Snappy_Compress( 122 | const char* input, 123 | size_t input_length, 124 | std::string* output) { 125 | return false; 126 | } 127 | 128 | // TODO(gabor): Implement uncompress 129 | inline bool Snappy_Uncompress( 130 | const char* input_data, 131 | size_t input_length, 132 | std::string* output) { 133 | return false; 134 | } 135 | 136 | inline uint64_t ThreadIdentifier() { 137 | pthread_t tid = pthread_self(); 138 | uint64_t r = 0; 139 | memcpy(&r, &tid, sizeof(r) < sizeof(tid) ? sizeof(r) : sizeof(tid)); 140 | return r; 141 | } 142 | 143 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 144 | return false; 145 | } 146 | 147 | } 148 | } 149 | 150 | #endif // STORAGE_LEVELDB_PORT_PORT_ANDROID_H_ 151 | -------------------------------------------------------------------------------- /leveldb/db/write_batch.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // WriteBatch::rep_ := 6 | // sequence: fixed64 7 | // count: fixed32 8 | // data: record[count] 9 | // record := 10 | // kTypeValue varstring varstring | 11 | // kTypeDeletion varstring 12 | // varstring := 13 | // len: varint32 14 | // data: uint8[len] 15 | 16 | #include "leveldb/write_batch.h" 17 | 18 | #include "leveldb/db.h" 19 | #include "db/dbformat.h" 20 | #include "db/memtable.h" 21 | #include "db/write_batch_internal.h" 22 | #include "util/coding.h" 23 | 24 | namespace leveldb { 25 | 26 | WriteBatch::WriteBatch() { 27 | Clear(); 28 | } 29 | 30 | WriteBatch::~WriteBatch() { } 31 | 32 | WriteBatch::Handler::~Handler() { } 33 | 34 | void WriteBatch::Clear() { 35 | rep_.clear(); 36 | rep_.resize(12); 37 | } 38 | 39 | Status WriteBatch::Iterate(Handler* handler) const { 40 | Slice input(rep_); 41 | if (input.size() < 12) { 42 | return Status::Corruption("malformed WriteBatch (too small)"); 43 | } 44 | 45 | input.remove_prefix(12); 46 | Slice key, value; 47 | int found = 0; 48 | while (!input.empty()) { 49 | found++; 50 | char tag = input[0]; 51 | input.remove_prefix(1); 52 | switch (tag) { 53 | case kTypeValue: 54 | if (GetLengthPrefixedSlice(&input, &key) && 55 | GetLengthPrefixedSlice(&input, &value)) { 56 | handler->Put(key, value); 57 | } else { 58 | return Status::Corruption("bad WriteBatch Put"); 59 | } 60 | break; 61 | case kTypeDeletion: 62 | if (GetLengthPrefixedSlice(&input, &key)) { 63 | handler->Delete(key); 64 | } else { 65 | return Status::Corruption("bad WriteBatch Delete"); 66 | } 67 | break; 68 | default: 69 | return Status::Corruption("unknown WriteBatch tag"); 70 | } 71 | } 72 | if (found != WriteBatchInternal::Count(this)) { 73 | return Status::Corruption("WriteBatch has wrong count"); 74 | } else { 75 | return Status::OK(); 76 | } 77 | } 78 | 79 | int WriteBatchInternal::Count(const WriteBatch* b) { 80 | return DecodeFixed32(b->rep_.data() + 8); 81 | } 82 | 83 | void WriteBatchInternal::SetCount(WriteBatch* b, int n) { 84 | EncodeFixed32(&b->rep_[8], n); 85 | } 86 | 87 | SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) { 88 | return SequenceNumber(DecodeFixed64(b->rep_.data())); 89 | } 90 | 91 | void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) { 92 | EncodeFixed64(&b->rep_[0], seq); 93 | } 94 | 95 | void WriteBatch::Put(const Slice& key, const Slice& value) { 96 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 97 | rep_.push_back(static_cast(kTypeValue)); 98 | PutLengthPrefixedSlice(&rep_, key); 99 | PutLengthPrefixedSlice(&rep_, value); 100 | } 101 | 102 | void WriteBatch::Delete(const Slice& key) { 103 | WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1); 104 | rep_.push_back(static_cast(kTypeDeletion)); 105 | PutLengthPrefixedSlice(&rep_, key); 106 | } 107 | 108 | namespace { 109 | class MemTableInserter : public WriteBatch::Handler { 110 | public: 111 | SequenceNumber sequence_; 112 | MemTable* mem_; 113 | 114 | virtual void Put(const Slice& key, const Slice& value) { 115 | mem_->Add(sequence_, kTypeValue, key, value); 116 | sequence_++; 117 | } 118 | virtual void Delete(const Slice& key) { 119 | mem_->Add(sequence_, kTypeDeletion, key, Slice()); 120 | sequence_++; 121 | } 122 | }; 123 | } 124 | 125 | Status WriteBatchInternal::InsertInto(const WriteBatch* b, 126 | MemTable* memtable) { 127 | MemTableInserter inserter; 128 | inserter.sequence_ = WriteBatchInternal::Sequence(b); 129 | inserter.mem_ = memtable; 130 | return b->Iterate(&inserter); 131 | } 132 | 133 | void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) { 134 | assert(contents.size() >= 12); 135 | b->rep_.assign(contents.data(), contents.size()); 136 | } 137 | 138 | } 139 | -------------------------------------------------------------------------------- /leveldb/db/dbformat_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "db/dbformat.h" 6 | #include "util/logging.h" 7 | #include "util/testharness.h" 8 | 9 | namespace leveldb { 10 | 11 | static std::string IKey(const std::string& user_key, 12 | uint64_t seq, 13 | ValueType vt) { 14 | std::string encoded; 15 | AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); 16 | return encoded; 17 | } 18 | 19 | static std::string Shorten(const std::string& s, const std::string& l) { 20 | std::string result = s; 21 | InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l); 22 | return result; 23 | } 24 | 25 | static std::string ShortSuccessor(const std::string& s) { 26 | std::string result = s; 27 | InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result); 28 | return result; 29 | } 30 | 31 | static void TestKey(const std::string& key, 32 | uint64_t seq, 33 | ValueType vt) { 34 | std::string encoded = IKey(key, seq, vt); 35 | 36 | Slice in(encoded); 37 | ParsedInternalKey decoded("", 0, kTypeValue); 38 | 39 | ASSERT_TRUE(ParseInternalKey(in, &decoded)); 40 | ASSERT_EQ(key, decoded.user_key.ToString()); 41 | ASSERT_EQ(seq, decoded.sequence); 42 | ASSERT_EQ(vt, decoded.type); 43 | 44 | ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); 45 | } 46 | 47 | class FormatTest { }; 48 | 49 | TEST(FormatTest, InternalKey_EncodeDecode) { 50 | const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; 51 | const uint64_t seq[] = { 52 | 1, 2, 3, 53 | (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, 54 | (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, 55 | (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 56 | }; 57 | for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { 58 | for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { 59 | TestKey(keys[k], seq[s], kTypeValue); 60 | TestKey("hello", 1, kTypeDeletion); 61 | } 62 | } 63 | } 64 | 65 | TEST(FormatTest, InternalKeyShortSeparator) { 66 | // When user keys are same 67 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 68 | Shorten(IKey("foo", 100, kTypeValue), 69 | IKey("foo", 99, kTypeValue))); 70 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 71 | Shorten(IKey("foo", 100, kTypeValue), 72 | IKey("foo", 101, kTypeValue))); 73 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 74 | Shorten(IKey("foo", 100, kTypeValue), 75 | IKey("foo", 100, kTypeValue))); 76 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 77 | Shorten(IKey("foo", 100, kTypeValue), 78 | IKey("foo", 100, kTypeDeletion))); 79 | 80 | // When user keys are misordered 81 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 82 | Shorten(IKey("foo", 100, kTypeValue), 83 | IKey("bar", 99, kTypeValue))); 84 | 85 | // When user keys are different, but correctly ordered 86 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 87 | Shorten(IKey("foo", 100, kTypeValue), 88 | IKey("hello", 200, kTypeValue))); 89 | 90 | // When start user key is prefix of limit user key 91 | ASSERT_EQ(IKey("foo", 100, kTypeValue), 92 | Shorten(IKey("foo", 100, kTypeValue), 93 | IKey("foobar", 200, kTypeValue))); 94 | 95 | // When limit user key is prefix of start user key 96 | ASSERT_EQ(IKey("foobar", 100, kTypeValue), 97 | Shorten(IKey("foobar", 100, kTypeValue), 98 | IKey("foo", 200, kTypeValue))); 99 | } 100 | 101 | TEST(FormatTest, InternalKeyShortestSuccessor) { 102 | ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), 103 | ShortSuccessor(IKey("foo", 100, kTypeValue))); 104 | ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue), 105 | ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); 106 | } 107 | 108 | } 109 | 110 | int main(int argc, char** argv) { 111 | return leveldb::test::RunAllTests(); 112 | } 113 | -------------------------------------------------------------------------------- /leveldb/db/filename.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include "db/filename.h" 8 | #include "db/dbformat.h" 9 | #include "leveldb/env.h" 10 | #include "util/logging.h" 11 | 12 | namespace leveldb { 13 | 14 | static std::string MakeFileName(const std::string& name, uint64_t number, 15 | const char* suffix) { 16 | char buf[100]; 17 | snprintf(buf, sizeof(buf), "/%06llu.%s", 18 | static_cast(number), 19 | suffix); 20 | return name + buf; 21 | } 22 | 23 | std::string LogFileName(const std::string& name, uint64_t number) { 24 | assert(number > 0); 25 | return MakeFileName(name, number, "log"); 26 | } 27 | 28 | std::string TableFileName(const std::string& name, uint64_t number) { 29 | assert(number > 0); 30 | return MakeFileName(name, number, "sst"); 31 | } 32 | 33 | std::string DescriptorFileName(const std::string& dbname, uint64_t number) { 34 | assert(number > 0); 35 | char buf[100]; 36 | snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", 37 | static_cast(number)); 38 | return dbname + buf; 39 | } 40 | 41 | std::string CurrentFileName(const std::string& dbname) { 42 | return dbname + "/CURRENT"; 43 | } 44 | 45 | std::string LockFileName(const std::string& dbname) { 46 | return dbname + "/LOCK"; 47 | } 48 | 49 | std::string TempFileName(const std::string& dbname, uint64_t number) { 50 | assert(number > 0); 51 | return MakeFileName(dbname, number, "dbtmp"); 52 | } 53 | 54 | std::string InfoLogFileName(const std::string& dbname) { 55 | return dbname + "/LOG"; 56 | } 57 | 58 | // Return the name of the old info log file for "dbname". 59 | std::string OldInfoLogFileName(const std::string& dbname) { 60 | return dbname + "/LOG.old"; 61 | } 62 | 63 | 64 | // Owned filenames have the form: 65 | // dbname/CURRENT 66 | // dbname/LOCK 67 | // dbname/LOG 68 | // dbname/LOG.old 69 | // dbname/MANIFEST-[0-9]+ 70 | // dbname/[0-9]+.(log|sst) 71 | bool ParseFileName(const std::string& fname, 72 | uint64_t* number, 73 | FileType* type) { 74 | Slice rest(fname); 75 | if (rest == "CURRENT") { 76 | *number = 0; 77 | *type = kCurrentFile; 78 | } else if (rest == "LOCK") { 79 | *number = 0; 80 | *type = kDBLockFile; 81 | } else if (rest == "LOG" || rest == "LOG.old") { 82 | *number = 0; 83 | *type = kInfoLogFile; 84 | } else if (rest.starts_with("MANIFEST-")) { 85 | rest.remove_prefix(strlen("MANIFEST-")); 86 | uint64_t num; 87 | if (!ConsumeDecimalNumber(&rest, &num)) { 88 | return false; 89 | } 90 | if (!rest.empty()) { 91 | return false; 92 | } 93 | *type = kDescriptorFile; 94 | *number = num; 95 | } else { 96 | // Avoid strtoull() to keep filename format independent of the 97 | // current locale 98 | uint64_t num; 99 | if (!ConsumeDecimalNumber(&rest, &num)) { 100 | return false; 101 | } 102 | Slice suffix = rest; 103 | if (suffix == Slice(".log")) { 104 | *type = kLogFile; 105 | } else if (suffix == Slice(".sst")) { 106 | *type = kTableFile; 107 | } else if (suffix == Slice(".dbtmp")) { 108 | *type = kTempFile; 109 | } else { 110 | return false; 111 | } 112 | *number = num; 113 | } 114 | return true; 115 | } 116 | 117 | Status SetCurrentFile(Env* env, const std::string& dbname, 118 | uint64_t descriptor_number) { 119 | // Remove leading "dbname/" and add newline to manifest file name 120 | std::string manifest = DescriptorFileName(dbname, descriptor_number); 121 | Slice contents = manifest; 122 | assert(contents.starts_with(dbname + "/")); 123 | contents.remove_prefix(dbname.size() + 1); 124 | std::string tmp = TempFileName(dbname, descriptor_number); 125 | Status s = WriteStringToFile(env, contents.ToString() + "\n", tmp); 126 | if (s.ok()) { 127 | s = env->RenameFile(tmp, CurrentFileName(dbname)); 128 | } 129 | if (!s.ok()) { 130 | env->DeleteFile(tmp); 131 | } 132 | return s; 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /leveldb/port/port_example.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // This file contains the specification, but not the implementations, 6 | // of the types/operations/etc. that should be defined by a platform 7 | // specific port_.h file. Use this file as a reference for 8 | // how to port this package to a new platform. 9 | 10 | #ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 11 | #define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 12 | 13 | namespace leveldb { 14 | namespace port { 15 | 16 | // TODO(jorlow): Many of these belong more in the environment class rather than 17 | // here. We should try moving them and see if it affects perf. 18 | 19 | // The following boolean constant must be true on a little-endian machine 20 | // and false otherwise. 21 | static const bool kLittleEndian = true /* or some other expression */; 22 | 23 | // ------------------ Threading ------------------- 24 | 25 | // A Mutex represents an exclusive lock. 26 | class Mutex { 27 | public: 28 | Mutex(); 29 | ~Mutex(); 30 | 31 | // Lock the mutex. Waits until other lockers have exited. 32 | // Will deadlock if the mutex is already locked by this thread. 33 | void Lock(); 34 | 35 | // Unlock the mutex. 36 | // REQUIRES: This mutex was locked by this thread. 37 | void Unlock(); 38 | 39 | // Optionally crash if this thread does not hold this mutex. 40 | // The implementation must be fast, especially if NDEBUG is 41 | // defined. The implementation is allowed to skip all checks. 42 | void AssertHeld(); 43 | }; 44 | 45 | class CondVar { 46 | public: 47 | explicit CondVar(Mutex* mu); 48 | ~CondVar(); 49 | 50 | // Atomically release *mu and block on this condition variable until 51 | // either a call to SignalAll(), or a call to Signal() that picks 52 | // this thread to wakeup. 53 | // REQUIRES: this thread holds *mu 54 | void Wait(); 55 | 56 | // If there are some threads waiting, wake up at least one of them. 57 | void Signal(); 58 | 59 | // Wake up all waiting threads. 60 | void SignallAll(); 61 | }; 62 | 63 | // A type that holds a pointer that can be read or written atomically 64 | // (i.e., without word-tearing.) 65 | class AtomicPointer { 66 | private: 67 | intptr_t rep_; 68 | public: 69 | // Initialize to arbitrary value 70 | AtomicPointer(); 71 | 72 | // Initialize to hold v 73 | explicit AtomicPointer(void* v) : rep_(v) { } 74 | 75 | // Read and return the stored pointer with the guarantee that no 76 | // later memory access (read or write) by this thread can be 77 | // reordered ahead of this read. 78 | void* Acquire_Load() const; 79 | 80 | // Set v as the stored pointer with the guarantee that no earlier 81 | // memory access (read or write) by this thread can be reordered 82 | // after this store. 83 | void Release_Store(void* v); 84 | 85 | // Read the stored pointer with no ordering guarantees. 86 | void* NoBarrier_Load() const; 87 | 88 | // Set va as the stored pointer with no ordering guarantees. 89 | void NoBarrier_Store(void* v); 90 | }; 91 | 92 | // ------------------ Compression ------------------- 93 | 94 | // Store the snappy compression of "input[0,input_length-1]" in *output. 95 | // Returns false if snappy is not supported by this port. 96 | extern bool Snappy_Compress(const char* input, size_t input_length, 97 | std::string* output); 98 | 99 | // Attempt to snappy uncompress input[0,input_length-1] into *output. 100 | // Returns true if successful, false if the input is invalid lightweight 101 | // compressed data. 102 | extern bool Snappy_Uncompress(const char* input_data, size_t input_length, 103 | std::string* output); 104 | 105 | // ------------------ Miscellaneous ------------------- 106 | 107 | // If heap profiling is not supported, returns false. 108 | // Else repeatedly calls (*func)(arg, data, n) and then returns true. 109 | // The concatenation of all "data[0,n-1]" fragments is the heap profile. 110 | extern bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); 111 | 112 | } 113 | } 114 | 115 | #endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 116 | -------------------------------------------------------------------------------- /leveldb/util/coding.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Endian-neutral encoding: 6 | // * Fixed-length numbers are encoded with least-significant byte first 7 | // * In addition we support variable length "varint" encoding 8 | // * Strings are encoded prefixed by their length in varint format 9 | 10 | #ifndef STORAGE_LEVELDB_UTIL_CODING_H_ 11 | #define STORAGE_LEVELDB_UTIL_CODING_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | #include "leveldb/slice.h" 17 | #include "port/port.h" 18 | 19 | namespace leveldb { 20 | 21 | // Standard Put... routines append to a string 22 | extern void PutFixed32(std::string* dst, uint32_t value); 23 | extern void PutFixed64(std::string* dst, uint64_t value); 24 | extern void PutVarint32(std::string* dst, uint32_t value); 25 | extern void PutVarint64(std::string* dst, uint64_t value); 26 | extern void PutLengthPrefixedSlice(std::string* dst, const Slice& value); 27 | 28 | // Standard Get... routines parse a value from the beginning of a Slice 29 | // and advance the slice past the parsed value. 30 | extern bool GetVarint32(Slice* input, uint32_t* value); 31 | extern bool GetVarint64(Slice* input, uint64_t* value); 32 | extern bool GetLengthPrefixedSlice(Slice* input, Slice* result); 33 | 34 | // Pointer-based variants of GetVarint... These either store a value 35 | // in *v and return a pointer just past the parsed value, or return 36 | // NULL on error. These routines only look at bytes in the range 37 | // [p..limit-1] 38 | extern const char* GetVarint32Ptr(const char* p,const char* limit, uint32_t* v); 39 | extern const char* GetVarint64Ptr(const char* p,const char* limit, uint64_t* v); 40 | 41 | // Returns the length of the varint32 or varint64 encoding of "v" 42 | extern int VarintLength(uint64_t v); 43 | 44 | // Lower-level versions of Put... that write directly into a character buffer 45 | // REQUIRES: dst has enough space for the value being written 46 | extern void EncodeFixed32(char* dst, uint32_t value); 47 | extern void EncodeFixed64(char* dst, uint64_t value); 48 | 49 | // Lower-level versions of Put... that write directly into a character buffer 50 | // and return a pointer just past the last byte written. 51 | // REQUIRES: dst has enough space for the value being written 52 | extern char* EncodeVarint32(char* dst, uint32_t value); 53 | extern char* EncodeVarint64(char* dst, uint64_t value); 54 | 55 | // Lower-level versions of Get... that read directly from a character buffer 56 | // without any bounds checking. 57 | 58 | inline uint32_t DecodeFixed32(const char* ptr) { 59 | if (port::kLittleEndian) { 60 | // Load the raw bytes 61 | uint32_t result; 62 | memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load 63 | return result; 64 | } else { 65 | return ((static_cast(ptr[0])) 66 | | (static_cast(ptr[1]) << 8) 67 | | (static_cast(ptr[2]) << 16) 68 | | (static_cast(ptr[3]) << 24)); 69 | } 70 | } 71 | 72 | inline uint64_t DecodeFixed64(const char* ptr) { 73 | if (port::kLittleEndian) { 74 | // Load the raw bytes 75 | uint64_t result; 76 | memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load 77 | return result; 78 | } else { 79 | uint64_t lo = DecodeFixed32(ptr); 80 | uint64_t hi = DecodeFixed32(ptr + 4); 81 | return (hi << 32) | lo; 82 | } 83 | } 84 | 85 | // Internal routine for use by fallback path of GetVarint32Ptr 86 | extern const char* GetVarint32PtrFallback(const char* p, 87 | const char* limit, 88 | uint32_t* value); 89 | inline const char* GetVarint32Ptr(const char* p, 90 | const char* limit, 91 | uint32_t* value) { 92 | if (p < limit) { 93 | uint32_t result = *(reinterpret_cast(p)); 94 | if ((result & 128) == 0) { 95 | *value = result; 96 | return p + 1; 97 | } 98 | } 99 | return GetVarint32PtrFallback(p, limit, value); 100 | } 101 | 102 | } 103 | 104 | #endif // STORAGE_LEVELDB_UTIL_CODING_H_ 105 | -------------------------------------------------------------------------------- /leveldb/table/format.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/format.h" 6 | 7 | #include "leveldb/env.h" 8 | #include "port/port.h" 9 | #include "table/block.h" 10 | #include "util/coding.h" 11 | #include "util/crc32c.h" 12 | 13 | namespace leveldb { 14 | 15 | void BlockHandle::EncodeTo(std::string* dst) const { 16 | // Sanity check that all fields have been set 17 | assert(offset_ != ~static_cast(0)); 18 | assert(size_ != ~static_cast(0)); 19 | PutVarint64(dst, offset_); 20 | PutVarint64(dst, size_); 21 | } 22 | 23 | Status BlockHandle::DecodeFrom(Slice* input) { 24 | if (GetVarint64(input, &offset_) && 25 | GetVarint64(input, &size_)) { 26 | return Status::OK(); 27 | } else { 28 | return Status::Corruption("bad block handle"); 29 | } 30 | } 31 | 32 | void Footer::EncodeTo(std::string* dst) const { 33 | #ifndef NDEBUG 34 | const size_t original_size = dst->size(); 35 | #endif 36 | metaindex_handle_.EncodeTo(dst); 37 | index_handle_.EncodeTo(dst); 38 | dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding 39 | PutFixed32(dst, static_cast(kTableMagicNumber & 0xffffffffu)); 40 | PutFixed32(dst, static_cast(kTableMagicNumber >> 32)); 41 | assert(dst->size() == original_size + kEncodedLength); 42 | } 43 | 44 | Status Footer::DecodeFrom(Slice* input) { 45 | const char* magic_ptr = input->data() + kEncodedLength - 8; 46 | const uint32_t magic_lo = DecodeFixed32(magic_ptr); 47 | const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4); 48 | const uint64_t magic = ((static_cast(magic_hi) << 32) | 49 | (static_cast(magic_lo))); 50 | if (magic != kTableMagicNumber) { 51 | return Status::InvalidArgument("not an sstable (bad magic number)"); 52 | } 53 | 54 | Status result = metaindex_handle_.DecodeFrom(input); 55 | if (result.ok()) { 56 | result = index_handle_.DecodeFrom(input); 57 | } 58 | if (result.ok()) { 59 | // We skip over any leftover data (just padding for now) in "input" 60 | const char* end = magic_ptr + 8; 61 | *input = Slice(end, input->data() + input->size() - end); 62 | } 63 | return result; 64 | } 65 | 66 | Status ReadBlock(RandomAccessFile* file, 67 | const ReadOptions& options, 68 | const BlockHandle& handle, 69 | Block** block) { 70 | *block = NULL; 71 | 72 | // Read the block contents as well as the type/crc footer. 73 | // See table_builder.cc for the code that built this structure. 74 | size_t n = static_cast(handle.size()); 75 | char* buf = new char[n + kBlockTrailerSize]; 76 | Slice contents; 77 | Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf); 78 | if (!s.ok()) { 79 | delete[] buf; 80 | return s; 81 | } 82 | if (contents.size() != n + kBlockTrailerSize) { 83 | delete[] buf; 84 | return Status::Corruption("truncated block read"); 85 | } 86 | 87 | // Check the crc of the type and the block contents 88 | const char* data = contents.data(); // Pointer to where Read put the data 89 | if (options.verify_checksums) { 90 | const uint32_t crc = crc32c::Unmask(DecodeFixed32(data + n + 1)); 91 | const uint32_t actual = crc32c::Value(data, n + 1); 92 | if (actual != crc) { 93 | delete[] buf; 94 | s = Status::Corruption("block checksum mismatch"); 95 | return s; 96 | } 97 | } 98 | 99 | switch (data[n]) { 100 | case kNoCompression: 101 | if (data != buf) { 102 | // File implementation gave us pointer to some other data. 103 | // Copy into buf[]. 104 | memcpy(buf, data, n + kBlockTrailerSize); 105 | } 106 | 107 | // Ok 108 | break; 109 | case kSnappyCompression: { 110 | std::string decompressed; 111 | if (!port::Snappy_Uncompress(data, n, &decompressed)) { 112 | delete[] buf; 113 | s = Status::Corruption("corrupted compressed block contents"); 114 | return s; 115 | } 116 | delete[] buf; // Done with uncompressed data 117 | buf = new char[decompressed.size()]; 118 | memcpy(buf, decompressed.data(), decompressed.size()); 119 | n = decompressed.size(); 120 | break; 121 | } 122 | default: 123 | delete[] buf; 124 | return Status::Corruption("bad block type"); 125 | } 126 | 127 | *block = new Block(buf, n); // Block takes ownership of buf[] 128 | return Status::OK(); 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /leveldb/util/testharness.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_TESTHARNESS_H_ 6 | #define STORAGE_LEVELDB_UTIL_TESTHARNESS_H_ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "leveldb/env.h" 12 | #include "leveldb/slice.h" 13 | #include "util/random.h" 14 | 15 | namespace leveldb { 16 | namespace test { 17 | 18 | // Run all tests registered by the TEST() macro. 19 | // Returns 0 if all tests pass. 20 | // Dies or returns a non-zero value if some test fails. 21 | extern int RunAllTests(); 22 | 23 | // Return the directory to use for temporary storage. 24 | extern std::string TmpDir(); 25 | 26 | // Return a randomization seed for this run. Typically returns the 27 | // same number on repeated invocations of this binary, but automated 28 | // runs may be able to vary the seed. 29 | extern int RandomSeed(); 30 | 31 | // An instance of Tester is allocated to hold temporary state during 32 | // the execution of an assertion. 33 | class Tester { 34 | private: 35 | bool ok_; 36 | const char* fname_; 37 | int line_; 38 | std::stringstream ss_; 39 | 40 | public: 41 | Tester(const char* f, int l) 42 | : ok_(true), fname_(f), line_(l) { 43 | } 44 | 45 | ~Tester() { 46 | if (!ok_) { 47 | fprintf(stderr, "%s:%d:%s\n", fname_, line_, ss_.str().c_str()); 48 | exit(1); 49 | } 50 | } 51 | 52 | Tester& Is(bool b, const char* msg) { 53 | if (!b) { 54 | ss_ << " Assertion failure " << msg; 55 | ok_ = false; 56 | } 57 | return *this; 58 | } 59 | 60 | Tester& IsOk(const Status& s) { 61 | if (!s.ok()) { 62 | ss_ << " " << s.ToString(); 63 | ok_ = false; 64 | } 65 | return *this; 66 | } 67 | 68 | #define BINARY_OP(name,op) \ 69 | template \ 70 | Tester& name(const X& x, const Y& y) { \ 71 | if (! (x op y)) { \ 72 | ss_ << " failed: " << x << (" " #op " ") << y; \ 73 | ok_ = false; \ 74 | } \ 75 | return *this; \ 76 | } 77 | 78 | BINARY_OP(IsEq, ==) 79 | BINARY_OP(IsNe, !=) 80 | BINARY_OP(IsGe, >=) 81 | BINARY_OP(IsGt, >) 82 | BINARY_OP(IsLe, <=) 83 | BINARY_OP(IsLt, <) 84 | #undef BINARY_OP 85 | 86 | // Attach the specified value to the error message if an error has occurred 87 | template 88 | Tester& operator<<(const V& value) { 89 | if (!ok_) { 90 | ss_ << " " << value; 91 | } 92 | return *this; 93 | } 94 | }; 95 | 96 | #define ASSERT_TRUE(c) ::leveldb::test::Tester(__FILE__, __LINE__).Is((c), #c) 97 | #define ASSERT_OK(s) ::leveldb::test::Tester(__FILE__, __LINE__).IsOk((s)) 98 | #define ASSERT_EQ(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsEq((a),(b)) 99 | #define ASSERT_NE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsNe((a),(b)) 100 | #define ASSERT_GE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGe((a),(b)) 101 | #define ASSERT_GT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGt((a),(b)) 102 | #define ASSERT_LE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLe((a),(b)) 103 | #define ASSERT_LT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLt((a),(b)) 104 | 105 | #define TCONCAT(a,b) TCONCAT1(a,b) 106 | #define TCONCAT1(a,b) a##b 107 | 108 | #define TEST(base,name) \ 109 | class TCONCAT(_Test_,name) : public base { \ 110 | public: \ 111 | void _Run(); \ 112 | static void _RunIt() { \ 113 | TCONCAT(_Test_,name) t; \ 114 | t._Run(); \ 115 | } \ 116 | }; \ 117 | bool TCONCAT(_Test_ignored_,name) = \ 118 | ::leveldb::test::RegisterTest(#base, #name, &TCONCAT(_Test_,name)::_RunIt); \ 119 | void TCONCAT(_Test_,name)::_Run() 120 | 121 | // Register the specified test. Typically not used directly, but 122 | // invoked via the macro expansion of TEST. 123 | extern bool RegisterTest(const char* base, const char* name, void (*func)()); 124 | 125 | 126 | } 127 | } 128 | 129 | #endif // STORAGE_LEVELDB_UTIL_TESTHARNESS_H_ 130 | -------------------------------------------------------------------------------- /leveldb/util/histogram.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include 7 | #include "port/port.h" 8 | #include "util/histogram.h" 9 | 10 | namespace leveldb { 11 | 12 | const double Histogram::kBucketLimit[kNumBuckets] = { 13 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 35, 40, 45, 14 | 50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200, 250, 300, 350, 400, 450, 15 | 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000, 16 | 3500, 4000, 4500, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000, 17 | 16000, 18000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 60000, 18 | 70000, 80000, 90000, 100000, 120000, 140000, 160000, 180000, 200000, 19 | 250000, 300000, 350000, 400000, 450000, 500000, 600000, 700000, 800000, 20 | 900000, 1000000, 1200000, 1400000, 1600000, 1800000, 2000000, 2500000, 21 | 3000000, 3500000, 4000000, 4500000, 5000000, 6000000, 7000000, 8000000, 22 | 9000000, 10000000, 12000000, 14000000, 16000000, 18000000, 20000000, 23 | 25000000, 30000000, 35000000, 40000000, 45000000, 50000000, 60000000, 24 | 70000000, 80000000, 90000000, 100000000, 120000000, 140000000, 160000000, 25 | 180000000, 200000000, 250000000, 300000000, 350000000, 400000000, 26 | 450000000, 500000000, 600000000, 700000000, 800000000, 900000000, 27 | 1000000000, 1200000000, 1400000000, 1600000000, 1800000000, 2000000000, 28 | 2500000000.0, 3000000000.0, 3500000000.0, 4000000000.0, 4500000000.0, 29 | 5000000000.0, 6000000000.0, 7000000000.0, 8000000000.0, 9000000000.0, 30 | 1e200, 31 | }; 32 | 33 | void Histogram::Clear() { 34 | min_ = kBucketLimit[kNumBuckets-1]; 35 | max_ = 0; 36 | num_ = 0; 37 | sum_ = 0; 38 | sum_squares_ = 0; 39 | for (int i = 0; i < kNumBuckets; i++) { 40 | buckets_[i] = 0; 41 | } 42 | } 43 | 44 | void Histogram::Add(double value) { 45 | // Linear search is fast enough for our usage in db_bench 46 | int b = 0; 47 | while (b < kNumBuckets - 1 && kBucketLimit[b] <= value) { 48 | b++; 49 | } 50 | buckets_[b] += 1.0; 51 | if (min_ > value) min_ = value; 52 | if (max_ < value) max_ = value; 53 | num_++; 54 | sum_ += value; 55 | sum_squares_ += (value * value); 56 | } 57 | 58 | double Histogram::Median() const { 59 | return Percentile(50.0); 60 | } 61 | 62 | double Histogram::Percentile(double p) const { 63 | double threshold = num_ * (p / 100.0); 64 | double sum = 0; 65 | for (int b = 0; b < kNumBuckets; b++) { 66 | sum += buckets_[b]; 67 | if (sum >= threshold) { 68 | // Scale linearly within this bucket 69 | double left_point = (b == 0) ? 0 : kBucketLimit[b-1]; 70 | double right_point = kBucketLimit[b]; 71 | double left_sum = sum - buckets_[b]; 72 | double right_sum = sum; 73 | double pos = (threshold - left_sum) / (right_sum - left_sum); 74 | double r = left_point + (right_point - left_point) * pos; 75 | if (r < min_) r = min_; 76 | if (r > max_) r = max_; 77 | return r; 78 | } 79 | } 80 | return max_; 81 | } 82 | 83 | double Histogram::Average() const { 84 | if (num_ == 0.0) return 0; 85 | return sum_ / num_; 86 | } 87 | 88 | double Histogram::StandardDeviation() const { 89 | if (num_ == 0.0) return 0; 90 | double variance = (sum_squares_ * num_ - sum_ * sum_) / (num_ * num_); 91 | return sqrt(variance); 92 | } 93 | 94 | std::string Histogram::ToString() const { 95 | std::string r; 96 | char buf[200]; 97 | snprintf(buf, sizeof(buf), 98 | "Count: %.0f Average: %.4f StdDev: %.2f\n", 99 | num_, Average(), StandardDeviation()); 100 | r.append(buf); 101 | snprintf(buf, sizeof(buf), 102 | "Min: %.4f Median: %.4f Max: %.4f\n", 103 | (num_ == 0.0 ? 0.0 : min_), Median(), max_); 104 | r.append(buf); 105 | r.append("------------------------------------------------------\n"); 106 | const double mult = 100.0 / num_; 107 | double sum = 0; 108 | for (int b = 0; b < kNumBuckets; b++) { 109 | if (buckets_[b] <= 0.0) continue; 110 | sum += buckets_[b]; 111 | snprintf(buf, sizeof(buf), 112 | "[ %7.0f, %7.0f ) %7.0f %7.3f%% %7.3f%% ", 113 | ((b == 0) ? 0.0 : kBucketLimit[b-1]), // left 114 | kBucketLimit[b], // right 115 | buckets_[b], // count 116 | mult * buckets_[b], // percentage 117 | mult * sum); // cumulative percentage 118 | r.append(buf); 119 | 120 | // Add hash marks based on percentage; 20 marks for 100%. 121 | int marks = static_cast(20*(buckets_[b] / num_) + 0.5); 122 | r.append(marks, '#'); 123 | r.push_back('\n'); 124 | } 125 | return r; 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /leveldb/util/cache_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "leveldb/cache.h" 6 | 7 | #include 8 | #include "util/coding.h" 9 | #include "util/testharness.h" 10 | 11 | namespace leveldb { 12 | 13 | // Conversions between numeric keys/values and the types expected by Cache. 14 | static std::string EncodeKey(int k) { 15 | std::string result; 16 | PutFixed32(&result, k); 17 | return result; 18 | } 19 | static int DecodeKey(const Slice& k) { 20 | assert(k.size() == 4); 21 | return DecodeFixed32(k.data()); 22 | } 23 | static void* EncodeValue(uintptr_t v) { return reinterpret_cast(v); } 24 | static int DecodeValue(void* v) { return reinterpret_cast(v); } 25 | 26 | class CacheTest { 27 | public: 28 | static CacheTest* current_; 29 | 30 | static void Deleter(const Slice& key, void* v) { 31 | current_->deleted_keys_.push_back(DecodeKey(key)); 32 | current_->deleted_values_.push_back(DecodeValue(v)); 33 | } 34 | 35 | static const int kCacheSize = 100; 36 | std::vector deleted_keys_; 37 | std::vector deleted_values_; 38 | Cache* cache_; 39 | 40 | CacheTest() : cache_(NewLRUCache(kCacheSize)) { 41 | current_ = this; 42 | } 43 | 44 | ~CacheTest() { 45 | delete cache_; 46 | } 47 | 48 | int Lookup(int key) { 49 | Cache::Handle* handle = cache_->Lookup(EncodeKey(key)); 50 | const int r = (handle == NULL) ? -1 : DecodeValue(cache_->Value(handle)); 51 | if (handle != NULL) { 52 | cache_->Release(handle); 53 | } 54 | return r; 55 | } 56 | 57 | void Insert(int key, int value, int charge = 1) { 58 | cache_->Release(cache_->Insert(EncodeKey(key), EncodeValue(value), charge, 59 | &CacheTest::Deleter)); 60 | } 61 | 62 | void Erase(int key) { 63 | cache_->Erase(EncodeKey(key)); 64 | } 65 | }; 66 | CacheTest* CacheTest::current_; 67 | 68 | TEST(CacheTest, HitAndMiss) { 69 | ASSERT_EQ(-1, Lookup(100)); 70 | 71 | Insert(100, 101); 72 | ASSERT_EQ(101, Lookup(100)); 73 | ASSERT_EQ(-1, Lookup(200)); 74 | ASSERT_EQ(-1, Lookup(300)); 75 | 76 | Insert(200, 201); 77 | ASSERT_EQ(101, Lookup(100)); 78 | ASSERT_EQ(201, Lookup(200)); 79 | ASSERT_EQ(-1, Lookup(300)); 80 | 81 | Insert(100, 102); 82 | ASSERT_EQ(102, Lookup(100)); 83 | ASSERT_EQ(201, Lookup(200)); 84 | ASSERT_EQ(-1, Lookup(300)); 85 | 86 | ASSERT_EQ(1, deleted_keys_.size()); 87 | ASSERT_EQ(100, deleted_keys_[0]); 88 | ASSERT_EQ(101, deleted_values_[0]); 89 | } 90 | 91 | TEST(CacheTest, Erase) { 92 | Erase(200); 93 | ASSERT_EQ(0, deleted_keys_.size()); 94 | 95 | Insert(100, 101); 96 | Insert(200, 201); 97 | Erase(100); 98 | ASSERT_EQ(-1, Lookup(100)); 99 | ASSERT_EQ(201, Lookup(200)); 100 | ASSERT_EQ(1, deleted_keys_.size()); 101 | ASSERT_EQ(100, deleted_keys_[0]); 102 | ASSERT_EQ(101, deleted_values_[0]); 103 | 104 | Erase(100); 105 | ASSERT_EQ(-1, Lookup(100)); 106 | ASSERT_EQ(201, Lookup(200)); 107 | ASSERT_EQ(1, deleted_keys_.size()); 108 | } 109 | 110 | TEST(CacheTest, EntriesArePinned) { 111 | Insert(100, 101); 112 | Cache::Handle* h1 = cache_->Lookup(EncodeKey(100)); 113 | ASSERT_EQ(101, DecodeValue(cache_->Value(h1))); 114 | 115 | Insert(100, 102); 116 | Cache::Handle* h2 = cache_->Lookup(EncodeKey(100)); 117 | ASSERT_EQ(102, DecodeValue(cache_->Value(h2))); 118 | ASSERT_EQ(0, deleted_keys_.size()); 119 | 120 | cache_->Release(h1); 121 | ASSERT_EQ(1, deleted_keys_.size()); 122 | ASSERT_EQ(100, deleted_keys_[0]); 123 | ASSERT_EQ(101, deleted_values_[0]); 124 | 125 | Erase(100); 126 | ASSERT_EQ(-1, Lookup(100)); 127 | ASSERT_EQ(1, deleted_keys_.size()); 128 | 129 | cache_->Release(h2); 130 | ASSERT_EQ(2, deleted_keys_.size()); 131 | ASSERT_EQ(100, deleted_keys_[1]); 132 | ASSERT_EQ(102, deleted_values_[1]); 133 | } 134 | 135 | TEST(CacheTest, EvictionPolicy) { 136 | Insert(100, 101); 137 | Insert(200, 201); 138 | 139 | // Frequently used entry must be kept around 140 | for (int i = 0; i < kCacheSize; i++) { 141 | Insert(1000+i, 2000+i); 142 | ASSERT_EQ(2000+i, Lookup(1000+i)); 143 | ASSERT_EQ(101, Lookup(100)); 144 | } 145 | ASSERT_EQ(101, Lookup(100)); 146 | ASSERT_EQ(2, deleted_keys_.size()); 147 | ASSERT_EQ(200, deleted_keys_[0]); 148 | ASSERT_EQ(201, deleted_values_[0]); 149 | } 150 | 151 | TEST(CacheTest, HeavyEntry) { 152 | Insert(100, 101); 153 | Insert(200, 201, kCacheSize); 154 | ASSERT_EQ(1, deleted_keys_.size()); 155 | ASSERT_EQ(100, deleted_keys_[0]); 156 | ASSERT_EQ(101, deleted_values_[0]); 157 | } 158 | 159 | TEST(CacheTest, NewId) { 160 | uint64_t a = cache_->NewId(); 161 | uint64_t b = cache_->NewId(); 162 | ASSERT_NE(a, b); 163 | } 164 | 165 | } 166 | 167 | int main(int argc, char** argv) { 168 | return leveldb::test::RunAllTests(); 169 | } 170 | -------------------------------------------------------------------------------- /leveldb/util/coding_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/coding.h" 6 | 7 | #include "util/testharness.h" 8 | 9 | namespace leveldb { 10 | 11 | class Coding { }; 12 | 13 | TEST(Coding, Fixed32) { 14 | std::string s; 15 | for (uint32_t v = 0; v < 100000; v++) { 16 | PutFixed32(&s, v); 17 | } 18 | 19 | const char* p = s.data(); 20 | for (uint32_t v = 0; v < 100000; v++) { 21 | uint32_t actual = DecodeFixed32(p); 22 | ASSERT_EQ(v, actual); 23 | p += sizeof(uint32_t); 24 | } 25 | } 26 | 27 | TEST(Coding, Fixed64) { 28 | std::string s; 29 | for (int power = 0; power <= 63; power++) { 30 | uint64_t v = static_cast(1) << power; 31 | PutFixed64(&s, v - 1); 32 | PutFixed64(&s, v + 0); 33 | PutFixed64(&s, v + 1); 34 | } 35 | 36 | const char* p = s.data(); 37 | for (int power = 0; power <= 63; power++) { 38 | uint64_t v = static_cast(1) << power; 39 | uint64_t actual; 40 | actual = DecodeFixed64(p); 41 | ASSERT_EQ(v-1, actual); 42 | p += sizeof(uint64_t); 43 | 44 | actual = DecodeFixed64(p); 45 | ASSERT_EQ(v+0, actual); 46 | p += sizeof(uint64_t); 47 | 48 | actual = DecodeFixed64(p); 49 | ASSERT_EQ(v+1, actual); 50 | p += sizeof(uint64_t); 51 | } 52 | } 53 | 54 | TEST(Coding, Varint32) { 55 | std::string s; 56 | for (uint32_t i = 0; i < (32 * 32); i++) { 57 | uint32_t v = (i / 32) << (i % 32); 58 | PutVarint32(&s, v); 59 | } 60 | 61 | const char* p = s.data(); 62 | const char* limit = p + s.size(); 63 | for (uint32_t i = 0; i < (32 * 32); i++) { 64 | uint32_t expected = (i / 32) << (i % 32); 65 | uint32_t actual; 66 | const char* start = p; 67 | p = GetVarint32Ptr(p, limit, &actual); 68 | ASSERT_TRUE(p != NULL); 69 | ASSERT_EQ(expected, actual); 70 | ASSERT_EQ(VarintLength(actual), p - start); 71 | } 72 | ASSERT_EQ(p, s.data() + s.size()); 73 | } 74 | 75 | TEST(Coding, Varint64) { 76 | // Construct the list of values to check 77 | std::vector values; 78 | // Some special values 79 | values.push_back(0); 80 | values.push_back(100); 81 | values.push_back(~static_cast(0)); 82 | values.push_back(~static_cast(0) - 1); 83 | for (uint32_t k = 0; k < 64; k++) { 84 | // Test values near powers of two 85 | const uint64_t power = 1ull << k; 86 | values.push_back(power); 87 | values.push_back(power-1); 88 | values.push_back(power+1); 89 | }; 90 | 91 | std::string s; 92 | for (int i = 0; i < values.size(); i++) { 93 | PutVarint64(&s, values[i]); 94 | } 95 | 96 | const char* p = s.data(); 97 | const char* limit = p + s.size(); 98 | for (int i = 0; i < values.size(); i++) { 99 | ASSERT_TRUE(p < limit); 100 | uint64_t actual; 101 | const char* start = p; 102 | p = GetVarint64Ptr(p, limit, &actual); 103 | ASSERT_TRUE(p != NULL); 104 | ASSERT_EQ(values[i], actual); 105 | ASSERT_EQ(VarintLength(actual), p - start); 106 | } 107 | ASSERT_EQ(p, limit); 108 | 109 | } 110 | 111 | TEST(Coding, Varint32Overflow) { 112 | uint32_t result; 113 | std::string input("\x81\x82\x83\x84\x85\x11"); 114 | ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(), &result) 115 | == NULL); 116 | } 117 | 118 | TEST(Coding, Varint32Truncation) { 119 | uint32_t large_value = (1u << 31) + 100; 120 | std::string s; 121 | PutVarint32(&s, large_value); 122 | uint32_t result; 123 | for (int len = 0; len < s.size() - 1; len++) { 124 | ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == NULL); 125 | } 126 | ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) != NULL); 127 | ASSERT_EQ(large_value, result); 128 | } 129 | 130 | TEST(Coding, Varint64Overflow) { 131 | uint64_t result; 132 | std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11"); 133 | ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(), &result) 134 | == NULL); 135 | } 136 | 137 | TEST(Coding, Varint64Truncation) { 138 | uint64_t large_value = (1ull << 63) + 100ull; 139 | std::string s; 140 | PutVarint64(&s, large_value); 141 | uint64_t result; 142 | for (int len = 0; len < s.size() - 1; len++) { 143 | ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == NULL); 144 | } 145 | ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) != NULL); 146 | ASSERT_EQ(large_value, result); 147 | } 148 | 149 | TEST(Coding, Strings) { 150 | std::string s; 151 | PutLengthPrefixedSlice(&s, Slice("")); 152 | PutLengthPrefixedSlice(&s, Slice("foo")); 153 | PutLengthPrefixedSlice(&s, Slice("bar")); 154 | PutLengthPrefixedSlice(&s, Slice(std::string(200, 'x'))); 155 | 156 | Slice input(s); 157 | Slice v; 158 | ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); 159 | ASSERT_EQ("", v.ToString()); 160 | ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); 161 | ASSERT_EQ("foo", v.ToString()); 162 | ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); 163 | ASSERT_EQ("bar", v.ToString()); 164 | ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); 165 | ASSERT_EQ(std::string(200, 'x'), v.ToString()); 166 | ASSERT_EQ("", input.ToString()); 167 | } 168 | 169 | } 170 | 171 | int main(int argc, char** argv) { 172 | return leveldb::test::RunAllTests(); 173 | } 174 | -------------------------------------------------------------------------------- /leveldb/table/two_level_iterator.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "table/two_level_iterator.h" 6 | 7 | #include "leveldb/table.h" 8 | #include "table/block.h" 9 | #include "table/format.h" 10 | #include "table/iterator_wrapper.h" 11 | 12 | namespace leveldb { 13 | 14 | namespace { 15 | 16 | typedef Iterator* (*BlockFunction)(void*, const ReadOptions&, const Slice&); 17 | 18 | class TwoLevelIterator: public Iterator { 19 | public: 20 | TwoLevelIterator( 21 | Iterator* index_iter, 22 | BlockFunction block_function, 23 | void* arg, 24 | const ReadOptions& options); 25 | 26 | virtual ~TwoLevelIterator(); 27 | 28 | virtual void Seek(const Slice& target); 29 | virtual void SeekToFirst(); 30 | virtual void SeekToLast(); 31 | virtual void Next(); 32 | virtual void Prev(); 33 | 34 | virtual bool Valid() const { 35 | return data_iter_.Valid(); 36 | } 37 | virtual Slice key() const { 38 | assert(Valid()); 39 | return data_iter_.key(); 40 | } 41 | virtual Slice value() const { 42 | assert(Valid()); 43 | return data_iter_.value(); 44 | } 45 | virtual Status status() const { 46 | // It'd be nice if status() returned a const Status& instead of a Status 47 | if (!index_iter_.status().ok()) { 48 | return index_iter_.status(); 49 | } else if (data_iter_.iter() != NULL && !data_iter_.status().ok()) { 50 | return data_iter_.status(); 51 | } else { 52 | return status_; 53 | } 54 | } 55 | 56 | private: 57 | void SaveError(const Status& s) { 58 | if (status_.ok() && !s.ok()) status_ = s; 59 | } 60 | void SkipEmptyDataBlocksForward(); 61 | void SkipEmptyDataBlocksBackward(); 62 | void SetDataIterator(Iterator* data_iter); 63 | void InitDataBlock(); 64 | 65 | BlockFunction block_function_; 66 | void* arg_; 67 | const ReadOptions options_; 68 | Status status_; 69 | IteratorWrapper index_iter_; 70 | IteratorWrapper data_iter_; // May be NULL 71 | // If data_iter_ is non-NULL, then "data_block_handle_" holds the 72 | // "index_value" passed to block_function_ to create the data_iter_. 73 | std::string data_block_handle_; 74 | }; 75 | 76 | TwoLevelIterator::TwoLevelIterator( 77 | Iterator* index_iter, 78 | BlockFunction block_function, 79 | void* arg, 80 | const ReadOptions& options) 81 | : block_function_(block_function), 82 | arg_(arg), 83 | options_(options), 84 | index_iter_(index_iter), 85 | data_iter_(NULL) { 86 | } 87 | 88 | TwoLevelIterator::~TwoLevelIterator() { 89 | } 90 | 91 | void TwoLevelIterator::Seek(const Slice& target) { 92 | index_iter_.Seek(target); 93 | InitDataBlock(); 94 | if (data_iter_.iter() != NULL) data_iter_.Seek(target); 95 | SkipEmptyDataBlocksForward(); 96 | } 97 | 98 | void TwoLevelIterator::SeekToFirst() { 99 | index_iter_.SeekToFirst(); 100 | InitDataBlock(); 101 | if (data_iter_.iter() != NULL) data_iter_.SeekToFirst(); 102 | SkipEmptyDataBlocksForward(); 103 | } 104 | 105 | void TwoLevelIterator::SeekToLast() { 106 | index_iter_.SeekToLast(); 107 | InitDataBlock(); 108 | if (data_iter_.iter() != NULL) data_iter_.SeekToLast(); 109 | SkipEmptyDataBlocksBackward(); 110 | } 111 | 112 | void TwoLevelIterator::Next() { 113 | assert(Valid()); 114 | data_iter_.Next(); 115 | SkipEmptyDataBlocksForward(); 116 | } 117 | 118 | void TwoLevelIterator::Prev() { 119 | assert(Valid()); 120 | data_iter_.Prev(); 121 | SkipEmptyDataBlocksBackward(); 122 | } 123 | 124 | 125 | void TwoLevelIterator::SkipEmptyDataBlocksForward() { 126 | while (data_iter_.iter() == NULL || !data_iter_.Valid()) { 127 | // Move to next block 128 | if (!index_iter_.Valid()) { 129 | SetDataIterator(NULL); 130 | return; 131 | } 132 | index_iter_.Next(); 133 | InitDataBlock(); 134 | if (data_iter_.iter() != NULL) data_iter_.SeekToFirst(); 135 | } 136 | } 137 | 138 | void TwoLevelIterator::SkipEmptyDataBlocksBackward() { 139 | while (data_iter_.iter() == NULL || !data_iter_.Valid()) { 140 | // Move to next block 141 | if (!index_iter_.Valid()) { 142 | SetDataIterator(NULL); 143 | return; 144 | } 145 | index_iter_.Prev(); 146 | InitDataBlock(); 147 | if (data_iter_.iter() != NULL) data_iter_.SeekToLast(); 148 | } 149 | } 150 | 151 | void TwoLevelIterator::SetDataIterator(Iterator* data_iter) { 152 | if (data_iter_.iter() != NULL) SaveError(data_iter_.status()); 153 | data_iter_.Set(data_iter); 154 | } 155 | 156 | void TwoLevelIterator::InitDataBlock() { 157 | if (!index_iter_.Valid()) { 158 | SetDataIterator(NULL); 159 | } else { 160 | Slice handle = index_iter_.value(); 161 | if (data_iter_.iter() != NULL && handle.compare(data_block_handle_) == 0) { 162 | // data_iter_ is already constructed with this iterator, so 163 | // no need to change anything 164 | } else { 165 | Iterator* iter = (*block_function_)(arg_, options_, handle); 166 | data_block_handle_.assign(handle.data(), handle.size()); 167 | SetDataIterator(iter); 168 | } 169 | } 170 | } 171 | 172 | } 173 | 174 | Iterator* NewTwoLevelIterator( 175 | Iterator* index_iter, 176 | BlockFunction block_function, 177 | void* arg, 178 | const ReadOptions& options) { 179 | return new TwoLevelIterator(index_iter, block_function, arg, options); 180 | } 181 | 182 | } 183 | -------------------------------------------------------------------------------- /leveldb/util/coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "util/coding.h" 6 | 7 | namespace leveldb { 8 | 9 | void EncodeFixed32(char* buf, uint32_t value) { 10 | #if __BYTE_ORDER == __LITTLE_ENDIAN 11 | memcpy(buf, &value, sizeof(value)); 12 | #else 13 | buf[0] = value & 0xff; 14 | buf[1] = (value >> 8) & 0xff; 15 | buf[2] = (value >> 16) & 0xff; 16 | buf[3] = (value >> 24) & 0xff; 17 | #endif 18 | } 19 | 20 | void EncodeFixed64(char* buf, uint64_t value) { 21 | #if __BYTE_ORDER == __LITTLE_ENDIAN 22 | memcpy(buf, &value, sizeof(value)); 23 | #else 24 | buf[0] = value & 0xff; 25 | buf[1] = (value >> 8) & 0xff; 26 | buf[2] = (value >> 16) & 0xff; 27 | buf[3] = (value >> 24) & 0xff; 28 | buf[4] = (value >> 32) & 0xff; 29 | buf[5] = (value >> 40) & 0xff; 30 | buf[6] = (value >> 48) & 0xff; 31 | buf[7] = (value >> 56) & 0xff; 32 | #endif 33 | } 34 | 35 | void PutFixed32(std::string* dst, uint32_t value) { 36 | char buf[sizeof(value)]; 37 | EncodeFixed32(buf, value); 38 | dst->append(buf, sizeof(buf)); 39 | } 40 | 41 | void PutFixed64(std::string* dst, uint64_t value) { 42 | char buf[sizeof(value)]; 43 | EncodeFixed64(buf, value); 44 | dst->append(buf, sizeof(buf)); 45 | } 46 | 47 | char* EncodeVarint32(char* dst, uint32_t v) { 48 | // Operate on characters as unsigneds 49 | unsigned char* ptr = reinterpret_cast(dst); 50 | static const int B = 128; 51 | if (v < (1<<7)) { 52 | *(ptr++) = v; 53 | } else if (v < (1<<14)) { 54 | *(ptr++) = v | B; 55 | *(ptr++) = v>>7; 56 | } else if (v < (1<<21)) { 57 | *(ptr++) = v | B; 58 | *(ptr++) = (v>>7) | B; 59 | *(ptr++) = v>>14; 60 | } else if (v < (1<<28)) { 61 | *(ptr++) = v | B; 62 | *(ptr++) = (v>>7) | B; 63 | *(ptr++) = (v>>14) | B; 64 | *(ptr++) = v>>21; 65 | } else { 66 | *(ptr++) = v | B; 67 | *(ptr++) = (v>>7) | B; 68 | *(ptr++) = (v>>14) | B; 69 | *(ptr++) = (v>>21) | B; 70 | *(ptr++) = v>>28; 71 | } 72 | return reinterpret_cast(ptr); 73 | } 74 | 75 | void PutVarint32(std::string* dst, uint32_t v) { 76 | char buf[5]; 77 | char* ptr = EncodeVarint32(buf, v); 78 | dst->append(buf, ptr - buf); 79 | } 80 | 81 | char* EncodeVarint64(char* dst, uint64_t v) { 82 | static const int B = 128; 83 | unsigned char* ptr = reinterpret_cast(dst); 84 | while (v >= B) { 85 | *(ptr++) = (v & (B-1)) | B; 86 | v >>= 7; 87 | } 88 | *(ptr++) = static_cast(v); 89 | return reinterpret_cast(ptr); 90 | } 91 | 92 | void PutVarint64(std::string* dst, uint64_t v) { 93 | char buf[10]; 94 | char* ptr = EncodeVarint64(buf, v); 95 | dst->append(buf, ptr - buf); 96 | } 97 | 98 | void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { 99 | PutVarint32(dst, value.size()); 100 | dst->append(value.data(), value.size()); 101 | } 102 | 103 | int VarintLength(uint64_t v) { 104 | int len = 1; 105 | while (v >= 128) { 106 | v >>= 7; 107 | len++; 108 | } 109 | return len; 110 | } 111 | 112 | const char* GetVarint32PtrFallback(const char* p, 113 | const char* limit, 114 | uint32_t* value) { 115 | uint32_t result = 0; 116 | for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { 117 | uint32_t byte = *(reinterpret_cast(p)); 118 | p++; 119 | if (byte & 128) { 120 | // More bytes are present 121 | result |= ((byte & 127) << shift); 122 | } else { 123 | result |= (byte << shift); 124 | *value = result; 125 | return reinterpret_cast(p); 126 | } 127 | } 128 | return NULL; 129 | } 130 | 131 | bool GetVarint32(Slice* input, uint32_t* value) { 132 | const char* p = input->data(); 133 | const char* limit = p + input->size(); 134 | const char* q = GetVarint32Ptr(p, limit, value); 135 | if (q == NULL) { 136 | return false; 137 | } else { 138 | *input = Slice(q, limit - q); 139 | return true; 140 | } 141 | } 142 | 143 | const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { 144 | uint64_t result = 0; 145 | for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { 146 | uint64_t byte = *(reinterpret_cast(p)); 147 | p++; 148 | if (byte & 128) { 149 | // More bytes are present 150 | result |= ((byte & 127) << shift); 151 | } else { 152 | result |= (byte << shift); 153 | *value = result; 154 | return reinterpret_cast(p); 155 | } 156 | } 157 | return NULL; 158 | } 159 | 160 | bool GetVarint64(Slice* input, uint64_t* value) { 161 | const char* p = input->data(); 162 | const char* limit = p + input->size(); 163 | const char* q = GetVarint64Ptr(p, limit, value); 164 | if (q == NULL) { 165 | return false; 166 | } else { 167 | *input = Slice(q, limit - q); 168 | return true; 169 | } 170 | } 171 | 172 | const char* GetLengthPrefixedSlice(const char* p, const char* limit, 173 | Slice* result) { 174 | uint32_t len; 175 | p = GetVarint32Ptr(p, limit, &len); 176 | if (p == NULL) return NULL; 177 | if (p + len > limit) return NULL; 178 | *result = Slice(p, len); 179 | return p + len; 180 | } 181 | 182 | bool GetLengthPrefixedSlice(Slice* input, Slice* result) { 183 | uint32_t len; 184 | if (GetVarint32(input, &len) && 185 | input->size() >= len) { 186 | *result = Slice(input->data(), len); 187 | input->remove_prefix(len); 188 | return true; 189 | } else { 190 | return false; 191 | } 192 | } 193 | 194 | } 195 | --------------------------------------------------------------------------------