├── .gitignore ├── .travis.yml ├── README.md ├── cpp ├── .gitignore ├── AUTHORS ├── CMakeLists.txt ├── LICENSE ├── README ├── TODO ├── babudb-config.cmake ├── include │ ├── babudb │ │ ├── babudb.h │ │ ├── buffer.h │ │ ├── database.h │ │ ├── key.h │ │ ├── log │ │ │ ├── log.h │ │ │ ├── log_iterator.h │ │ │ ├── log_section.h │ │ │ ├── log_storage.h │ │ │ ├── record_frame.h │ │ │ ├── record_iterator.h │ │ │ └── sequential_file.h │ │ ├── lookup_iterator.h │ │ ├── profiles │ │ │ ├── int_key.h │ │ │ ├── string_db.h │ │ │ └── string_key.h │ │ ├── test.h │ │ └── test_helper.h │ └── yield │ │ └── platform │ │ ├── assert.h │ │ ├── debug.h │ │ ├── directory_walker.h │ │ ├── disk_operations.h │ │ ├── exception.h │ │ ├── file.h │ │ ├── memory_mapped_file.h │ │ ├── path.h │ │ ├── platform_exception.h │ │ ├── platform_types.h │ │ ├── stat.h │ │ ├── windows.h │ │ └── yunit.h └── src │ ├── database.cpp │ ├── database_test.cpp │ ├── index │ ├── index.cpp │ ├── index.h │ ├── index_test.cpp │ ├── index_writer.cpp │ ├── index_writer.h │ ├── index_writer_test.cpp │ ├── merger.cpp │ └── merger.h │ ├── log │ ├── log.cpp │ ├── log_iterator.cpp │ ├── log_iterator_test.cpp │ ├── log_section.cpp │ ├── log_section_iterator.cpp │ ├── log_section_iterator.h │ ├── log_storage.cpp │ ├── log_test.cpp │ ├── record_frame.cpp │ ├── record_iterator.cpp │ ├── sequential_file.cpp │ └── sequential_file_test.cpp │ ├── log_index.cpp │ ├── log_index.h │ ├── log_index_test.cpp │ ├── lookup_iterator.cpp │ ├── lookup_iterator_test.cpp │ ├── merged_index.cpp │ ├── merged_index.h │ ├── merged_index_test.cpp │ ├── string_db.cpp │ ├── string_db_test.cpp │ ├── test_main.cpp │ ├── util.cpp │ ├── util.h │ └── yield_platform │ ├── directory_walker.cpp │ ├── directory_walker_test.cpp │ ├── disk_operations.cpp │ ├── disk_operations_test.cpp │ ├── file.cpp │ ├── file_test.cpp │ ├── memory_mapped_file.cpp │ ├── memory_mapped_file_test.cpp │ ├── path.cpp │ ├── path_test.cpp │ ├── platform_exception.cpp │ ├── stat.cpp │ └── stat_test.cpp └── java ├── .gitignore ├── README.md ├── babudb-core ├── AUTHORS ├── ChangeLog ├── LICENSE ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── xtreemfs │ │ │ └── babudb │ │ │ ├── BabuDBFactory.java │ │ │ ├── BabuDBImpl.java │ │ │ ├── BabuDBRequestResultImpl.java │ │ │ ├── ResponseManagerImpl.java │ │ │ ├── TransactionManagerImpl.java │ │ │ ├── api │ │ │ ├── BabuDB.java │ │ │ ├── Checkpointer.java │ │ │ ├── DatabaseManager.java │ │ │ ├── SnapshotManager.java │ │ │ ├── StaticInitialization.java │ │ │ ├── database │ │ │ │ ├── Database.java │ │ │ │ ├── DatabaseInsertGroup.java │ │ │ │ ├── DatabaseRO.java │ │ │ │ ├── DatabaseRequestListener.java │ │ │ │ ├── DatabaseRequestResult.java │ │ │ │ ├── ResultSet.java │ │ │ │ └── UserDefinedLookup.java │ │ │ ├── dev │ │ │ │ ├── BabuDBInternal.java │ │ │ │ ├── CheckpointerInternal.java │ │ │ │ ├── DatabaseInternal.java │ │ │ │ ├── DatabaseManagerInternal.java │ │ │ │ ├── ResponseManagerInternal.java │ │ │ │ ├── SnapshotManagerInternal.java │ │ │ │ ├── plugin │ │ │ │ │ └── PluginMain.java │ │ │ │ └── transaction │ │ │ │ │ ├── InMemoryProcessing.java │ │ │ │ │ ├── OperationInternal.java │ │ │ │ │ ├── TransactionInternal.java │ │ │ │ │ └── TransactionManagerInternal.java │ │ │ ├── exception │ │ │ │ └── BabuDBException.java │ │ │ ├── index │ │ │ │ └── ByteRangeComparator.java │ │ │ └── transaction │ │ │ │ ├── Operation.java │ │ │ │ ├── Transaction.java │ │ │ │ └── TransactionListener.java │ │ │ ├── config │ │ │ ├── BabuDBConfig.java │ │ │ ├── Config.java │ │ │ ├── ConfigBuilder.java │ │ │ └── PluginConfig.java │ │ │ ├── conversion │ │ │ ├── AutoConverter.java │ │ │ ├── BabuDBVersionReader.java │ │ │ ├── DBWriter.java │ │ │ └── jars │ │ │ │ └── 3.jar │ │ │ ├── index │ │ │ ├── ByteRange.java │ │ │ ├── DefaultByteRangeComparator.java │ │ │ ├── LSMTree.java │ │ │ ├── OverlayMergeIterator.java │ │ │ ├── overlay │ │ │ │ ├── MultiOverlayBufferTree.java │ │ │ │ ├── MultiOverlayStringTree.java │ │ │ │ └── MultiOverlayTree.java │ │ │ ├── reader │ │ │ │ ├── BlockReader.java │ │ │ │ ├── CompressedBlockReader.java │ │ │ │ ├── DefaultBlockReader.java │ │ │ │ ├── DiskIndex.java │ │ │ │ ├── DiskIndexIterator.java │ │ │ │ ├── DiskIndexIteratorBase.java │ │ │ │ ├── FixedLenMiniPage.java │ │ │ │ ├── InternalBufferUtil.java │ │ │ │ ├── InternalDiskIndexIterator.java │ │ │ │ ├── InternalMergeIterator.java │ │ │ │ ├── MiniPage.java │ │ │ │ ├── SearchUtil.java │ │ │ │ └── VarLenMiniPage.java │ │ │ └── writer │ │ │ │ ├── BlockWriter.java │ │ │ │ ├── CompressedBlockWriter.java │ │ │ │ ├── DefaultBlockWriter.java │ │ │ │ ├── DiskIndexWriter.java │ │ │ │ ├── SerializedBlock.java │ │ │ │ └── SerializedPage.java │ │ │ ├── log │ │ │ ├── DiskLogFile.java │ │ │ ├── DiskLogIterator.java │ │ │ ├── DiskLogger.java │ │ │ ├── LogEntry.java │ │ │ ├── LogEntryException.java │ │ │ └── SyncListener.java │ │ │ ├── lsmdb │ │ │ ├── BabuDBInsertGroup.java │ │ │ ├── BabuDBTransaction.java │ │ │ ├── CheckpointerImpl.java │ │ │ ├── DBConfig.java │ │ │ ├── DatabaseImpl.java │ │ │ ├── DatabaseManagerImpl.java │ │ │ ├── InsertRecordGroup.java │ │ │ ├── LSMDBRequest.java │ │ │ ├── LSMDBWorker.java │ │ │ ├── LSMDatabase.java │ │ │ ├── LSMLookupInterface.java │ │ │ └── LSN.java │ │ │ ├── plugin │ │ │ └── PluginLoader.java │ │ │ ├── sandbox │ │ │ ├── BabuDBBenchmark.java │ │ │ ├── BenchmarkWorkerThread.java │ │ │ ├── CLIParser.java │ │ │ ├── ContinuesRandomGenerator.java │ │ │ ├── DataGenerator.java │ │ │ ├── DiskIndexPerformanceTest.java │ │ │ ├── HugeDBTest.java │ │ │ ├── LSMTreePerformanceTest.java │ │ │ ├── RandomGenerator.java │ │ │ ├── ReplicationLongrunTestConfig.java │ │ │ ├── SampleApplication.java │ │ │ ├── SimpleDemo.java │ │ │ ├── StressTest.java │ │ │ └── longruntest.java │ │ │ ├── snapshots │ │ │ ├── BabuDBView.java │ │ │ ├── DefaultSnapshotConfig.java │ │ │ ├── DiskIndexView.java │ │ │ ├── InMemoryView.java │ │ │ ├── Snapshot.java │ │ │ ├── SnapshotConfig.java │ │ │ └── SnapshotManagerImpl.java │ │ │ └── tools │ │ │ └── DBDumpTool.java │ └── resources │ │ └── config │ │ ├── babuDB.properties │ │ └── default-config.properties │ └── test │ └── java │ └── org │ └── xtreemfs │ └── babudb │ ├── BabuDBTest.java │ ├── BackupTest.java │ ├── ConcurrencyTest.java │ ├── CopyDatabaseTest.java │ ├── TransactionTest.java │ ├── index │ ├── DiskIndexTest.java │ ├── LSMTreeTest.java │ └── MultiOverlayTreeTest.java │ ├── log │ ├── DiskLoggerTest.java │ └── LSNTest.java │ ├── sandbox │ ├── ContinuesRandomGeneratorTest.java │ └── RandomGeneratorTest.java │ └── snapshots │ └── SnapshotTest.java ├── babudb-replication ├── .gitignore ├── AUTHORS ├── ChangeLog ├── LICENSE ├── README ├── doc │ ├── babuDBReplicationSequences │ │ ├── 1PlainReplication.sqd │ │ ├── 2LogEntryRequest.sqd │ │ └── 3Load.sqd │ ├── masterLogic │ │ └── MasterRequestDispatcherLogic.acd │ └── slaveLogic │ │ ├── 1SlaveRequestDispatcherLogic.acd │ │ ├── 2ReplicationStage.acd │ │ ├── 3BasicReplicationLogic.acd │ │ ├── 4LogEntryRequestLogic.acd │ │ ├── 5InitialLoadLogic.acd │ │ └── HeartbeatThread.acd ├── pom.xml └── src │ ├── main │ ├── interface │ │ ├── Common.proto │ │ ├── GlobalTypes.proto │ │ ├── PBRPC.proto │ │ ├── RemoteAccess.proto │ │ └── Replication.proto │ └── java │ │ └── org │ │ └── xtreemfs │ │ └── babudb │ │ ├── config │ │ ├── DependencyConfig.java │ │ └── ReplicationConfig.java │ │ └── replication │ │ ├── BabuDBInterface.java │ │ ├── FleaseMessageReceiver.java │ │ ├── Layer.java │ │ ├── LockableService.java │ │ ├── Main.java │ │ ├── ReplicationManager.java │ │ ├── TopLayer.java │ │ ├── control │ │ ├── ControlLayer.java │ │ ├── ControlLayerInterface.java │ │ ├── FleaseEventListener.java │ │ ├── FleaseHolder.java │ │ ├── FleaseMessageSender.java │ │ └── TimeDriftDetector.java │ │ ├── policy │ │ ├── MasterOnly.java │ │ ├── NoRestriction.java │ │ ├── Policy.java │ │ └── WriteRestriction.java │ │ ├── proxy │ │ ├── BabuDBProxy.java │ │ ├── DatabaseManagerProxy.java │ │ ├── DatabaseProxy.java │ │ ├── ListenerWrapper.java │ │ ├── ProxyAccessClient.java │ │ ├── ProxyRequestHandler.java │ │ ├── TransactionManagerProxy.java │ │ └── operations │ │ │ ├── GetDatabaseByIdOperation.java │ │ │ ├── GetDatabaseByNameOperation.java │ │ │ ├── GetDatabasesOperation.java │ │ │ ├── LookupOperation.java │ │ │ ├── MakePersistentOperation.java │ │ │ ├── PrefixLookupOperation.java │ │ │ ├── PrefixLookupReverseOperation.java │ │ │ ├── RangeLookupOperation.java │ │ │ └── RangeLookupReverseOperation.java │ │ ├── service │ │ ├── HeartbeatThread.java │ │ ├── Pacemaker.java │ │ ├── ReplicationRequestHandler.java │ │ ├── ReplicationStage.java │ │ ├── RequestManagement.java │ │ ├── ServiceLayer.java │ │ ├── ServiceToControlInterface.java │ │ ├── SlaveView.java │ │ ├── StageRequest.java │ │ ├── accounting │ │ │ ├── LatestLSNUpdateListener.java │ │ │ ├── ParticipantsOverview.java │ │ │ ├── ParticipantsStates.java │ │ │ ├── ReplicateResponse.java │ │ │ └── StatesManipulation.java │ │ ├── clients │ │ │ ├── ClientInterface.java │ │ │ ├── ClientResponseFuture.java │ │ │ ├── ConditionClient.java │ │ │ ├── MasterClient.java │ │ │ └── SlaveClient.java │ │ ├── logic │ │ │ ├── BasicLogic.java │ │ │ ├── LoadLogic.java │ │ │ ├── Logic.java │ │ │ ├── LogicID.java │ │ │ └── RequestLogic.java │ │ └── operations │ │ │ ├── ChunkOperation.java │ │ │ ├── FleaseOperation.java │ │ │ ├── HeartbeatOperation.java │ │ │ ├── LoadOperation.java │ │ │ ├── LocalTimeOperation.java │ │ │ ├── ReplicaOperation.java │ │ │ ├── ReplicateOperation.java │ │ │ ├── StateOperation.java │ │ │ ├── SynchronizeOperation.java │ │ │ └── VolatileStateOperation.java │ │ └── transmission │ │ ├── ErrorCode.java │ │ ├── FileIO.java │ │ ├── FileIOInterface.java │ │ ├── TransmissionLayer.java │ │ ├── TransmissionToServiceInterface.java │ │ ├── client │ │ ├── ClientFactory.java │ │ ├── ProxyAccessClientAdapter.java │ │ └── ReplicationClientAdapter.java │ │ └── dispatcher │ │ ├── Operation.java │ │ ├── Request.java │ │ ├── RequestControl.java │ │ ├── RequestDispatcher.java │ │ └── RequestHandler.java │ └── test │ ├── java │ └── org │ │ └── xtreemfs │ │ └── babudb │ │ ├── mock │ │ ├── BabuDBMock.java │ │ ├── CheckpointerMock.java │ │ ├── DatabaseManagerMock.java │ │ ├── DatabaseMock.java │ │ ├── Mock.java │ │ ├── RequestHandlerMock.java │ │ └── TransactionManagerMock.java │ │ └── replication │ │ ├── IntegrationTest.java │ │ ├── ReusableBufferUsageTest.java │ │ ├── TestParameters.java │ │ ├── service │ │ ├── HeartbeatTest.java │ │ └── operations │ │ │ ├── MasterReplicationOperationsTest.java │ │ │ └── SlaveReplicationOperationsTest.java │ │ └── transmission │ │ ├── DirectFileIOTest.java │ │ ├── PBRPCTest.java │ │ └── RequestHandlerTest.java │ └── resources │ └── config │ ├── replication.properties │ ├── replication_server0.test │ ├── replication_server1.test │ └── replication_server2.test └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | .classpath 4 | .project 5 | .settings 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **BabuDB** is an embedded non-relational database system. Its lean and simple design allows it to persistently store large amounts of key-value pairs without the overhead and complexity of similar approaches such as BerkeleyDB. 2 | 3 | Key features: 4 | 5 | * Support for large-scale databases that exceed the system's main memory 6 | * Efficient crash recovery 7 | * Snapshots and asynchronous dumps 8 | * Prefix and range lookups 9 | * Transparent replication with tuneable consistency/performance trade-offs 10 | * BabuDB has been independently implemented for Java and C++ (Win32/Linux). APIs and database formats of both implementations are not compatible. 11 | 12 | Much of the simplicity and efficiency of the BabuDB design comes from the use of small mutable overlay-trees (also known as LSM-trees) layered on a larger immutable memory-mapped on-disk index, an architecture that was made popular by Google's BigTable?. 13 | BabuDB is used as the database engine in the replicated metadata server of the XtreemFS file system. 14 | 15 | You can contact the BabuDB developers at: babudb@googlegroups.com 16 | 17 | [![Build Status](https://travis-ci.org/xtreemfs/babudb.svg?branch=master)](https://travis-ci.org/xtreemfs/babudb) 18 | -------------------------------------------------------------------------------- /cpp/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | -------------------------------------------------------------------------------- /cpp/AUTHORS: -------------------------------------------------------------------------------- 1 | Felix Hupfeld (Zuse Institute Berlin) 2 | Jan Stender (Zuse Institute Berlin) 3 | Bjoern Kolbeck (Zuse Institute Berlin) 4 | Mikael Hoegqvist (Zuse Institute Berlin) -------------------------------------------------------------------------------- /cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8) 2 | project (babudb) 3 | 4 | find_path(SRC_DIR CMakeLists.txt PATHS .) 5 | message(STATUS "babudb root directory: " ${SRC_DIR}) 6 | # Important for installing YIELD_platform to lib/ 7 | set(CMAKE_INSTALL_PREFIX ${SRC_DIR}) 8 | 9 | include_directories(include/ src/) 10 | 11 | # Get all files 12 | file(GLOB ALL_FILES 13 | src/*.cpp 14 | src/log/*.cpp 15 | src/index/*.cpp 16 | src/yield_platform/*.cpp 17 | src/*.h 18 | src/log/*.h 19 | src/index/*.h 20 | src/yield_platform/*.h 21 | include/babudb/log/*.h 22 | include/babudb/index/*.h 23 | include/babudb/yield_platform/*.h) 24 | 25 | # Separate code and tests 26 | set(SRC_FILES "") 27 | set(TEST_FILES "") 28 | foreach(FILE ${ALL_FILES}) 29 | if(FILE MATCHES ".*_test.cpp") 30 | list(APPEND TEST_FILES ${FILE}) 31 | else() 32 | list(APPEND SRC_FILES ${FILE}) 33 | endif() 34 | endforeach(FILE) 35 | 36 | add_library(babudb ${SRC_FILES}) 37 | 38 | source_group(platform ".*platform.*") 39 | source_group(log ".*/log/.*") 40 | source_group(index ".*/index/.*") 41 | 42 | list(APPEND TEST_FILES "src/test_main.cpp") 43 | add_executable(babudb_tests ${TEST_FILES}) 44 | target_link_libraries(babudb_tests babudb) 45 | 46 | if (MSVC) 47 | set_target_properties(babudb PROPERTIES COMPILE_FLAGS "/MTd") 48 | set_target_properties(babudb_tests PROPERTIES COMPILE_FLAGS "/MTd") 49 | endif (MSVC) 50 | if (CMAKE_COMPILER_IS_GNUCXX) 51 | set_target_properties(babudb PROPERTIES COMPILE_FLAGS "-Wall -std=c++0x") 52 | set_target_properties(babudb_tests PROPERTIES COMPILE_FLAGS "-Wall -std=c++0x") 53 | target_link_libraries(babudb_tests pthread rt) 54 | endif (CMAKE_COMPILER_IS_GNUCXX) 55 | 56 | install(TARGETS babudb babudb_tests 57 | RUNTIME DESTINATION "${SRC_DIR}/bin" 58 | LIBRARY DESTINATION "${SRC_DIR}/lib" 59 | ARCHIVE DESTINATION "${SRC_DIR}/lib") 60 | -------------------------------------------------------------------------------- /cpp/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008 Felix Hupfeld, Jan Stender, Bjoern Kolbeck, 2 | Mikael Hoegqvist, Zuse Institute Berlin. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above 10 | copyright notice, this list of conditions and the 11 | following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials 15 | provided with the distribution. 16 | * Neither the name of the Zuse Institute Berlin nor the 17 | names of its contributors may be used to endorse or promote 18 | products derived from this software without specific prior 19 | written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /cpp/README: -------------------------------------------------------------------------------- 1 | -- General 2 | 3 | babudb/cpp is a library that implements several database abstractions that can 4 | be combined to build an embedded database. The glue code for building a 5 | non-replicated database with string-type index keys is provided. 6 | 7 | babudb is built around the concept of log-structured merge (LSM) trees. A LSM 8 | tree is an index that consists of an immutable persistent base index and one or 9 | several overlays. In combination the base and its overlays yield the current 10 | state of the index. The oldest overlay index can be merged with the base index 11 | in order to generated a more recent base. 12 | 13 | babudb manages the consistency of indices with the write-ahead operation log. It 14 | does not define an operation semantics by itself, but merely uses the concept of 15 | log-sequence numbers (LSNs) to associate logged operations with index state. 16 | 17 | For more information please refer to the Database class in database.h. 18 | 19 | A simple database with several indices that map string keys to string values 20 | can be found in profiles/string_db.h 21 | 22 | 23 | -- Strength and Weaknesses 24 | 25 | babudb is designed as a light-weight database system around an 26 | application-defined log format. If you need full control over the log, babudb 27 | is for you. 28 | 29 | As babudb comes with wrapper classes for a simple mult-index database, it can 30 | also help applications that need a light-weight BSD-licensed embedded database. 31 | 32 | Features: 33 | * full control over the operations log 34 | * light-weight and BSD-licensed 35 | * multi-platform, no dependencies 36 | 37 | babudb's on-disk formats are still pretty basic and lack several features, 38 | mostly around compression, but also around corruption handling. They will 39 | change. 40 | 41 | Therefore don't use babudb if you need: 42 | * a future-proof on-disk format 43 | * thread or process-level concurrency handling 44 | * a mature code base 45 | * a query engine 46 | * large databases on 32 bit systems 47 | * a relational data model 48 | 49 | 50 | --- Building 51 | 52 | babudb/cpp's build system is based on CMake (http://www.cmake.com). With CMake 53 | you can generate build or project files for your favorite build environment 54 | and platform. 55 | 56 | CMake exports two targets: a library for linking with your applications and a 57 | unit test executable. 58 | 59 | babudb contains yield_platform, an old snapshot of the platform foundation 60 | classes of yield (http://yield.googlecode.com). 61 | -------------------------------------------------------------------------------- /cpp/TODO: -------------------------------------------------------------------------------- 1 | == General 2 | - implement multi-value keys (needs Remove(key, value)) 3 | - get rid of LogStats 4 | 5 | == ImmutableIndex 6 | - change format from current set of key records to single record with (relative) offset array (saves record frames) 7 | - add gap compression for keys 8 | - add entropy compression for values 9 | 10 | == LogIndex 11 | - Put keys in tree nodes (better cache locality) 12 | 13 | == Log 14 | - remove LSN support in the log (should be done at user-level) -------------------------------------------------------------------------------- /cpp/babudb-config.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtreemfs/babudb/324aad93030066ab4fe1b57117baf52142d59813/cpp/babudb-config.cmake -------------------------------------------------------------------------------- /cpp/include/babudb/babudb.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #include "babudb/database.h" 5 | #include "babudb/key.h" 6 | #include "babudb/buffer.h" 7 | #include "babudb/log/log_iterator.h" 8 | #include "babudb/lookup_iterator.h" 9 | -------------------------------------------------------------------------------- /cpp/include/babudb/key.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #ifndef BABUDB_KEYORDER_H 10 | #define BABUDB_KEYORDER_H 11 | 12 | #include "babudb/buffer.h" 13 | 14 | namespace babudb { 15 | 16 | class KeyOrder { 17 | public: 18 | /* a strict less than operator < */ 19 | virtual bool less(const Buffer& l, const Buffer& r) const = 0; 20 | }; 21 | 22 | /* The operator that implements less with prefix matches for std::map using KeyOrder */ 23 | class MapCompare { 24 | public: 25 | explicit MapCompare(const KeyOrder& order) : order(order) {} 26 | bool operator () (const Buffer& l, const Buffer& r) const { 27 | return order.less(l,r); 28 | } 29 | 30 | private: 31 | const KeyOrder& order; 32 | }; 33 | 34 | 35 | /*inline bool MapCompare::operator () (const std::pair& l, const std::pair& r) const { 36 | // ASSERT_FALSE(l.second && r.second, "Map Wrong"); 37 | 38 | if(l.second && order.match(r.first,l.first)) // l is prefix key 39 | return false; 40 | else if(r.second && order.match(l.first,r.first)) // r is prefix key 41 | return false; 42 | else 43 | return order.less(l.first,r.first); 44 | } 45 | */ 46 | 47 | /* The operator that implements simple less for std::map using KeyOrder */ 48 | class SimpleMapCompare { 49 | public: 50 | explicit SimpleMapCompare(KeyOrder& order) : order(order) {} 51 | bool operator () (const Buffer& l, const Buffer& r) const { 52 | return order.less(l,r); 53 | } 54 | 55 | private: 56 | KeyOrder& order; 57 | }; 58 | 59 | } 60 | 61 | #endif 62 | 63 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/log.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // The Log bundles a series of LogSections with increasing LSNs. The Log classes 10 | // do not keep a record of the current LSN. 11 | // Example usage: 12 | // Log log("/path/mylog"); 13 | // log.LoadSections(0, true); 14 | // ... iterate/recover and find last lsn ... 15 | // LogSection* tail = log.GetTail(last_lsn + 1); 16 | // 17 | // The log is stored as a set of files whose file names contain the 18 | // minimal LSN: prefix_.log 19 | // 20 | // LSNs don't need to be contiguous, but must increase between log commits. 21 | // 22 | 23 | #ifndef BABUDB_LOG_LOG_H 24 | #define BABUDB_LOG_LOG_H 25 | 26 | #include 27 | #include 28 | 29 | #include "babudb/log/log_section.h" 30 | #include "babudb/log/log_iterator.h" 31 | 32 | namespace babudb { 33 | 34 | class Log { 35 | public: 36 | // Construct a volatile in-memory log 37 | explicit Log(Buffer data); 38 | // A log file with the specific prefix. Individual sections 39 | // will be named /na/me/prefix_.log 40 | explicit Log(const std::string& name_prefix); 41 | // Also Close()s the log 42 | ~Log(); 43 | 44 | // Load all log sections that contain at least records including 45 | // starting_from_lsn. 46 | void Open(lsn_t starting_from_lsn); 47 | // Close all log sections 48 | void Close(); 49 | 50 | // Cleanup log by moving all 51 | void Cleanup(lsn_t to_lsn, const std::string& to); 52 | 53 | // Get a writable tail. Creates a new LogSection if there is none. 54 | LogSection* GetTail(babudb::lsn_t next_lsn); 55 | // Force creation of a new writable tail. 56 | void AdvanceTail(); 57 | 58 | typedef LogIterator iterator; 59 | // An iterator starting at the first record in the log. 60 | iterator* First() const; 61 | // An iterator starting at the last record in the log. 62 | iterator* Last() const; 63 | 64 | int NumberOfSections() const; 65 | 66 | private: 67 | std::vector sections; 68 | LogSection* tail; 69 | std::string name_prefix; 70 | }; 71 | 72 | } // namespace babudb 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/log_iterator.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef BABUDB_LOG_LOGITERATOR_H 5 | #define BABUDB_LOG_LOGITERATOR_H 6 | 7 | #include 8 | 9 | #include "babudb/buffer.h" 10 | #include "babudb/log/record_iterator.h" 11 | 12 | namespace babudb { 13 | class LogSectionIterator; 14 | 15 | class LogIterator { 16 | public: 17 | ~LogIterator(); 18 | 19 | bool GetNext(); 20 | bool GetPrevious(); 21 | 22 | bool operator != (const LogIterator&) const; 23 | bool operator == (const LogIterator&) const; 24 | 25 | Buffer operator * () const; 26 | Buffer AsData() const { 27 | return this->operator *(); 28 | } 29 | Buffer GetOperationWithFrame() const; 30 | RecordIterator GetRecordIterator() const { 31 | return record_iterator; 32 | } 33 | bool IsValid() const; 34 | 35 | private: 36 | LogIterator(LogSectionIterator* current_section); 37 | 38 | friend class Log; 39 | static LogIterator* First(LogSectionIterator* first_section); 40 | static LogIterator* Last(LogSectionIterator* last_section); 41 | 42 | std::auto_ptr current_section; 43 | RecordIterator record_iterator; 44 | }; 45 | 46 | } 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/log_section.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // The LogSection augments the records of a SequentialFile with LSNs. 10 | 11 | #ifndef BABUDB_LOG_LOGSECTION_H 12 | #define BABUDB_LOG_LOGSECTION_H 13 | 14 | #include "babudb/log/sequential_file.h" 15 | #include "babudb/buffer.h" 16 | 17 | namespace babudb { 18 | 19 | class LogStorage; 20 | 21 | // The interface to an application log record 22 | class Serializable { 23 | public: 24 | virtual size_t GetSize() const = 0; 25 | virtual void Serialize(const Buffer& buffer) const = 0; 26 | virtual int GetType() const { return 0; } 27 | virtual ~Serializable() {} 28 | }; 29 | 30 | class LogSection : public SequentialFile { 31 | public: 32 | LogSection(LogStorage*, lsn_t first); 33 | 34 | lsn_t getFirstLSN() const; 35 | 36 | // Append entry, start a new transaction if necessary 37 | void Append(const Serializable& entry); 38 | // Make the current transaction durable 39 | void Commit(); 40 | void Erase(const iterator& it); 41 | 42 | private: 43 | lsn_t first_lsn; // the first lsn in this file 44 | }; 45 | 46 | } // namespace babudb 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/log_storage.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2010, Felix Hupfeld 4 | // Licensed under the BSD License, see LICENSE file for details. 5 | // 6 | // Author: Felix Hupfeld (felix@storagebox.org) 7 | 8 | // Abstract from logs in files and in memory 9 | 10 | #ifndef BABUDB_LOGSTORAGE_H 11 | #define BABUDB_LOGSTORAGE_H 12 | 13 | #include "babudb/buffer.h" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | namespace yield { 20 | class MemoryMappedFile; 21 | } 22 | 23 | namespace babudb { 24 | 25 | class LogStorage { 26 | public: 27 | LogStorage() : size(0), start(NULL) {} 28 | virtual ~LogStorage() {} 29 | 30 | inline char* Start() { return start; } 31 | inline char* End() { return start + size; } 32 | inline size_t Size() { return size; } 33 | 34 | virtual void WriteBack() = 0; 35 | virtual void WriteBack(void* ptr, size_t length) = 0; 36 | 37 | virtual void Resize(size_t new_size) = 0; 38 | virtual bool Close() = 0; 39 | virtual bool IsWritable() = 0; 40 | 41 | protected: 42 | size_t size; 43 | char* start; 44 | }; 45 | 46 | class VolatileLogStorage : public LogStorage { 47 | public: 48 | VolatileLogStorage(Buffer initial_data); 49 | VolatileLogStorage(size_t initial_size); 50 | ~VolatileLogStorage(); 51 | 52 | virtual void WriteBack() {} 53 | virtual void WriteBack(void* ptr, size_t length) {} 54 | 55 | virtual void Resize(size_t new_size); 56 | virtual bool Close() { return true; } 57 | virtual bool IsWritable() { return false; } 58 | 59 | // Do not delete underlying memory at destruction time 60 | void KeepData() { keep_data = true; } 61 | protected: 62 | bool keep_data; 63 | }; 64 | 65 | class PersistentLogStorage : public LogStorage { 66 | public: 67 | static PersistentLogStorage* Open(const std::string& name); 68 | static PersistentLogStorage* OpenReadOnly(const std::string& name); 69 | 70 | virtual void WriteBack(); 71 | virtual void WriteBack(void* ptr, size_t length); 72 | 73 | virtual void Resize(size_t new_size); 74 | virtual bool Close(); 75 | virtual bool IsWritable(); 76 | protected: 77 | PersistentLogStorage(yield::MemoryMappedFile*); 78 | std::auto_ptr memory; 79 | }; 80 | 81 | } // namespace babudb 82 | 83 | #endif // BABUDB_LOGSTORAGE_H 84 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/record_iterator.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // A RecordIterator is a container for the state necessary to iterate over the Records in a SequentialFile 10 | 11 | #ifndef LOG__RECORD_ITERATOR_H 12 | #define LOG__RECORD_ITERATOR_H 13 | 14 | #include 15 | #include "babudb/log/record_frame.h" 16 | 17 | namespace babudb { 18 | 19 | class RecordFrame; 20 | class Buffer; 21 | class SequentialFile; 22 | 23 | class RecordIterator { 24 | public: 25 | RecordIterator(const RecordIterator& other); 26 | RecordIterator(); 27 | 28 | static RecordIterator First(void* start, size_t size); 29 | static RecordIterator Last(void* start, size_t size); 30 | 31 | RecordFrame* GetNext(); 32 | RecordFrame* GetPrevious(); 33 | 34 | bool operator != (const RecordIterator& other) const; 35 | bool operator == (const RecordIterator& other) const; 36 | 37 | // The following operations refer to the current record as 38 | // returned by GetNext/GetPrevious 39 | void* operator * () const; 40 | size_t GetSize() const; 41 | RecordFrame* GetRecord() const; 42 | Buffer AsData() const; 43 | bool IsValid() const; 44 | 45 | protected: 46 | friend class SequentialFile; 47 | RecordIterator(void* start, size_t size, RecordFrame* pos); 48 | 49 | RecordFrame* FindNextRecord(RecordFrame*) const; 50 | RecordFrame* FindPreviousRecord(RecordFrame*) const; 51 | 52 | bool IsCompatible(const RecordIterator& other) const; 53 | bool IsValidPosition(RecordFrame* pos) const; 54 | char* RegionEnd() const; 55 | 56 | RecordFrame* current; 57 | void* region_start; 58 | size_t region_size; 59 | }; 60 | 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /cpp/include/babudb/log/sequential_file.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // A SequentialFile is a sequentially written series of Records. Supports transactional appends. 10 | 11 | #ifndef BABUDB_LOG_SEQUENTIALFILE_H 12 | #define BABUDB_LOG_SEQUENTIALFILE_H 13 | 14 | #include 15 | #include 16 | 17 | #include "babudb/log/record_frame.h" 18 | #include "babudb/log/record_iterator.h" 19 | 20 | namespace babudb { 21 | class RecordIterator; 22 | class LogStats; 23 | class LogStorage; 24 | 25 | typedef uint64_t offset_t; 26 | static const offset_t INVALID_OFFSET = 0xFFFFffffFFFFffffULL; 27 | 28 | /** An append-only file of records. 29 | */ 30 | 31 | #define SEQUENTIALFILE_DB_VERSION 0x01 32 | 33 | class SequentialFile 34 | { 35 | public: 36 | typedef class RecordFrame Record; 37 | typedef class RecordIterator iterator; 38 | 39 | explicit SequentialFile(LogStorage*); 40 | ~SequentialFile(); 41 | 42 | void close(); 43 | unsigned short getVersion() { return database_version; } 44 | void writeBack( Record* ); 45 | void writeBack(); 46 | 47 | void* getFreeSpace(size_t); 48 | void enlarge(); 49 | void truncate(); 50 | 51 | iterator First() const; 52 | iterator Last() const; 53 | iterator at(void* pointer) const; // payload pointer 54 | iterator at(Record* record) const; 55 | iterator at(offset_t offset) const; 56 | 57 | bool empty(); 58 | bool isWritable(); 59 | 60 | void frameData(void* location, size_t size); 61 | void* append(size_t size); 62 | void AppendRaw(void* data, size_t size); 63 | void moveRecord( offset_t at, offset_t to ); 64 | void erase( offset_t ); 65 | 66 | void commit(); 67 | unsigned int rollback(); 68 | 69 | void* offset2pointer( offset_t offset ) const; 70 | offset_t pointer2offset( void* ) const; 71 | 72 | Record* offset2record( offset_t offset ) const; 73 | offset_t record2offset( Record* ) const; 74 | 75 | bool isValid(Record* record); 76 | void setFlush(bool do_flush); 77 | void compact(); 78 | 79 | private: 80 | int initialize(); 81 | offset_t findNextAllocatedWord(offset_t); 82 | bool assertValidRecordChain( void* ); 83 | 84 | void copyRecord( Record*, void* ); 85 | 86 | std::auto_ptr memory; 87 | 88 | offset_t next_write_offset; 89 | unsigned short database_version; 90 | }; 91 | 92 | } 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /cpp/include/babudb/lookup_iterator.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, 2010 Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // The overlay lookup logic 10 | 11 | #ifndef BABUDB_LOOKUPITERATOR_H 12 | #define BABUDB_LOOKUPITERATOR_H 13 | 14 | #include 15 | #include 16 | 17 | #include "babudb/key.h" 18 | #include "babudb/buffer.h" 19 | 20 | namespace babudb { 21 | 22 | class LogIndex; 23 | class ImmutableIndex; 24 | class Buffer; 25 | class ImmutableIndexIterator; 26 | 27 | class LookupIterator { 28 | public: 29 | explicit LookupIterator(const KeyOrder& order); 30 | LookupIterator( 31 | const std::vector& idx, ImmutableIndex* iidx, 32 | const KeyOrder& order, const Buffer& start_key, 33 | const Buffer& end_key); 34 | LookupIterator( 35 | const std::vector& idx, ImmutableIndex* iidx, 36 | const KeyOrder& order); 37 | LookupIterator(const LookupIterator& other); 38 | 39 | ~LookupIterator(); 40 | 41 | void operator ++ (); 42 | std::pair operator * () const; 43 | 44 | bool hasMore() const; 45 | 46 | private: 47 | void InitializeOverlays(const std::vector&); 48 | void findMinimalIterator(); 49 | void advanceIterator(int); 50 | void assureNonDeletedCursor(); 51 | void CheckInvariant(); 52 | void DebugPrint(); 53 | 54 | int current_depth; 55 | std::vector::const_iterator> logi_it; // MSI to LSI 56 | const KeyOrder& order; 57 | const Buffer* end_key; 58 | std::vector logi; 59 | ImmutableIndex* iidx; 60 | ImmutableIndexIterator* iidx_it; 61 | }; 62 | 63 | } 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /cpp/include/babudb/profiles/int_key.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #ifndef BABUDB_INTCONFIGURATION_H 10 | #define BABUDB_INTCONFIGURATION_H 11 | 12 | #include "babudb/key.h" 13 | 14 | namespace babudb { 15 | 16 | class Int32Order : public KeyOrder { 17 | public: 18 | virtual bool less(const Buffer& l, const Buffer& r) const { 19 | return *(int*)l.data < *(int*)r.data; 20 | } 21 | 22 | virtual bool match(const Buffer& l, const Buffer& r) const { 23 | return *(int*)l.data == *(int*)r.data; 24 | } 25 | }; 26 | 27 | /* Represents an array of N fixed-size columns of type T. 28 | 29 | TODO: replace with recursive template 30 | */ 31 | 32 | template 33 | class MultiOrder : public KeyOrder { 34 | public: 35 | virtual bool less(const Buffer& l, const Buffer& r) const { 36 | T* left = (T*)l.data; 37 | T* right = (T*)r.data; 38 | 39 | for(i=N-1; i >= 0; i--) { 40 | if(left[i] >= right[i]) 41 | return false; 42 | } 43 | 44 | return true; 45 | } 46 | 47 | virtual bool match(const Buffer& l, const Buffer& r) const { 48 | T* left = (T*)l.data; 49 | T* right = (T*)r.data; 50 | 51 | for(i=N-1; i >= 0; i--) { 52 | if(left[i] != right[i]) 53 | return false; 54 | } 55 | 56 | return true; 57 | } 58 | }; 59 | 60 | } 61 | #endif 62 | -------------------------------------------------------------------------------- /cpp/include/babudb/profiles/string_db.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // A database for string-type data. Also acts as an example on 10 | // how to put to together babudb's classes to a complete 11 | // embedded database. 12 | 13 | #ifndef BABUDB_STRING_DB_H 14 | #define BABUDB_STRING_DB_H 15 | 16 | #include "babudb/profiles/string_key.h" 17 | 18 | #include 19 | using std::string; 20 | 21 | namespace babudb { 22 | 23 | class Database; 24 | class Log; 25 | 26 | class StringDB { 27 | public: 28 | static StringDB* Open(const string& name, const std::vector& indices); 29 | ~StringDB(); 30 | 31 | void Add(const string& index_name, const string& key, const string& value); 32 | void Remove(const string& index_name, const string& key); 33 | void Commit(); 34 | 35 | string Lookup(const string& index, const string& key); 36 | // TODO: hide the LookupIterator 37 | LookupIterator Lookup(const string& index, const string& lower, const string& upper); 38 | 39 | // Merge the log and indices; truncate the log; prefix the obsolete data with "to" 40 | void Compact(const string& to); 41 | 42 | private: 43 | StringDB(const string& name); 44 | StringOrder key_order; 45 | 46 | Database* db; 47 | Log* log; 48 | std::vector index_names; 49 | string name; 50 | }; 51 | 52 | } 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /cpp/include/babudb/test.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef BABUDB_TEST_H 5 | #define BABUDB_TEST_H 6 | 7 | #include "yield/platform/yunit.h" 8 | #include "yield/platform/platform_exception.h" 9 | #include "yield/platform/disk_operations.h" 10 | #include "yield/platform/directory_walker.h" 11 | 12 | #define TEST_OUTPUT_DIR "test_out" 13 | 14 | class TestCaseTmpDir : public yield::TestCase { 15 | public: 16 | TestCaseTmpDir(const char* short_description, yield::TestSuite& test_suite) 17 | : yield::TestCase(short_description, test_suite) {} 18 | 19 | void setUp() { 20 | if(yield::DiskOperations::exists(yield::Path(TEST_OUTPUT_DIR) + __short_description)) 21 | yield::DiskOperations::rmtree(yield::Path(TEST_OUTPUT_DIR) + __short_description); 22 | try { 23 | yield::DiskOperations::mkdir(yield::Path(TEST_OUTPUT_DIR)); 24 | } catch(yield::PlatformException& ) {} 25 | 26 | yield::DiskOperations::mkdir(yield::Path(TEST_OUTPUT_DIR) + __short_description); 27 | } 28 | 29 | yield::Path testPath(const std::string& filename = "") { 30 | return yield::Path(TEST_OUTPUT_DIR) + __short_description + filename; 31 | } 32 | }; 33 | 34 | #define EXPECT_EQUAL(stat_a,stat_b) \ 35 | { if ( !( (stat_a) == (stat_b) ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat_a" != "#stat_b ); } 36 | #define EXPECT_TRUE(stat) \ 37 | { if ( !( (stat) == true ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat" != true" ); } 38 | #define EXPECT_FALSE(stat) \ 39 | { if ( !( (stat) == false ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat" != false" ); } 40 | 41 | 42 | #define TEST_TMPDIR( short_description, TestSuiteName ) \ 43 | extern yield::TestSuite& TestSuiteName##TestSuite(); \ 44 | class short_description##Test : public TestCaseTmpDir \ 45 | { \ 46 | public:\ 47 | short_description##Test() : TestCaseTmpDir( #short_description "Test", TestSuiteName##TestSuite() ) { }\ 48 | void runTest();\ 49 | };\ 50 | short_description##Test short_description##Test_inst;\ 51 | void short_description##Test::runTest() 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /cpp/include/babudb/test_helper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef BABUDB_TEST_HELPER_H 5 | #define BABUDB_TEST_HELPER_H 6 | 7 | #include 8 | using std::string; 9 | 10 | #include "babudb/buffer.h" 11 | #include "babudb/log/log_section.h" 12 | 13 | #define DUMMY_OPERATION_TYPE 5 14 | 15 | class DummyOperation : public babudb::Serializable { 16 | public: 17 | DummyOperation(int i) : value(i) {} 18 | 19 | virtual void Serialize(const babudb::Buffer& data) const { 20 | *((int*)data.data) = value; 21 | } 22 | 23 | virtual size_t GetSize() const { 24 | return sizeof(int); 25 | } 26 | 27 | virtual int GetType() const { 28 | return DUMMY_OPERATION_TYPE; 29 | } 30 | 31 | DummyOperation& Deserialize(const babudb::Buffer& data) { 32 | value = *((int*)data.data); 33 | return *this; 34 | } 35 | 36 | int value; 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/assert.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_ASSERT_H 5 | #define YIELD_PLATFORM_ASSERT_H 6 | 7 | #include "yield/platform/exception.h" 8 | 9 | #ifdef _WIN32 10 | #include 11 | #else 12 | #include 13 | #endif 14 | 15 | 16 | namespace yield 17 | { 18 | class AssertionException : public Exception 19 | { 20 | public: 21 | AssertionException( const char* file_name, int line_number, const char* info = "" ) 22 | { 23 | #ifdef _WIN32 24 | _snprintf_s( what_buffer, 1024, "line number %d in %s (%s)", line_number, file_name, info ); 25 | #else 26 | std::snprintf( what_buffer, 1024, "line number %d in %s (%s)", line_number, file_name, info ); 27 | #endif 28 | } 29 | 30 | virtual const char* what() const throw() { return what_buffer; } 31 | 32 | private: 33 | char what_buffer[1024]; 34 | }; 35 | } 36 | 37 | 38 | #define FAIL() throw yield::AssertionException( __FILE__, __LINE__ ); 39 | #define ASSERT_TRUE( stat ) { if ( !( ( stat ) == true ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat" != true" ); } 40 | #define ASSERT_FALSE( stat ) { if ( !( ( stat ) == false ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat" != false" ); } 41 | #define ASSERT_EQUAL( stat_a, stat_b ) { if ( !( ( stat_a ) == ( stat_b ) ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat_a" != "#stat_b ); } 42 | #define ASSERT_NE( stat_a, stat_b ) { if ( !( ( stat_a ) != ( stat_b ) ) ) throw yield::AssertionException( __FILE__, __LINE__, #stat_a" == "#stat_b ); } 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/debug.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_DEBUG_H 5 | #define YIELD_PLATFORM_DEBUG_H 6 | 7 | namespace yield 8 | { 9 | #ifdef _WIN32 10 | extern "C" 11 | { 12 | __declspec( dllimport ) void __stdcall DebugBreak(); 13 | } 14 | #else 15 | inline void DebugBreak() 16 | { 17 | *((int*)0) = 0xabadcafe; 18 | } 19 | #endif 20 | } 21 | 22 | #endif 23 | 24 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/directory_walker.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_DIRECTORY_WALKER_H 5 | #define YIELD_PLATFORM_DIRECTORY_WALKER_H 6 | 7 | #include "yield/platform/stat.h" 8 | 9 | #include 10 | 11 | 12 | namespace yield 13 | { 14 | class DirectoryEntry : public Stat 15 | { 16 | public: 17 | const Path& getPath() { return path; } 18 | 19 | private: 20 | friend class DirectoryWalker; 21 | 22 | DirectoryEntry( const Path& path ) 23 | : Stat( path ), path( path ) 24 | { } 25 | 26 | DirectoryEntry( const Path& path, StatType type, size_t size, int64_t last_write_time, int64_t creation_time, int64_t last_access_time, bool is_hidden ) 27 | : Stat( type, size, last_write_time, creation_time, last_access_time, is_hidden ), path( path ) 28 | { } 29 | 30 | Path path; 31 | }; 32 | 33 | 34 | class DirectoryWalker 35 | { 36 | public: 37 | DirectoryWalker( const Path& root_dir_path ); 38 | DirectoryWalker( const Path& root_dir_path, const Path& match_file_name_prefix ); 39 | ~DirectoryWalker(); 40 | 41 | bool hasNext(); 42 | std::auto_ptr getNext() { return next_directory_entry; } 43 | 44 | private: 45 | void init(); 46 | 47 | Path root_dir_path, match_file_name_prefix; 48 | 49 | std::auto_ptr next_directory_entry; 50 | 51 | void* scan_handle; 52 | #ifdef _WIN32 53 | std::wstring search_pattern; 54 | #endif 55 | }; 56 | } 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/disk_operations.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_DISK_OPERATIONS_H 5 | #define YIELD_PLATFORM_DISK_OPERATIONS_H 6 | 7 | #include "yield/platform/platform_types.h" 8 | #include "yield/platform/path.h" 9 | 10 | #include 11 | 12 | #ifdef _WIN32 13 | #define O_SYNC 010000 14 | #define O_FSYNC 010000 15 | #define O_ASYNC 020000 16 | #define O_DIRECT 040000 17 | #define O_HIDDEN 0100000 18 | #endif 19 | 20 | #define O_THROW_EXCEPTIONS 04000000 21 | #define O_CLOSE_ON_DESTRUCT 010000000 22 | #define O_SPARSE 020000000 23 | 24 | namespace yield 25 | { 26 | class DiskOperations 27 | { 28 | public: 29 | static fd_t open( const Path&, unsigned long flags = O_RDONLY ); 30 | static bool close( fd_t, unsigned long flags = 0 ); 31 | static bool exists( const Path& ); 32 | static bool touch( const Path&, unsigned long flags = 0 ); 33 | static bool unlink( const Path&, unsigned long flags = 0 ); 34 | static bool rename( const Path& from_path, const Path& to_path , unsigned long flags = 0 ); 35 | static bool mkdir( const Path&, unsigned long flags = 0 ); 36 | static bool mktree( const Path&, unsigned long flags = 0 ); 37 | static bool makedirs( const Path& path, unsigned long flags = 0 ) { return mktree( path ); } // Python function name 38 | static bool rmdir( const Path&, unsigned long flags = 0 ); 39 | static bool rmtree( const Path&, unsigned long flags = 0 ); 40 | 41 | private: 42 | static bool _rmtree( const Path&, unsigned long flags ); 43 | }; 44 | } 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/exception.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_EXCEPTION_H 5 | #define YIELD_PLATFORM_EXCEPTION_H 6 | 7 | #include 8 | 9 | namespace yield 10 | { 11 | class Exception : public std::exception 12 | { 13 | public: 14 | Exception() : _what( "" ) { } 15 | Exception( const char* what ) : _what( what ) { } 16 | virtual ~Exception() throw() { } 17 | 18 | // exception 19 | virtual const char* what() const throw() { return _what; } 20 | 21 | private: 22 | const char* _what; 23 | }; 24 | } 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_FILE_H 5 | #define YIELD_PLATFORM_FILE_H 6 | 7 | #include "yield/platform/disk_operations.h" 8 | 9 | namespace yield 10 | { 11 | class Path; 12 | 13 | class File 14 | { 15 | public: 16 | File( const Path& path, unsigned long flags = O_RDONLY|O_THROW_EXCEPTIONS|O_CLOSE_ON_DESTRUCT ); 17 | #ifdef _WIN32 18 | // Since fd_t is void* on Windows we need these to delegate to the const Path& variant rather than the fd_t one 19 | File( char* path, unsigned long flags = O_RDONLY|O_THROW_EXCEPTIONS|O_CLOSE_ON_DESTRUCT ); 20 | File( const char* path, unsigned long flags = O_RDONLY|O_THROW_EXCEPTIONS|O_CLOSE_ON_DESTRUCT ); 21 | #endif 22 | static File* open( const Path& path, unsigned long flags = O_RDONLY|O_CLOSE_ON_DESTRUCT ); 23 | explicit File( fd_t fd, unsigned long flags = O_THROW_EXCEPTIONS|O_CLOSE_ON_DESTRUCT ); 24 | File( const File& ); 25 | File(); 26 | virtual ~File(); 27 | 28 | inline unsigned long getFlags() { return flags; } 29 | inline fd_t getFD() { return fd; } 30 | 31 | uint64_t seek( uint64_t offset, unsigned char whence ); 32 | ssize_t read( void* buf, size_t nbyte ); 33 | ssize_t write( const void* buf, size_t nbyte ); 34 | ssize_t write( const std::string& buf ) { return write( buf.c_str(), buf.size() ); } 35 | ssize_t write( const char* buf ) { return write( buf, std::strlen( buf ) ); } 36 | virtual bool close(); 37 | bool isOpen(); 38 | 39 | static bool CopyFile(const std::string& from, const std::string& to); 40 | 41 | protected: 42 | fd_t fd; 43 | unsigned long flags; 44 | }; 45 | } 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/memory_mapped_file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_MEMORY_MAPPED_FILE_H 5 | #define YIELD_PLATFORM_MEMORY_MAPPED_FILE_H 6 | 7 | #include "yield/platform/file.h" 8 | 9 | namespace yield 10 | { 11 | class MemoryMappedFile : public File 12 | { 13 | public: 14 | MemoryMappedFile( const Path& path, size_t minimum_size = 0, unsigned long flags = O_RDWR|O_SYNC ); 15 | virtual ~MemoryMappedFile() { close(); } 16 | 17 | void resize( size_t ); 18 | inline char* getRegionStart() { return start; } 19 | inline char* getRegionEnd() { return start + size; } 20 | inline size_t getRegionSize() { return size; } 21 | 22 | virtual void writeBack(); 23 | virtual void writeBack( size_t offset, size_t length ); 24 | virtual void writeBack( void* ptr, size_t length ); 25 | 26 | // File 27 | bool close(); 28 | 29 | private: 30 | size_t size; 31 | char* start; 32 | #ifdef _WIN32 33 | void* mapping; 34 | #endif 35 | }; 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/path.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_PATH_H 5 | #define YIELD_PLATFORM_PATH_H 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #ifdef _WIN32 12 | #define DISK_PATH_SEPARATOR '\\' 13 | #define DISK_PATH_SEPARATOR_STRING "\\" 14 | #define DISK_PATH_SEPARATOR_WIDE_STRING L"\\" 15 | #else 16 | #define HOST_CHARSET "UTF-8" 17 | #define DISK_PATH_SEPARATOR '/' 18 | #define DISK_PATH_SEPARATOR_STRING "/" 19 | #endif 20 | 21 | 22 | namespace yield 23 | { 24 | class Path 25 | { 26 | public: 27 | Path() { } 28 | Path( const char* path, size_t path_len = 0, bool path_is_host_charset = true ) { init( path, ( path_len != 0 ) ? path_len : std::strlen( path ), path_is_host_charset ); } 29 | Path( const std::string& path, bool path_is_host_charset = true ) { init( path.c_str(), path.size(), path_is_host_charset ); } 30 | #ifdef _WIN32 31 | Path( const wchar_t* wide_path, size_t wide_path_len = 0 ); 32 | Path( const std::wstring& wide_path ); 33 | #endif 34 | Path( const Path& ); 35 | 36 | // These functions return the platform path, e.g. with \\ on Windows 37 | const std::string& getHostCharsetPath() const { return host_charset_path; } 38 | const std::string& getUTF8Path(); 39 | #ifdef _WIN32 40 | const std::wstring& getWidePath() const { return wide_path; } 41 | operator const std::wstring&() { return wide_path; } 42 | operator const wchar_t*() { return wide_path.c_str(); } 43 | #endif 44 | operator const std::string&() { return host_charset_path; } 45 | Path operator+( const Path& other ) const { return join( other ); } 46 | bool operator==( const Path& ) const; 47 | bool operator!=( const Path& ) const; 48 | 49 | Path join( const Path& ) const; 50 | std::pair split() const; 51 | Path abspath() const; 52 | 53 | private: 54 | void init( const char*, size_t, bool ); 55 | 56 | std::string host_charset_path, utf8_path; 57 | #ifdef _WIN32 58 | std::wstring wide_path; 59 | #else 60 | void MultiByteToMultiByte( const char* fromcode, const std::string& frompath, const char* tocode, std::string& topath ); 61 | #endif 62 | }; 63 | } 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/platform_exception.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_PLATFORM_EXCEPTION_H 5 | #define YIELD_PLATFORM_PLATFORM_EXCEPTION_H 6 | 7 | #include "yield/platform/exception.h" 8 | 9 | namespace yield 10 | { 11 | class PlatformException : public Exception 12 | { 13 | public: 14 | static void strerror( unsigned long error_code, char* error_code_str, unsigned int error_code_str_len ); 15 | 16 | PlatformException(); // Get error code from errno or GetLastError() 17 | PlatformException( unsigned long error_code ); 18 | 19 | // exception 20 | virtual const char* what() const throw(); 21 | 22 | private: 23 | unsigned long error_code; 24 | char what_buffer[100]; 25 | }; 26 | } 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/platform_types.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_PLATFORM_TYPES_H 5 | #define YIELD_PLATFORM_PLATFORM_TYPES_H 6 | 7 | #include 8 | #include // For size_t 9 | 10 | namespace yield 11 | { 12 | typedef uint32_t timeout_ns_t; // This is a uint32_t (which can only store ~4.3s in ns) because Solaris+gcc (4.0, 4.2) mangles uint64_t's on function calls in 32-bit binaries (no explanation for that yet) 13 | 14 | #ifdef _WIN32 15 | typedef void* fd_t; 16 | typedef unsigned int socket_t; 17 | 18 | struct iovec_t 19 | { 20 | size_t len; 21 | void* buf; 22 | }; 23 | 24 | typedef __int64 ssize_t; 25 | 26 | #ifndef MAX_PATH 27 | #define MAX_PATH 260 28 | #endif 29 | #else 30 | typedef int fd_t; 31 | typedef int socket_t; 32 | #define INVALID_HANDLE_VALUE -1 33 | 34 | struct iovec_t 35 | { 36 | void* buf; 37 | size_t len; 38 | }; 39 | 40 | #ifdef __MACH__ 41 | #define MAX_PATH PATH_MAX 42 | #else 43 | #define MAX_PATH 512 44 | #endif 45 | #endif 46 | 47 | inline uint32_t upper32( uint64_t val ) { return ( uint32_t )( val >> 32 ); } 48 | inline uint32_t lower32( uint64_t val ) { return ( uint32_t )( val & 0xffFFffFF ); } 49 | inline uint64_t create_uint64( uint32_t upper, uint32_t lower ) { return ( ( uint64_t )upper ) << 32 | ( uint64_t )lower; } 50 | } 51 | 52 | #ifndef SIZE_MAX 53 | #define SIZE_MAX ( ( size_t ) - 1 ) 54 | #endif 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/stat.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_STAT_H 5 | #define YIELD_PLATFORM_STAT_H 6 | 7 | #include "yield/platform/platform_types.h" 8 | #include "yield/platform/path.h" 9 | 10 | 11 | namespace yield 12 | { 13 | class Stat 14 | { 15 | public: 16 | enum StatType { Volume, File, Directory }; 17 | 18 | Stat( const Path& path ) { init( path ); } 19 | #ifdef _WIN32 20 | // Since fd_t is void* on Windows we need these to delegate to the const Path& variant rather than the fd_t one 21 | Stat( char* path ) { init( Path( path ) ); } 22 | Stat( const char* path ) { init( Path( path ) ); } 23 | #endif 24 | Stat( fd_t fd ) { init( fd ); } 25 | Stat( StatType type, size_t _st_size, int64_t _st_mtime, int64_t _st_ctime, int64_t _st_atime, bool is_hidden = false ); 26 | Stat( const Stat& ); 27 | 28 | StatType getType() { return type; } 29 | size_t getSize() const { return _st_size; } 30 | int64_t getLastWriteTime() const { return _st_mtime; } 31 | int64_t getCreationTime() const { return _st_ctime; } 32 | int64_t getLastAccessTime() const { return _st_atime; } 33 | bool isHidden() const { return is_hidden; } 34 | 35 | protected: 36 | void init( const Path& ); // Opens path, fstats the file descriptor, and closes it 37 | void init( fd_t ); // fstats the file descriptor but doesn't close it 38 | 39 | StatType type; 40 | size_t _st_size; int64_t _st_mtime, _st_ctime, _st_atime; // Unix doesn't like using the actual names, thus the _ 41 | bool is_hidden; 42 | }; 43 | } 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /cpp/include/yield/platform/windows.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #ifndef YIELD_PLATFORM_WINDOWS_H 5 | #define YIELD_PLATFORM_WINDOWS_H 6 | 7 | #ifdef _WIN32 8 | #ifndef WINVER 9 | #define WINVER 0x0500 10 | #endif 11 | #define _WIN32_WINNT 0x0500 12 | #define WIN32_LEAN_AND_MEAN 13 | //#define UNICODE 14 | #define NOMINMAX 15 | #include 16 | 17 | #undef Yield 18 | #undef WRITE_RESTRICTED 19 | #endif 20 | 21 | #endif 22 | 23 | -------------------------------------------------------------------------------- /cpp/src/index/index.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, 2010 Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // ImmutableIndex implements a persistent immutable search tree. It represents 10 | // the state for an index up to a LSN. 11 | // Indices are named with the following scheme: 12 | // databasename-indexname_lastlsn.idx 13 | 14 | #ifndef BABUB_IMMUTABLEINDEX_H 15 | #define BABUB_IMMUTABLEINDEX_H 16 | 17 | #include 18 | #include 19 | 20 | #include "babudb/key.h" 21 | #include "babudb/log/sequential_file.h" 22 | 23 | namespace babudb { 24 | 25 | class LogIndex; 26 | class ImmutableIndexWriter; 27 | 28 | class ImmutableIndex { 29 | public: 30 | static ImmutableIndex* Load( 31 | const std::string& file_name, lsn_t lsn, const KeyOrder& order); 32 | static ImmutableIndexWriter* Create( 33 | const std::string& name, lsn_t lsn, size_t chunk_size); 34 | 35 | typedef class ImmutableIndexIterator iterator; 36 | Buffer Lookup(Buffer search_key); 37 | iterator Find(Buffer key); 38 | 39 | iterator begin() const; 40 | iterator end() const; 41 | 42 | lsn_t GetLastLSN() { return latest_lsn; } 43 | 44 | typedef std::vector > DiskIndices; 45 | static DiskIndices FindIndices(const std::string& name_prefix); 46 | static ImmutableIndex* LoadLatestIntactIndex(DiskIndices& on_disk, 47 | const KeyOrder& order); 48 | static std::string GetIndexName(const std::string& name, lsn_t lsn); 49 | void CleanupObsolete(const std::string& file_name, 50 | const std::string& obsolete_prefix); 51 | 52 | private: 53 | ImmutableIndex(LogStorage* mm, const KeyOrder& order, lsn_t); 54 | bool LoadRoot(); 55 | typedef std::map Tree; 56 | 57 | Tree::iterator findChunk(const Buffer& key); 58 | offset_t* getOffsetTable(offset_t offset_rec_offset); 59 | 60 | SequentialFile storage; 61 | Tree index; 62 | const KeyOrder& order; 63 | lsn_t latest_lsn; 64 | }; 65 | 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /cpp/src/index/index_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef BABUDB_IMMUTABLEINDEXWRITER_H 5 | #define BABUDB_IMMUTABLEINDEXWRITER_H 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "babudb/buffer.h" 12 | #include "babudb/key.h" 13 | #include "babudb/log/sequential_file.h" 14 | 15 | namespace babudb { 16 | 17 | const int RECORD_TYPE_KEY = 1; 18 | const int RECORD_TYPE_VALUE = 2; 19 | const int RECORD_TYPE_OFFSETS = 3; 20 | const int RECORD_TYPE_INDEX_KEY = 4; 21 | const int RECORD_TYPE_INDEX_OFFSETS = 5; 22 | const int RECORD_TYPE_FILE_FOOTER = 6; 23 | 24 | class LogIndex; 25 | class ImmutableIndex; 26 | class KeyOrder; 27 | 28 | class ImmutableIndexWriter { 29 | public: 30 | ImmutableIndexWriter(LogStorage* mm, size_t chunk_size) 31 | : storage(mm), chunk_size(chunk_size), data_in_buffer(0) {} 32 | 33 | void Add(Buffer key, Buffer value); 34 | void FlushBuffer(); 35 | void Finalize(); 36 | 37 | static char GetType(const SequentialFile::iterator& it); 38 | static babudb::Buffer GetData(const SequentialFile::iterator& it); 39 | 40 | private: 41 | void* WriteData(Buffer data, char type); 42 | typedef std::vector > WriteBuffer; 43 | WriteBuffer record_buffer; 44 | 45 | SequentialFile storage; 46 | size_t chunk_size; 47 | size_t data_in_buffer; 48 | 49 | std::vector index_keys; 50 | std::vector index_offsets; 51 | }; 52 | 53 | 54 | class ImmutableIndexIterator { 55 | public: 56 | ImmutableIndexIterator(const SequentialFile& file, bool end); 57 | ImmutableIndexIterator(const ImmutableIndexIterator& o); 58 | ImmutableIndexIterator(SequentialFile& file, offset_t* table, SequentialFile::iterator i, int n); 59 | 60 | void operator ++ (); 61 | std::pair operator * (); 62 | bool operator != (const ImmutableIndexIterator& other) const; 63 | bool operator == (const ImmutableIndexIterator& other) const; 64 | 65 | private: 66 | void findNextOffsetTable(SequentialFile::iterator it); 67 | 68 | const SequentialFile& file; 69 | offset_t* offset_table; 70 | 71 | SequentialFile::iterator key; // the current key if offset_table != NULL 72 | int key_no; // the ordinal number of the current key 73 | }; 74 | 75 | } 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /cpp/src/index/merger.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #ifndef INDEXMERGER_H 10 | #define INDEXMERGER_H 11 | 12 | #include "index.h" 13 | #include "log_index.h" 14 | 15 | #include 16 | #include 17 | 18 | namespace babudb { 19 | class ImmutableIndexWriter; 20 | 21 | // Creates a persistent index from a LogIndex 22 | class IndexMerger { 23 | public: 24 | // Step 1: Create the merger 25 | explicit IndexMerger(const std::string&, const KeyOrder&); 26 | explicit IndexMerger(const std::string&, const KeyOrder&, ImmutableIndex* base); 27 | ~IndexMerger(); 28 | 29 | // Step 2: Fill with data up to a certain LSN 30 | void Add(lsn_t lsn, const Buffer&, const Buffer&); 31 | void Remove(lsn_t lsn, const Buffer&); 32 | 33 | // Step 3: Merge with old ImmutableIndex or create a new one from scratch 34 | void Setup(); 35 | void Proceed(int steps); 36 | bool IsFinished(); 37 | 38 | void Run() { 39 | Setup(); 40 | while (!IsFinished()) { 41 | Proceed(100); 42 | } 43 | } 44 | 45 | protected: 46 | std::string file_name; 47 | const KeyOrder& order; 48 | ImmutableIndex* base; 49 | lsn_t last_lsn; 50 | LogIndex diff; 51 | 52 | ImmutableIndex::iterator* base_it; 53 | LogIndex::iterator diff_it; 54 | 55 | std::auto_ptr destination; 56 | }; 57 | 58 | } 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /cpp/src/log/log_iterator.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #include "babudb/log/log_iterator.h" 5 | #include "babudb/log/log_section.h" 6 | #include "babudb/log/sequential_file.h" 7 | 8 | #include "log/log_section_iterator.h" 9 | 10 | #include "yield/platform/yunit.h" 11 | 12 | #include 13 | 14 | namespace babudb { 15 | 16 | LogIterator::LogIterator(LogSectionIterator* current_section) 17 | : current_section(current_section) {} 18 | 19 | LogIterator::~LogIterator() {} // for auto_ptr 20 | 21 | LogIterator* LogIterator::First(LogSectionIterator* first_section) { 22 | LogIterator* it = new LogIterator(first_section); 23 | if (first_section->IsValid()) { 24 | it->record_iterator = (**first_section)->First(); 25 | } 26 | return it; 27 | } 28 | 29 | LogIterator* LogIterator::Last(LogSectionIterator* last_section) { 30 | LogIterator* it = new LogIterator(last_section); 31 | if (last_section->IsValid()) { 32 | it->record_iterator = (**last_section)->Last(); 33 | } 34 | return it; 35 | } 36 | 37 | bool LogIterator::GetNext() { 38 | if (!record_iterator.GetNext()) { 39 | if (!current_section->GetNext()) { 40 | return false; 41 | } else { 42 | record_iterator = (**current_section)->First(); 43 | return GetNext(); 44 | } 45 | } else { 46 | ASSERT_TRUE(record_iterator.IsValid()); 47 | return true; 48 | } 49 | } 50 | 51 | bool LogIterator::GetPrevious() { 52 | if (!record_iterator.GetPrevious()) { 53 | if (!current_section->GetPrevious()) { 54 | return false; 55 | } else { 56 | record_iterator = (**current_section)->Last(); 57 | return GetPrevious(); 58 | } 59 | } else { 60 | ASSERT_TRUE(record_iterator.IsValid()); 61 | return true; 62 | } 63 | } 64 | 65 | bool LogIterator::IsValid() const { 66 | return record_iterator.IsValid(); 67 | } 68 | 69 | bool LogIterator::operator != (const LogIterator& other) const { 70 | return *current_section != *other.current_section || record_iterator != other.record_iterator; 71 | } 72 | 73 | bool LogIterator::operator == (const LogIterator& other) const { 74 | return *current_section == *other.current_section && record_iterator == other.record_iterator; 75 | } 76 | 77 | Buffer LogIterator::operator * () const { 78 | return record_iterator.AsData(); 79 | } 80 | 81 | Buffer LogIterator::GetOperationWithFrame() const { 82 | return Buffer(record_iterator.GetRecord(), record_iterator.GetRecord()->GetRecordSize()); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /cpp/src/log/log_section.cpp: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #include "babudb/log/log_section.h" 10 | #include "babudb/log/log_storage.h" 11 | #include "babudb/buffer.h" 12 | 13 | #include "yield/platform/assert.h" 14 | 15 | namespace babudb { 16 | 17 | LogSection::LogSection(LogStorage* mmfile, lsn_t first) 18 | : SequentialFile(mmfile), first_lsn(first) { } 19 | 20 | lsn_t LogSection::getFirstLSN() const { 21 | return first_lsn; 22 | } 23 | 24 | void LogSection::Append(const Serializable& entry) { 25 | void* write_location = getFreeSpace(RECORD_MAX_SIZE); 26 | entry.Serialize(Buffer(write_location, RECORD_MAX_SIZE)); 27 | unsigned int payload_size = entry.GetSize(); 28 | ASSERT_TRUE(payload_size <= RECORD_MAX_SIZE); // be paranoid 29 | frameData(write_location, payload_size); 30 | } 31 | 32 | void LogSection::Commit() { 33 | commit(); 34 | } 35 | 36 | void LogSection::Erase(const iterator& it) { 37 | erase(record2offset(it.GetRecord())); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /cpp/src/log/log_section_iterator.cpp: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #include "log/log_section_iterator.h" 10 | #include "yield/platform/assert.h" 11 | 12 | namespace babudb { 13 | 14 | LogSectionIterator::LogSectionIterator( 15 | std::vector const* sections, 16 | std::vector::const_iterator current) 17 | : sections(sections), current_section(current) {} 18 | 19 | LogSectionIterator* LogSectionIterator::First( 20 | std::vector const* sections) { 21 | return new LogSectionIterator(sections, sections->begin()); 22 | } 23 | 24 | LogSectionIterator* LogSectionIterator::Last( 25 | std::vector const* sections) { 26 | if (sections->empty()) { 27 | return new LogSectionIterator(sections, sections->end()); 28 | } else { 29 | return new LogSectionIterator(sections, --(sections->end())); 30 | } 31 | } 32 | 33 | LogSection* LogSectionIterator::GetNext() { 34 | if (current_section == sections->end()) { 35 | return NULL; 36 | } 37 | ++current_section; 38 | if (current_section == sections->end()) { 39 | return NULL; 40 | } else { 41 | return *current_section; 42 | } 43 | } 44 | 45 | LogSection* LogSectionIterator::GetPrevious() { 46 | if (current_section == sections->begin()) { 47 | return NULL; 48 | } else { 49 | --current_section; 50 | return *current_section; 51 | } 52 | } 53 | 54 | bool LogSectionIterator::IsValid() const { 55 | return sections->size() > 0 && current_section != sections->end(); 56 | } 57 | 58 | bool LogSectionIterator::operator != (const LogSectionIterator& other) const { 59 | return !this->operator==(other); 60 | } 61 | 62 | bool LogSectionIterator::operator == (const LogSectionIterator& other) const { 63 | ASSERT_TRUE(sections->begin() == other.sections->begin()); 64 | return current_section == other.current_section; 65 | } 66 | 67 | LogSection* LogSectionIterator::operator * () const { 68 | if (current_section == sections->end()) { 69 | return NULL; 70 | } else { 71 | return *current_section; 72 | } 73 | } 74 | 75 | } // namespace babudb 76 | -------------------------------------------------------------------------------- /cpp/src/log/log_section_iterator.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // The LogSection augments the records of a SequentialFile with LSNs. 10 | 11 | #ifndef LOG_LOG_SECTION_ITERATOR_H 12 | #define LOG_LOG_SECTION_ITERATOR_H 13 | 14 | #include 15 | 16 | namespace babudb { 17 | 18 | class LogSection; 19 | 20 | class LogSectionIterator { 21 | public: 22 | static LogSectionIterator* First(std::vector const* sections); 23 | static LogSectionIterator* Last(std::vector const* sections); 24 | 25 | LogSection* GetNext(); 26 | LogSection* GetPrevious(); 27 | bool IsValid() const; 28 | bool operator != (const LogSectionIterator& other) const; 29 | bool operator == (const LogSectionIterator& other) const; 30 | 31 | LogSection* operator * () const; 32 | 33 | private: 34 | LogSectionIterator(std::vector const* sections, 35 | std::vector::const_iterator current); 36 | std::vector const* sections; 37 | std::vector::const_iterator current_section; 38 | }; 39 | 40 | } // namespace babudb 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /cpp/src/log/record_frame.cpp: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | 10 | #include "babudb/log/record_frame.h" 11 | 12 | #include 13 | using namespace yield; 14 | 15 | namespace babudb { 16 | 17 | void* RecordFrame::getPayload() const { 18 | return (char*)this + RECORD_FRAME_SIZE_BYTES; 19 | } 20 | RecordFrame* RecordFrame::GetRecord( void* p ) { return (RecordFrame*)((char*)p - RECORD_FRAME_SIZE_BYTES); } 21 | 22 | unsigned int RecordFrame::getPayloadSize() const { 23 | return _getLengthField(); 24 | } 25 | unsigned int RecordFrame::GetRecordSize() const { 26 | return (unsigned int)ALIGN(_getLengthField(), RECORD_FRAME_ALIGNMENT) + 2*RECORD_FRAME_SIZE_BYTES; 27 | } 28 | 29 | bool RecordFrame::isValid() { return mightBeHeader() && mightBeHeaderOf(getFooter()); } 30 | 31 | RecordFrame::RecordFrame(size_t size_in_bytes ) { 32 | ASSERT_TRUE(ISALIGNED((&header_data), RECORD_FRAME_ALIGNMENT)); 33 | ASSERT_TRUE(header_data.plain_header == 0); 34 | setLength(size_in_bytes); // ASSERT: memory was prev. filled with 0s 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /cpp/src/log_index.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #include "log_index.h" 5 | #include "babudb/log/log_section.h" 6 | 7 | #include 8 | 9 | #include 10 | using std::pair; 11 | 12 | namespace babudb { 13 | 14 | LogIndex::LogIndex(const KeyOrder& order, lsn_t first) 15 | : order(order), latest_value(MapCompare(order)), first_lsn(first) {} 16 | 17 | Buffer LogIndex::lookup(const Buffer& search_key) { 18 | Tree::iterator it = latest_value.find(search_key); 19 | 20 | if(it != latest_value.end()) { 21 | return it->second; 22 | } else { 23 | return Buffer::NotExists(); 24 | } 25 | } 26 | 27 | // Only add Buffer to the key. Removed keys are represented as Delete values because they 28 | // act as an overlay to less significant indices. 29 | bool LogIndex::Add(const Buffer& new_key, const Buffer& new_value) { 30 | Buffer tree_key = new_key.clone(); 31 | Buffer tree_value = new_value.clone(); 32 | // index in latest LogIndex 33 | pair old_entry = latest_value.insert(pair(tree_key, tree_value)); 34 | 35 | if(!old_entry.second) { // already existed 36 | Buffer old_key((old_entry.first)->first), old_value((old_entry.first)->second); 37 | 38 | latest_value.erase(old_entry.first); // remove and free it 39 | old_key.free(); 40 | old_value.free(); 41 | 42 | pair new_entry = latest_value.insert(pair(tree_key,tree_value)); 43 | ASSERT_TRUE(new_entry.second); 44 | } 45 | 46 | return !old_entry.second; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /cpp/src/log_index.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef LOGINDEX_H 5 | #define LOGINDEX_H 6 | 7 | #include "babudb/key.h" 8 | #include "babudb/buffer.h" 9 | #include "babudb/log/sequential_file.h" 10 | 11 | #include 12 | #include 13 | 14 | namespace babudb { 15 | 16 | class LogSection; 17 | class LogIndex; 18 | 19 | class LogIndex { 20 | public: 21 | LogIndex(const KeyOrder& order, lsn_t first); 22 | 23 | Buffer lookup(const Buffer& key); 24 | 25 | bool Add(const Buffer&, const Buffer&); 26 | 27 | lsn_t getFirstLSN() { return first_lsn; } 28 | 29 | typedef std::map Tree; 30 | typedef Tree::const_iterator iterator; 31 | 32 | iterator begin() const { return latest_value.begin(); } 33 | iterator end() const { return latest_value.end(); } 34 | 35 | iterator find(const Buffer& key) { 36 | return latest_value.lower_bound(key); 37 | } 38 | 39 | private: 40 | const KeyOrder& order; 41 | Tree latest_value; 42 | lsn_t first_lsn; 43 | }; 44 | 45 | } // namespace babudb 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /cpp/src/log_index_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #include "babudb/database.h" 5 | #include "babudb/profiles/string_key.h" 6 | 7 | #include "log_index.h" 8 | #include "merged_index.h" 9 | 10 | #include "yield/platform/memory_mapped_file.h" 11 | using yield::MemoryMappedFile; 12 | using namespace babudb; 13 | 14 | #include "babudb/test.h" 15 | 16 | TEST_TMPDIR(LogIndex,babudb) 17 | { 18 | StringOrder myorder; 19 | 20 | std::vector indices; 21 | indices.push_back(std::make_pair("testidx", &myorder)); 22 | 23 | Database* db = Database::Open(testPath("test").getHostCharsetPath(), indices); 24 | 25 | StringSetOperation(1, "testidx", "Key1", "data1").ApplyTo(*db, 1); 26 | 27 | Buffer result = db->Lookup("testidx", Buffer("Key1")); 28 | EXPECT_FALSE(result.isEmpty()); 29 | EXPECT_TRUE(strncmp((char*)result.data,"data1",5) == 0); 30 | 31 | result = db->Lookup("testidx", Buffer("Key2")); 32 | EXPECT_TRUE(result.isNotExists()); 33 | 34 | StringSetOperation(2, "testidx", "Key2", "data2").ApplyTo(*db, 2); 35 | result = db->Lookup("testidx", Buffer("Key2")); 36 | EXPECT_FALSE(result.isEmpty()); 37 | 38 | // Overwrite 39 | StringSetOperation(3, "testidx", "Key1", "data3").ApplyTo(*db, 3); 40 | 41 | result = db->Lookup("testidx", Buffer("Key1")); 42 | EXPECT_FALSE(result.isEmpty()); 43 | EXPECT_TRUE(strncmp((char*)result.data,"data3",5) == 0); 44 | 45 | // Prefix 46 | StringSetOperation(4, "testidx", "Ke4", "data4").ApplyTo(*db, 4); 47 | 48 | /* vector > results = db->match("testidx",Buffer("Key2",4)); 49 | EXPECT_TRUE(results.size() == 1); 50 | 51 | results = db->match("testidx",Buffer("Key")); 52 | EXPECT_TRUE(results.size() == 2); 53 | 54 | results = db->match("testidx",Buffer("Ke")); 55 | EXPECT_TRUE(results.size() == 3); 56 | */ 57 | delete db; 58 | } 59 | -------------------------------------------------------------------------------- /cpp/src/merged_index.h: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | // Bundles ImmutableIndices and LogIndices into one consistent index 10 | 11 | #ifndef MERGEDINDEX_H 12 | #define MERGEDINDEX_H 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #include "babudb/buffer.h" 19 | #include "babudb/key.h" 20 | 21 | namespace babudb { 22 | class LogSection; 23 | class LogIndex; 24 | class ImmutableIndex; 25 | class KeyOrder; 26 | class LookupIterator; 27 | class Log; 28 | 29 | class MergedIndex { 30 | public: 31 | explicit MergedIndex(const std::string& name, const KeyOrder& order); 32 | ~MergedIndex(); 33 | 34 | lsn_t GetLastPersistentLSN(); 35 | // Rename obsolete immutable indices 36 | void Cleanup(const std::string& to); 37 | void Snapshot(lsn_t current_lsn); 38 | LookupIterator GetSnapshot(lsn_t snapshot_lsn); 39 | 40 | typedef std::map ResultMap; 41 | Buffer Lookup(const Buffer& key); 42 | LookupIterator Lookup(const Buffer&, const Buffer&); 43 | 44 | void Add(const Buffer& key, const Buffer& value); 45 | void Remove(const Buffer& buffer); 46 | 47 | const KeyOrder& getOrder() { return order; } 48 | ImmutableIndex* GetBase() { return immutable_index; } 49 | 50 | private: 51 | LogIndex* tail; 52 | std::vector log_indices; // we currently use only one overlay 53 | ImmutableIndex* immutable_index; 54 | std::string name_prefix; 55 | const KeyOrder& order; 56 | }; 57 | 58 | } // namespace babudb 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /cpp/src/merged_index_test.cpp: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2010 Felix Hupfeld 4 | // Licensed under the BSD License, see LICENSE file for details. 5 | // 6 | // Author: Felix Hupfeld (felix@storagebox.org) 7 | 8 | #include "babudb/database.h" 9 | #include 10 | #include 11 | 12 | #include "babudb/profiles/string_key.h" 13 | #include "babudb/lookup_iterator.h" 14 | #include "babudb/test.h" 15 | #include "merged_index.h" 16 | 17 | #include "yield/platform/memory_mapped_file.h" 18 | using yield::MemoryMappedFile; 19 | using namespace babudb; 20 | 21 | TEST_TMPDIR(MergedIndex,babudb) 22 | { 23 | StringOrder sorder; 24 | MergedIndex index("test", sorder); 25 | index.Add(Buffer::wrap("key1"), Buffer::wrap("val1")); 26 | index.Add(Buffer::wrap("key2"), Buffer::Empty()); 27 | { 28 | Buffer value = index.Lookup(Buffer::wrap("key1")); 29 | EXPECT_EQUAL(value, Buffer::wrap("val1")); 30 | value = index.Lookup(Buffer::wrap("key2")); 31 | EXPECT_TRUE(value.isEmpty()); 32 | } 33 | 34 | // Snapshot, then overwrite the value 35 | index.Snapshot(2); 36 | index.Add(Buffer::wrap("key1"), Buffer::wrap("val2")); 37 | { 38 | Buffer value = index.Lookup(Buffer::wrap("key1")); 39 | EXPECT_EQUAL(value, Buffer::wrap("val2")); 40 | value = index.Lookup(Buffer::wrap("key2")); 41 | EXPECT_TRUE(value.isEmpty()); 42 | } 43 | 44 | // Check the snapshot 45 | LookupIterator snapshot = index.GetSnapshot(2); 46 | EXPECT_EQUAL(Buffer::wrap("val1"), (*snapshot).second); 47 | } 48 | -------------------------------------------------------------------------------- /cpp/src/string_db_test.cpp: -------------------------------------------------------------------------------- 1 | // This file is part of babudb/cpp 2 | // 3 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 4 | // Copyright (c) 2009, Felix Hupfeld 5 | // Licensed under the BSD License, see LICENSE file for details. 6 | // 7 | // Author: Felix Hupfeld (felix@storagebox.org) 8 | 9 | #include "babudb/profiles/string_db.h" 10 | #include 11 | #include 12 | 13 | #include "babudb/profiles/string_key.h" 14 | #include "babudb/test.h" 15 | #include "index/merger.h" 16 | 17 | #include "yield/platform/memory_mapped_file.h" 18 | using yield::MemoryMappedFile; 19 | using namespace babudb; 20 | 21 | TEST_TMPDIR(StringDB,babudb) 22 | { 23 | std::vector indices; indices.push_back("index"); 24 | 25 | StringDB* db = StringDB::Open(testPath("test"), indices); 26 | db->Add("index", "key1", "value1"); 27 | db->Commit(); 28 | ASSERT_TRUE(db->Lookup("index", "key1") == "value1"); 29 | ASSERT_TRUE(db->Lookup("index", "key2") == ""); 30 | delete db; 31 | 32 | db = StringDB::Open(testPath("test"), indices); 33 | ASSERT_TRUE(db->Lookup("index", "key1") == "value1"); 34 | ASSERT_TRUE(db->Lookup("index", "key2") == ""); 35 | db->Compact(testPath("alt_")); 36 | db->Add("index", "key2", "value2"); 37 | db->Commit(); 38 | ASSERT_TRUE(db->Lookup("index", "key1") == "value1"); 39 | ASSERT_TRUE(db->Lookup("index", "key2") == "value2"); 40 | delete db; 41 | 42 | db = StringDB::Open(testPath("test"), indices); 43 | ASSERT_TRUE(db->Lookup("index", "key1") == "value1"); 44 | ASSERT_TRUE(db->Lookup("index", "key2") == "value2"); 45 | db->Compact(testPath("alt_")); 46 | delete db; 47 | } 48 | -------------------------------------------------------------------------------- /cpp/src/test_main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #define YIELD_BUILDING_STANDALONE_TEST 1 5 | #include "babudb/test.h" 6 | 7 | TEST_SUITE(babudb); 8 | TEST_SUITE(babudb_single); 9 | TEST_MAIN(babudb); 10 | -------------------------------------------------------------------------------- /cpp/src/util.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #include "util.h" 5 | #include 6 | using namespace std; 7 | 8 | #include "yield/platform/path.h" 9 | 10 | namespace babudb { 11 | 12 | bool matchFilename(const yield::Path& fullpath, const string& desired_prefix, const string& desired_ext, unsigned int& lsn) { 13 | pair parts = fullpath.split(); 14 | 15 | std::istringstream tokenizer(parts.second.getHostCharsetPath()); 16 | 17 | string name; 18 | if(!std::getline(tokenizer,name,'_')) 19 | return false; 20 | 21 | string lsn_str; 22 | if(!std::getline(tokenizer,lsn_str,'.')) 23 | return false; 24 | 25 | std::istringstream lsn_conv(lsn_str); 26 | if(!(lsn_conv >> lsn)) 27 | return false; 28 | 29 | string ext; 30 | if(!std::getline(tokenizer,ext)) 31 | return false; 32 | 33 | return ext == desired_ext && name == desired_prefix; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /cpp/src/util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Felix Hupfeld, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, Zuse Institute Berlin. 2 | // Licensed under the BSD License, see LICENSE file for details. 3 | 4 | #ifndef UTIL_H 5 | #define UTIL_H 6 | 7 | #include 8 | 9 | namespace yield { 10 | class Path; 11 | } 12 | 13 | namespace babudb { 14 | bool matchFilename( 15 | const yield::Path& fullpath, const std::string& desired_prefix, 16 | const std::string& desired_ext, unsigned int& no); 17 | } // namespace babudb 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/directory_walker_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/yunit.h" 5 | #include "yield/platform/directory_walker.h" 6 | #include "yield/platform/disk_operations.h" 7 | using namespace yield; 8 | 9 | using namespace std; 10 | 11 | #define TEST_DIR_PATH "DirectoryWalker_test" 12 | #ifdef _WIN32 13 | #define TEST_FILE_PATH "DirectoryWalker_test\\file.txt" 14 | #define TEST_SUBDIR_PATH "DirectoryWalker_test\\subdir\\" 15 | #else 16 | #define TEST_FILE_PATH "DirectoryWalker_test/file.txt" 17 | #define TEST_SUBDIR_PATH "DirectoryWalker_test/subdir/" 18 | #endif 19 | 20 | 21 | DECLARE_TEST_SUITE( babudb ) 22 | 23 | class DirectoryWalkerTest : public TestCase 24 | { 25 | public: 26 | DirectoryWalkerTest() : TestCase( "DirectoryWalkerTest", babudbTestSuite() ) { } 27 | 28 | void setUp() 29 | { 30 | tearDown(); 31 | DiskOperations::mkdir( TEST_DIR_PATH ); 32 | DiskOperations::touch( TEST_FILE_PATH ); 33 | DiskOperations::mkdir( TEST_SUBDIR_PATH ); 34 | } 35 | 36 | void runTest() 37 | { 38 | DirectoryWalker walker( TEST_DIR_PATH ); 39 | 40 | bool seen_TEST_FILE_PATH = false, seen_TEST_SUBDIR_PATH = false; 41 | unsigned int seen_files_count = 0; 42 | while ( walker.hasNext() ) 43 | { 44 | auto_ptr next_directory_entry = walker.getNext(); 45 | const string& next_path = next_directory_entry.get()->getPath().getHostCharsetPath(); 46 | if ( next_path.compare( TEST_FILE_PATH ) == 0 ) seen_TEST_FILE_PATH = true; 47 | else if ( next_path.compare( TEST_SUBDIR_PATH ) == 0 ) seen_TEST_SUBDIR_PATH = true; 48 | seen_files_count++; 49 | } 50 | 51 | if ( seen_files_count != 2 ) 52 | { 53 | cerr << "expected 2 files, got " << seen_files_count; 54 | FAIL(); 55 | } 56 | ASSERT_TRUE( seen_TEST_FILE_PATH ); 57 | ASSERT_TRUE( seen_TEST_SUBDIR_PATH ); 58 | } 59 | 60 | void tearDown() 61 | { 62 | try { DiskOperations::rmdir( TEST_SUBDIR_PATH ); } catch ( ... ) { } 63 | try { DiskOperations::unlink( TEST_FILE_PATH ); } catch ( ... ) { } 64 | try { DiskOperations::rmdir( TEST_DIR_PATH ); } catch ( ... ) { } 65 | } 66 | }; 67 | 68 | DirectoryWalkerTest DirectoryWalkerTest_inst; 69 | 70 | TEST_MAIN( DirectoryWalker ) 71 | 72 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/disk_operations_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/yunit.h" 5 | #include "yield/platform/platform_exception.h" 6 | #include "yield/platform/disk_operations.h" 7 | using namespace yield; 8 | 9 | 10 | #define TEST_FILE_NAME "DiskOperations_test.txt" 11 | #define TEST_DIR_NAME "DiskOperations_test" 12 | 13 | 14 | TEST( DiskOperations, babudb ) 15 | { 16 | try 17 | { 18 | DiskOperations::touch( TEST_FILE_NAME ); 19 | ASSERT_TRUE( DiskOperations::exists( TEST_FILE_NAME ) ); 20 | ASSERT_TRUE( DiskOperations::exists( TEST_FILE_NAME ) ); 21 | fd_t fd = DiskOperations::open( TEST_FILE_NAME, O_RDONLY|O_THROW_EXCEPTIONS ); 22 | ASSERT_TRUE( DiskOperations::exists( TEST_FILE_NAME ) ); 23 | DiskOperations::close( fd ); 24 | DiskOperations::unlink( TEST_FILE_NAME ); 25 | ASSERT_FALSE( DiskOperations::exists( TEST_FILE_NAME ) ); 26 | 27 | DiskOperations::mkdir( TEST_DIR_NAME ); 28 | ASSERT_TRUE( DiskOperations::exists( TEST_DIR_NAME ) ); 29 | DiskOperations::rmdir( TEST_DIR_NAME ); 30 | ASSERT_FALSE( DiskOperations::exists( TEST_DIR_NAME ) ); 31 | 32 | Path tree_path( Path( TEST_DIR_NAME ) + DISK_PATH_SEPARATOR_STRING + Path( TEST_DIR_NAME ) + DISK_PATH_SEPARATOR_STRING + Path( TEST_DIR_NAME ) ); 33 | DiskOperations::mktree( tree_path ); 34 | ASSERT_TRUE( DiskOperations::exists( tree_path ) ); 35 | DiskOperations::touch( tree_path + TEST_FILE_NAME ); 36 | DiskOperations::rmtree( TEST_DIR_NAME ); 37 | ASSERT_FALSE( DiskOperations::exists( TEST_DIR_NAME ) ); 38 | } 39 | catch (PlatformException&) 40 | { 41 | try { DiskOperations::unlink( TEST_FILE_NAME ); } catch ( ... ) { } 42 | try { DiskOperations::rmtree( TEST_DIR_NAME ); } catch ( ... ) { } 43 | throw; 44 | } 45 | }; 46 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/file_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/yunit.h" 5 | #include "yield/platform/platform_exception.h" 6 | #include "yield/platform/file.h" 7 | using namespace yield; 8 | 9 | 10 | #define TEST_FILE_NAME "File_test.txt" 11 | 12 | TEST( FileConstructors, babudb ) 13 | { 14 | try 15 | { 16 | { 17 | File f( TEST_FILE_NAME, O_CREAT|O_TRUNC|O_WRONLY|O_CLOSE_ON_DESTRUCT ); 18 | } 19 | 20 | File* f = File::open( TEST_FILE_NAME ); 21 | ASSERT_NE( f, NULL ); 22 | delete f; 23 | 24 | { 25 | fd_t fd = DiskOperations::open( TEST_FILE_NAME ); 26 | File f( fd ); 27 | } 28 | } 29 | catch ( PlatformException& ) 30 | { 31 | try { DiskOperations::unlink( TEST_FILE_NAME ); } catch ( ... ) { } 32 | throw; 33 | } 34 | }; 35 | 36 | TEST( FileReadWrite, babudb ) 37 | { 38 | try 39 | { 40 | { 41 | File f( TEST_FILE_NAME, O_CREAT|O_TRUNC|O_WRONLY|O_CLOSE_ON_DESTRUCT ); 42 | f.write( "hello" ); 43 | } 44 | 45 | char hello[6]; hello[5] = 0; 46 | { 47 | File f( ( const char* )TEST_FILE_NAME ); 48 | ssize_t read_ret = f.read( hello, 5 ); 49 | ASSERT_EQUAL( read_ret, 5 ); 50 | ASSERT_TRUE( strcmp( hello, "hello" ) == 0 ); 51 | } 52 | } 53 | catch ( PlatformException& ) 54 | { 55 | try { DiskOperations::unlink( TEST_FILE_NAME ); } catch ( ... ) { } 56 | throw; 57 | } 58 | }; 59 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/memory_mapped_file_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/yunit.h" 5 | 6 | #include "yield/platform/memory_mapped_file.h" 7 | 8 | #include 9 | using std::memcpy; 10 | #include 11 | using std::strcmp; 12 | 13 | using namespace yield; 14 | 15 | #define TEST_FILE_NAME "MemoryMappedFile_test.dat" 16 | #define TESTSTRING "Test string" 17 | 18 | DECLARE_TEST_SUITE(babudb) 19 | 20 | class MemoryMappedFileTest : public TestCase 21 | { 22 | public: 23 | MemoryMappedFileTest() : TestCase( "MemoryMappedFileTest", babudbTestSuite() ) { } 24 | 25 | void setUp() 26 | { 27 | tearDown(); 28 | } 29 | 30 | void runTest() 31 | { 32 | { 33 | MemoryMappedFile mmf( TEST_FILE_NAME, strlen( TESTSTRING ) + 1, O_CREAT|O_RDWR|O_SYNC ); 34 | memcpy( mmf.getRegionStart(), TESTSTRING, strlen( TESTSTRING ) + 1 ); 35 | mmf.writeBack(); 36 | } 37 | 38 | MemoryMappedFile mmf( TEST_FILE_NAME, 0 ); 39 | ASSERT_EQUAL( mmf.getRegionSize(), strlen( TESTSTRING ) + 1 ); 40 | ASSERT_EQUAL( strcmp( mmf.getRegionStart(), TESTSTRING ), 0 ); 41 | } 42 | 43 | void tearDown() 44 | { 45 | try { DiskOperations::unlink( TEST_FILE_NAME ); } catch( ... ) { } 46 | } 47 | }; 48 | 49 | MemoryMappedFileTest MemoryMappedFileTest_inst; 50 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/platform_exception.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/platform_exception.h" 5 | using namespace yield; 6 | 7 | #ifdef _WIN32 8 | #include "yield/platform/windows.h" 9 | #include 10 | #else 11 | #include 12 | #include 13 | #include 14 | using std::strncpy; 15 | #ifdef __sun 16 | #include // For snprintf 17 | #endif 18 | #endif 19 | 20 | 21 | void PlatformException::strerror( unsigned long error_code, char* error_code_str, unsigned int error_code_str_len ) 22 | { 23 | #ifdef _WIN32 24 | DWORD dwStrLen = FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, error_code, MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ), error_code_str, error_code_str_len, NULL ); 25 | if ( dwStrLen > 2 ) error_code_str[dwStrLen-2] = 0; // Cut off trailing \r\n 26 | #else 27 | strncpy( error_code_str, std::strerror( error_code ), error_code_str_len ); 28 | #endif 29 | } 30 | 31 | PlatformException::PlatformException() 32 | { 33 | #ifdef _WIN32 34 | error_code = ::GetLastError(); 35 | #else 36 | error_code = ( unsigned long )errno; 37 | #endif 38 | } 39 | 40 | PlatformException::PlatformException( unsigned long error_code ) : error_code( error_code ) 41 | { } 42 | 43 | const char* PlatformException::what() const throw() 44 | { 45 | #ifdef _WIN32 46 | int written = _snprintf_s( ( char* )&what_buffer, 100, _TRUNCATE, "PlatformException: %lu ", error_code ); 47 | #else 48 | int written = snprintf( ( char* )&what_buffer, 100, "PlatformException: %lu ", error_code ); 49 | #endif 50 | 51 | if ( written > 0 && written < 100 ) 52 | PlatformException::strerror( error_code, ( char* )&what_buffer[written], 100-written-1 ); 53 | 54 | return what_buffer; 55 | } 56 | -------------------------------------------------------------------------------- /cpp/src/yield_platform/stat_test.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2009 Minor Gordon, with original implementations and ideas contributed by Felix Hupfeld. 2 | // This source comes from the Yield project. It is licensed under the GPLv2 (see COPYING for terms and conditions). 3 | 4 | #include "yield/platform/yunit.h" 5 | 6 | #include "yield/platform/stat.h" 7 | #include "yield/platform/disk_operations.h" 8 | 9 | using namespace yield; 10 | 11 | #define TEST_FILE_NAME "Stat_test.txt" 12 | #define TEST_DIR_NAME "Stat_test" 13 | 14 | 15 | TEST( Stat, babudb ) 16 | { 17 | try 18 | { 19 | DiskOperations::touch( TEST_FILE_NAME ); 20 | Stat file_stat( TEST_FILE_NAME ); 21 | DiskOperations::unlink( TEST_FILE_NAME ); 22 | ASSERT_EQUAL( file_stat.getType(), Stat::File ); 23 | ASSERT_EQUAL( file_stat.getSize(), 0 ); 24 | ASSERT_TRUE( file_stat.getCreationTime() != 0 && file_stat.getLastAccessTime() != 0 && file_stat.getLastWriteTime() != 0 ); 25 | ASSERT_FALSE( file_stat.isHidden() ); 26 | 27 | try { DiskOperations::mkdir( TEST_DIR_NAME ); } catch ( ... ) { } 28 | Stat dir_stat( TEST_DIR_NAME ); 29 | ASSERT_EQUAL( dir_stat.getType(), Stat::Directory ); 30 | DiskOperations::rmdir( TEST_DIR_NAME ); 31 | } 32 | catch ( std::exception& ) 33 | { 34 | try { DiskOperations::unlink( TEST_FILE_NAME ); } catch ( ... ) { } 35 | try { DiskOperations::rmdir( TEST_DIR_NAME ); } catch ( ... ) { } 36 | throw; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | BabuDB - Java Components 2 | ======================== 3 | 4 | In your `$HOME/.m2/settings.xml` add: 5 | ```XML 6 | 7 | 8 | 9 | 10 | 11 | 12 | babudb-dev 13 | 14 | 15 | central 16 | http://repo.maven.apache.org/maven2 17 | 18 | 19 | 20 | xtreemfs-repository 21 | https://xtreemfs.github.io/xtreemfs/maven 22 | 23 | true 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | ``` 34 | 35 | In your `pom.xml` add: 36 | ```XML 37 | 38 | 39 | 40 | 41 | 42 | 43 | org.xtreemfs.babudb 44 | babudb-core 45 | 0.6.0 46 | 47 | 48 | 49 | 50 | 51 | 52 | org.xtreemfs.babudb 53 | babudb-replication 54 | 0.6.0 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | ``` 68 | 69 | And build your project like so: 70 | ```Bash 71 | mvn install -Pbabudb-dev 72 | ``` 73 | -------------------------------------------------------------------------------- /java/babudb-core/AUTHORS: -------------------------------------------------------------------------------- 1 | Jan Stender (Zuse Institute Berlin) 2 | Bjoern Kolbeck (Zuse Institute Berlin) 3 | Mikael Hoegqvist (Zuse Institute Berlin) 4 | Felix Hupfeld 5 | Felix Langner (Zuse Institute Berlin) -------------------------------------------------------------------------------- /java/babudb-core/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 2 | Felix Hupfeld, Felix Langner, Zuse Institute Berlin 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above 10 | copyright notice, this list of conditions and the 11 | following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials 15 | provided with the distribution. 16 | * Neither the name of the Zuse Institute Berlin nor the 17 | names of its contributors may be used to endorse or promote 18 | products derived from this software without specific prior 19 | written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/Checkpointer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.api; 10 | 11 | import org.xtreemfs.babudb.api.exception.BabuDBException; 12 | 13 | /** 14 | * Interface for checkpointing support. It allows users to enforce database 15 | * checkpoints and to synchronously wait for a database checkpoint to complete. 16 | * 17 | * @author stenjan, bjko 18 | */ 19 | public interface Checkpointer { 20 | 21 | /** 22 | * Creates a checkpoint of all databases. The in-memory data is merged with 23 | * the on-disk data and is written to a new snapshot file. Database logs are 24 | * truncated. This operation is thread-safe. 25 | * 26 | * @throws BabuDBException 27 | * if the checkpoint was not successful 28 | * @throws InterruptedException 29 | */ 30 | public void checkpoint() throws BabuDBException, InterruptedException; 31 | 32 | /** 33 | * Wait until the current checkpoint is complete. 34 | * 35 | * @throws InterruptedException 36 | */ 37 | public void waitForCheckpoint() throws InterruptedException; 38 | } 39 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/StaticInitialization.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.api; 9 | 10 | /** 11 | *

12 | * Interface to provide static initialization code. The code will be executed 13 | * prior to initializing any plug-ins. 14 | *

15 | *

16 | * Static initialization code can e.g. be used to ensure that all replicas of a 17 | * replicated BabuDB installation have the same set of initial databases with 18 | * the same content. 19 | *

20 | * 21 | * @author flangner 22 | * @since 03/03/2010 23 | */ 24 | public interface StaticInitialization { 25 | 26 | /** 27 | * Method that provides an initial setup for {@link BabuDB}. 28 | * 29 | * @param dbMan the database manager. It can e.g. be used to create and retrieve databases. 30 | * @param sMan the snapshot manager. It can e.g. be used to create initial snapshots. 31 | */ 32 | public void initialize(DatabaseManager dbMan, SnapshotManager sMan); 33 | } 34 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/database/Database.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.api.database; 10 | 11 | import org.xtreemfs.babudb.api.index.ByteRangeComparator; 12 | 13 | /** 14 | * This interface contains all methods on a database. 15 | * 16 | * @author stenjan 17 | * 18 | */ 19 | public interface Database extends DatabaseRO { 20 | 21 | /** 22 | * Returns the name associated with the database. 23 | * 24 | * @return the database name 25 | */ 26 | public String getName(); 27 | 28 | /** 29 | * Creates a new group of inserts. 30 | * 31 | * @return an insert record group 32 | */ 33 | public DatabaseInsertGroup createInsertGroup(); 34 | 35 | /** 36 | * Returns an array of byte range comparators for all indices in the 37 | * database. The first entry contains the comparator for the first index, 38 | * the second entry for the second, and so on. 39 | * 40 | * @return an array of byte range comparators 41 | */ 42 | public ByteRangeComparator[] getComparators(); 43 | 44 | /** 45 | * Inserts a single key value pair. 46 | * 47 | * @param indexId 48 | * index id (0..NumIndices-1) 49 | * @param key 50 | * the key 51 | * @param value 52 | * the value 53 | * @param context 54 | * arbitrary context which is passed to the listener 55 | * @return a future as proxy for the request result. 56 | */ 57 | public DatabaseRequestResult singleInsert(int indexId, byte[] key, 58 | byte[] value, Object context); 59 | 60 | /** 61 | * Inserts a group of key value pairs. 62 | * 63 | * @param irg 64 | * the insert record group to execute 65 | * @param context 66 | * arbitrary context which is passed to the listener 67 | * @return a future as proxy for the request result. 68 | */ 69 | public DatabaseRequestResult insert(DatabaseInsertGroup irg, 70 | Object context); 71 | } 72 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/database/DatabaseInsertGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.api.database; 9 | 10 | /** 11 | * Interface of database operations that may be grouped and executed together. 12 | * 13 | * @author flangner 14 | * @since 11/03/2010 15 | */ 16 | public interface DatabaseInsertGroup { 17 | 18 | /** 19 | * Add a new insert operation to this group. Be aware of unpredictable behavior if a 20 | * key-value pair is manipulated twice within the same insert group. 21 | * 22 | * @param indexId - the index in which the key-value pair is inserted. 23 | * @param key - the key. 24 | * @param value - the value data. 25 | */ 26 | public void addInsert(int indexId, byte[] key, byte[] value); 27 | 28 | /** 29 | * Add a new delete operation to this group. Be aware of unpredictable behavior if a 30 | * key-value pair is manipulated twice within the same insert group. 31 | * 32 | * @param indexId - in which the key-value pair is located. 33 | * @param key - of the key-value pair to delete. 34 | */ 35 | public void addDelete(int indexId, byte[] key); 36 | } -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/database/ResultSet.java: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 2 | Felix Hupfeld, Felix Langner, Zuse Institute Berlin 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above 10 | copyright notice, this list of conditions and the 11 | following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials 15 | provided with the distribution. 16 | * Neither the name of the Zuse Institute Berlin nor the 17 | names of its contributors may be used to endorse or promote 18 | products derived from this software without specific prior 19 | written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | /* 35 | * AUTHORS: Jan Stender (ZIB) 36 | */ 37 | 38 | package org.xtreemfs.babudb.api.database; 39 | 40 | import java.util.Iterator; 41 | import java.util.Map.Entry; 42 | 43 | /** 44 | * A result set for prefix and range queries. 45 | * 46 | *

47 | * In addition to the methods inherited from the java.util.Iterator 48 | * interface, it defines an extra method to explicitly free any buffers bound to 49 | * the iterator. Applications that use prefix or range queries should invoke 50 | * free() once the result set is no longer needed, so as to keep 51 | * the memory footprint of BabuDB as low as possible. 52 | *

53 | * 54 | * @author stender 55 | * 56 | */ 57 | public interface ResultSet extends Iterator> { 58 | 59 | /** 60 | * Frees any resources attached to the iterator. 61 | */ 62 | public void free(); 63 | 64 | } 65 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/database/UserDefinedLookup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | 6 | package org.xtreemfs.babudb.api.database; 7 | 8 | import org.xtreemfs.babudb.api.exception.BabuDBException; 9 | import org.xtreemfs.babudb.lsmdb.LSMLookupInterface; 10 | 11 | /** 12 | * This interface can be used to execute complex lookup 13 | * routines in the thread of the database worker. 14 | * @author bjko 15 | */ 16 | public interface UserDefinedLookup { 17 | 18 | /** 19 | * The method which is executed by the worker thread. 20 | * @param database direct access to synchronous database lookups 21 | * @return the result which is passed on to the listener 22 | * @throws BabuDBException in case of an error, is passed to the listener 23 | */ 24 | public Object execute(final LSMLookupInterface database) 25 | throws BabuDBException; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/dev/ResponseManagerInternal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.api.dev; 9 | 10 | import org.xtreemfs.babudb.api.database.DatabaseRequestListener; 11 | import org.xtreemfs.babudb.api.exception.BabuDBException; 12 | import org.xtreemfs.foundation.LifeCycleThread; 13 | 14 | /** 15 | * Thread to process response handles for BabuDB request futures. This is necessary to decouple 16 | * internal BabuDB threads from user listeners. It does not prevent user listeners from deadlocking 17 | * them selves 18 | * 19 | * @author flangner 20 | * @since 05/29/2011 21 | */ 22 | public abstract class ResponseManagerInternal extends LifeCycleThread { 23 | 24 | /** 25 | * Default constructor to preinitialize a ResponseManager object. 26 | */ 27 | public ResponseManagerInternal() { 28 | super("RspMan"); 29 | } 30 | 31 | /** 32 | * This method enqueues a listener with the results that have to be passed to it. 33 | * 34 | * @param 35 | * @param listener 36 | * @param error 37 | * @param result 38 | * @param context 39 | * @throws InterruptedException 40 | */ 41 | public abstract void enqueueResponse(DatabaseRequestListener listener, 42 | BabuDBException error, T result, Object context) throws InterruptedException; 43 | } 44 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/dev/SnapshotManagerInternal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.api.dev; 9 | 10 | import org.xtreemfs.babudb.api.SnapshotManager; 11 | import org.xtreemfs.babudb.api.exception.BabuDBException; 12 | import org.xtreemfs.babudb.snapshots.SnapshotConfig; 13 | 14 | /** 15 | * Interface of {@link SnapshotManager} for internal usage. This should not be accessed 16 | * by any user application, but may be accessed by plugins. 17 | * 18 | * @author flangner 19 | * @since 03/18/2011 20 | */ 21 | 22 | public interface SnapshotManagerInternal extends SnapshotManager { 23 | 24 | /** 25 | * Method to retrieve the directory of the designated snapshot. 26 | * 27 | * @param dbName 28 | * @param snapshotName 29 | * @return the directory the designated snapshot has been stored. 30 | */ 31 | public String getSnapshotDir(String dbName, String snapshotName); 32 | 33 | /** 34 | * Invoked by the framework when snapshot creation has completed. 35 | * @param dbName 36 | * @param snap 37 | * 38 | * @throws BabuDBException 39 | */ 40 | public void snapshotComplete(String dbName, SnapshotConfig snap) throws BabuDBException; 41 | 42 | /** 43 | * Method to delete all available snapshots of a database. 44 | * 45 | * @param dbName 46 | * @throws BabuDBException 47 | */ 48 | public void deleteAllSnapshots(String dbName) throws BabuDBException; 49 | 50 | /** 51 | * Terminates the {@link SnapshotManager}. 52 | * 53 | * @throws BabuDBException if an error occurs. 54 | */ 55 | public void shutdown() throws BabuDBException; 56 | } 57 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/dev/transaction/InMemoryProcessing.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.api.dev.transaction; 9 | 10 | import org.xtreemfs.babudb.api.exception.BabuDBException; 11 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 12 | 13 | /** 14 | * In-memory operations directly connected with modifying the on-disk state 15 | * of the BabuDB data. This interface describes an algorithm and therefore 16 | * may not be stateful. 17 | * 18 | * @author flangner 19 | * @since 01/25/2011 20 | */ 21 | public abstract class InMemoryProcessing { 22 | 23 | /** 24 | * Method to deserialize the request. 25 | *

26 | * ATTENTION: Operation request serialization/deserialization has been replaced by the unified 27 | * transaction serialization scheme. 28 | *

29 | * 30 | * @param serialized - the serialized operation's arguments. 31 | * 32 | * @return deserialized operation's arguments. 33 | * 34 | * @throws BabuDBException if deserialization fails. 35 | */ 36 | @Deprecated 37 | public abstract Object[] deserializeRequest(ReusableBuffer serialized) throws BabuDBException; 38 | 39 | /** 40 | * Converts the operation retrieved from an old log entry of obsolete log file into a 41 | * transaction operation. 42 | * 43 | * @param args 44 | * @return a transaction for the given arguments. 45 | */ 46 | @Deprecated 47 | public abstract OperationInternal convertToOperation(Object[] args); 48 | 49 | /** 50 | * The database changes made by the operation can be read after this method returns. Although 51 | * there are no persistence guarantees on a database crash e.g. there is no entry written to the 52 | * log. 53 | * 54 | * @param operation 55 | * 56 | * @return a possible return value for the in-memory processing of the operation, may be null. 57 | * 58 | * @throws BabuDBException if the operation could not have been processed, due a user error. 59 | */ 60 | public abstract Object process(OperationInternal operation) throws BabuDBException; 61 | } 62 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/index/ByteRangeComparator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.api.index; 10 | 11 | import java.io.Serializable; 12 | import java.util.Comparator; 13 | 14 | import org.xtreemfs.babudb.index.ByteRange; 15 | 16 | /** 17 | * A comparator for byte buffers and byte ranges. 18 | *

19 | * Any implementation of this class should be serializable, so as to guarantee 20 | * that instances of custom ByteRangeComparators can be recorded in 21 | * the database log. 22 | *

23 | * 24 | * @author stender 25 | * 26 | */ 27 | public interface ByteRangeComparator extends Comparator, Serializable { 28 | 29 | /** 30 | * Compares a range of bytes from a potentially large buffer to the entire 31 | * content of a given buffer.
32 | * 33 | * This method should be implemented efficiently, as it may be invoked a 34 | * large number of times withe each database lookup. 35 | * 36 | * @param rng 37 | * the range 38 | * @param buf 39 | * the buffer 40 | * @return a negative value if buf is considered as smaller 41 | * than rng, 0 if both are considered as equal, and a 42 | * positive value if buf is considered as greater. 43 | */ 44 | public int compare(ByteRange rng, byte[] buf); 45 | 46 | /** 47 | * Converts a prefix to a range. The method is needed to translate prefix 48 | * queries into range queries. 49 | * 50 | * @param prefix 51 | * a buffer representing the prefix to query 52 | * @param ascending 53 | * if true, the lower bound is the first value in 54 | * the range; otherwise, it is the second value 55 | * @return an array consisting of two buffers, the first being the inclusive 56 | * lower bound of the range, the second being the exclusive upper 57 | * bound of the range 58 | */ 59 | public byte[][] prefixToRange(byte[] prefix, boolean ascending); 60 | 61 | } 62 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/api/transaction/TransactionListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.api.transaction; 10 | 11 | /** 12 | * A listener that observes lightweight BabuDB transactions. 13 | * 14 | * @author stenjan 15 | * 16 | */ 17 | public interface TransactionListener { 18 | 19 | /** 20 | * The method is invoked with each transaction that was executed. 21 | * 22 | * @param txn the executed transaction 23 | */ 24 | public void transactionPerformed(Transaction txn); 25 | 26 | } 27 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/config/PluginConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.config; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | import java.util.Properties; 13 | 14 | /** 15 | * Reading the Main class from a plugin's library. 16 | * 17 | * @author flangner 18 | * @since 04/15/2011 19 | */ 20 | public class PluginConfig extends Config { 21 | 22 | protected String pluginLibraryPath; 23 | 24 | public PluginConfig(Properties prop) { 25 | super(prop); 26 | read(); 27 | } 28 | 29 | public PluginConfig(String filename) throws IOException { 30 | super(filename); 31 | read(); 32 | } 33 | 34 | // for compatibility only 35 | public PluginConfig () { } 36 | 37 | private final void read() { 38 | pluginLibraryPath = readRequiredString("plugin.jar"); 39 | 40 | checkArgs(pluginLibraryPath); 41 | } 42 | 43 | private static void checkArgs(String pluginPath) { 44 | if (pluginPath == null || pluginPath == "") 45 | throw new IllegalArgumentException("Path to plugin's library jar is missing."); 46 | 47 | File f = new File(pluginPath); 48 | if (!f.exists() || !f.isFile()) 49 | throw new IllegalArgumentException("Path '" + pluginPath + "' does not exist, " + 50 | "or is not a file."); 51 | } 52 | 53 | public String getPluginLibraryPath() { 54 | return pluginLibraryPath; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/conversion/jars/3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xtreemfs/babudb/324aad93030066ab4fe1b57117baf52142d59813/java/babudb-core/src/main/java/org/xtreemfs/babudb/conversion/jars/3.jar -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/overlay/MultiOverlayBufferTree.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.overlay; 10 | 11 | import org.xtreemfs.babudb.api.database.ResultSet; 12 | import org.xtreemfs.babudb.api.index.ByteRangeComparator; 13 | 14 | public class MultiOverlayBufferTree extends MultiOverlayTree { 15 | 16 | private ByteRangeComparator comp; 17 | 18 | public MultiOverlayBufferTree(byte[] markerElement, ByteRangeComparator comp) { 19 | super(markerElement, comp); 20 | this.comp = comp; 21 | } 22 | 23 | public ResultSet prefixLookup(byte[] prefix, boolean includeDeletedEntries, 24 | boolean ascending) { 25 | 26 | byte[][] keyRange = comp.prefixToRange(prefix, ascending); 27 | assert (keyRange.length == 2); 28 | 29 | return rangeLookup(keyRange[0], keyRange[1], includeDeletedEntries, ascending); 30 | } 31 | 32 | public ResultSet prefixLookup(byte[] prefix, int overlayId, 33 | boolean includeDeletedEntries, boolean ascending) { 34 | 35 | byte[][] keyRange = comp.prefixToRange(prefix, ascending); 36 | assert (keyRange.length == 2); 37 | 38 | return rangeLookup(keyRange[0], keyRange[1], overlayId, includeDeletedEntries, ascending); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/overlay/MultiOverlayStringTree.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.overlay; 10 | 11 | import java.util.Iterator; 12 | import java.util.Map.Entry; 13 | 14 | public class MultiOverlayStringTree extends MultiOverlayTree { 15 | 16 | public MultiOverlayStringTree(V markerElement) { 17 | super(markerElement); 18 | } 19 | 20 | public Iterator> prefixLookup(String prefix, boolean includeDeletedEntries, 21 | boolean ascending) { 22 | return rangeLookup(prefix, getNextPrefix(prefix), includeDeletedEntries, ascending); 23 | } 24 | 25 | public Iterator> prefixLookup(String prefix, int overlayId, 26 | boolean includeDeletedEntries, boolean ascending) { 27 | return rangeLookup(prefix, getNextPrefix(prefix), overlayId, includeDeletedEntries, ascending); 28 | } 29 | 30 | private String getNextPrefix(String prefix) { 31 | 32 | if (prefix == null) 33 | return null; 34 | 35 | byte[] bytes = prefix.getBytes(); 36 | for (int i = bytes.length - 1; i >= 0; i--) { 37 | bytes[i]++; 38 | if (bytes[i] != Byte.MIN_VALUE) 39 | break; 40 | } 41 | 42 | return new String(bytes); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/reader/BlockReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.index.reader; 9 | 10 | import java.nio.ByteBuffer; 11 | 12 | import org.xtreemfs.babudb.api.database.ResultSet; 13 | import org.xtreemfs.babudb.api.index.ByteRangeComparator; 14 | import org.xtreemfs.babudb.index.ByteRange; 15 | import org.xtreemfs.foundation.buffer.BufferPool; 16 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 17 | 18 | /** 19 | * Base class for block readers. 20 | * 21 | * @author stenjan 22 | * 23 | */ 24 | public abstract class BlockReader { 25 | 26 | protected ByteBuffer buffer; // for buffered block readers 27 | 28 | protected ReusableBuffer readBuffer; // for streamed block readers 29 | 30 | protected int position; 31 | 32 | protected int limit; 33 | 34 | protected ByteRangeComparator comp; 35 | 36 | protected MiniPage keys; 37 | 38 | protected MiniPage values; 39 | 40 | protected int numEntries; 41 | 42 | protected final boolean isBuffered; 43 | 44 | protected BlockReader(boolean isBuffered) { 45 | this.isBuffered = isBuffered; 46 | } 47 | 48 | public BlockReader clone() { 49 | 50 | assert (isBuffered); 51 | 52 | buffer.position(0); 53 | return new DefaultBlockReader(buffer.slice(), position, limit, comp); 54 | } 55 | 56 | public abstract ByteRange lookup(byte[] key); 57 | 58 | public abstract ResultSet rangeLookup(byte[] from, byte[] to, 59 | final boolean ascending); 60 | 61 | public MiniPage getKeys() { 62 | return keys; 63 | } 64 | 65 | public MiniPage getValues() { 66 | return values; 67 | } 68 | 69 | public int getNumEntries() { 70 | return numEntries; 71 | } 72 | 73 | public void free() { 74 | if (readBuffer != null) 75 | BufferPool.free(readBuffer); 76 | } 77 | 78 | } -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/reader/FixedLenMiniPage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.reader; 10 | 11 | import java.nio.ByteBuffer; 12 | 13 | import org.xtreemfs.babudb.api.index.ByteRangeComparator; 14 | import org.xtreemfs.babudb.index.ByteRange; 15 | import org.xtreemfs.foundation.buffer.BufferPool; 16 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 17 | import org.xtreemfs.foundation.util.OutputUtils; 18 | 19 | public class FixedLenMiniPage extends MiniPage { 20 | 21 | private final int entrySize; 22 | 23 | private final int limit; 24 | 25 | public FixedLenMiniPage(int entrySize, int numEntries, ByteBuffer buf, int offset, int limit, 26 | ByteRangeComparator comp) { 27 | 28 | super(numEntries, buf, offset, comp); 29 | 30 | this.entrySize = entrySize; 31 | this.limit = limit; 32 | } 33 | 34 | public ByteRange getEntry(int n) { 35 | assert (offset < buf.limit()) : "offset == " + offset + ", buf.limit == " + buf.limit() 36 | + ", entrySize == " + entrySize + ", n == " + n; 37 | return new ByteRange(buf, offset + n * entrySize, offset + (n + 1) * entrySize - 1); 38 | } 39 | 40 | public String toString() { 41 | 42 | buf.position(offset); 43 | buf.limit(limit); 44 | ReusableBuffer newBuf = BufferPool.allocate(limit - buf.position()); 45 | newBuf.put(buf); 46 | String result = OutputUtils.byteArrayToFormattedHexString(newBuf.array()); 47 | BufferPool.free(newBuf); 48 | buf.clear(); 49 | 50 | return result; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/reader/InternalBufferUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.reader; 10 | 11 | import java.util.Map.Entry; 12 | 13 | import org.xtreemfs.babudb.index.ByteRange; 14 | 15 | /** 16 | * Utilities for internal buffer handling. 17 | * 18 | * @author stenjan 19 | * 20 | */ 21 | public class InternalBufferUtil { 22 | 23 | public static byte[] toBuffer(Object buf) { 24 | 25 | if (buf instanceof byte[]) 26 | return (byte[]) buf; 27 | else 28 | return ((ByteRange) buf).toBuffer(); 29 | 30 | } 31 | 32 | public static int size(Object buf) { 33 | 34 | if (buf instanceof byte[]) 35 | return ((byte[]) buf).length; 36 | else 37 | return ((ByteRange) buf).getSize(); 38 | 39 | } 40 | 41 | public static byte byteAt(Object buf, int offset) { 42 | 43 | if (buf instanceof byte[]) 44 | return ((byte[]) buf)[offset]; 45 | 46 | else { 47 | ByteRange range = (ByteRange) buf; 48 | 49 | assert (range.getSize() <= offset); 50 | return range.getBuf().get(range.getStartOffset() + offset); 51 | } 52 | } 53 | 54 | public static Entry cast(final Entry byteEntry) { 55 | 56 | Entry entry = new Entry() { 57 | 58 | @Override 59 | public Object getKey() { 60 | return byteEntry.getKey(); 61 | } 62 | 63 | @Override 64 | public Object getValue() { 65 | return byteEntry.getValue(); 66 | } 67 | 68 | @Override 69 | public Object setValue(Object value) { 70 | throw new UnsupportedOperationException(); 71 | } 72 | 73 | }; 74 | 75 | return entry; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/reader/VarLenMiniPage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.reader; 10 | 11 | import java.nio.ByteBuffer; 12 | 13 | import org.xtreemfs.babudb.api.index.ByteRangeComparator; 14 | import org.xtreemfs.babudb.index.ByteRange; 15 | import org.xtreemfs.foundation.buffer.BufferPool; 16 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 17 | import org.xtreemfs.foundation.util.OutputUtils; 18 | 19 | public class VarLenMiniPage extends MiniPage { 20 | 21 | private final int offsetListStart; 22 | 23 | public VarLenMiniPage(int numEntries, ByteBuffer buf, int offset, int limit, 24 | ByteRangeComparator comp) { 25 | 26 | super(numEntries, buf, offset, comp); 27 | 28 | // calculate the offset of the offset list 29 | offsetListStart = limit - numEntries * Integer.SIZE / 8; 30 | } 31 | 32 | public ByteRange getEntry(int n) { 33 | 34 | int offsetStart = offset; 35 | if (n > 0) 36 | offsetStart += buf.getInt(offsetListStart + (n - 1) * Integer.SIZE / 8); 37 | 38 | int offsetEnd = offset; 39 | offsetEnd += buf.getInt(offsetListStart + n * Integer.SIZE / 8); 40 | 41 | assert (offsetEnd > offsetStart); 42 | 43 | return new ByteRange(buf, offsetStart, offsetEnd); 44 | } 45 | 46 | public String toString() { 47 | 48 | buf.position(offset); 49 | buf.limit(offsetListStart + numEntries * Integer.SIZE / 8); 50 | ReusableBuffer newBuf = BufferPool.allocate(buf.limit() - buf.position()); 51 | newBuf.put(buf); 52 | String result = OutputUtils.byteArrayToFormattedHexString(newBuf.array()); 53 | BufferPool.free(newBuf); 54 | buf.clear(); 55 | 56 | return result; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/writer/BlockWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.writer; 10 | 11 | 12 | public interface BlockWriter { 13 | 14 | /** 15 | * Adds a new key-value pair to the writer. 16 | * 17 | * @param key 18 | * @param value 19 | */ 20 | public abstract void add(Object key, Object value); 21 | 22 | /** 23 | * Returns a serialized representation of all data previously added to the 24 | * block writer. Implementations may assume that this method will be only 25 | * invoked once, and that no more key-value-pairs will be added afterwards. 26 | * 27 | * @return a serialized representation of all data previously added to the 28 | * block writer 29 | */ 30 | public abstract SerializedBlock serialize(); 31 | 32 | /** 33 | * Returns the block key, i.e. the first key in the block. 34 | * 35 | * @return the block key 36 | */ 37 | public abstract Object getBlockKey(); 38 | 39 | } -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/writer/SerializedBlock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.writer; 10 | 11 | import java.util.Iterator; 12 | import java.util.LinkedList; 13 | import java.util.List; 14 | 15 | /** 16 | * A serialized representation of a block. 17 | * 18 | * @author stenjan 19 | * 20 | */ 21 | public class SerializedBlock { 22 | 23 | private List> multiList; 24 | 25 | private int size; 26 | 27 | public SerializedBlock() { 28 | multiList = new LinkedList>(); 29 | } 30 | 31 | public void addBuffers(int size, List... bufferList) { 32 | for (List l : bufferList) 33 | multiList.add(l); 34 | this.size += size; 35 | } 36 | 37 | public int size() { 38 | return size; 39 | } 40 | 41 | public Iterator iterator() { 42 | 43 | return new Iterator() { 44 | 45 | private Iterator currentIterator; 46 | 47 | private Iterator> multiListIterator; 48 | 49 | { 50 | multiListIterator = multiList.iterator(); 51 | currentIterator = multiListIterator.hasNext() ? multiListIterator.next().iterator() : null; 52 | } 53 | 54 | @Override 55 | public boolean hasNext() { 56 | return currentIterator != null && currentIterator.hasNext(); 57 | } 58 | 59 | @Override 60 | public Object next() { 61 | 62 | Object next = currentIterator.next(); 63 | if (!currentIterator.hasNext()) 64 | currentIterator = multiListIterator.hasNext() ? multiListIterator.next().iterator() 65 | : null; 66 | 67 | return next; 68 | } 69 | 70 | @Override 71 | public void remove() { 72 | throw new UnsupportedOperationException(); 73 | } 74 | }; 75 | 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/index/writer/SerializedPage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.index.writer; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * A serialized representation of a page. 15 | * 16 | * @author stenjan 17 | * 18 | */ 19 | public class SerializedPage { 20 | 21 | public SerializedPage(int size, List... entries) { 22 | this.entries = entries; 23 | this.size = size; 24 | } 25 | 26 | public final List[] entries; 27 | 28 | public final int size; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/log/LogEntryException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.log; 10 | 11 | /** 12 | * 13 | * @author bjko 14 | */ 15 | public class LogEntryException extends Exception { 16 | 17 | private static final long serialVersionUID = 2911195875170424834L; 18 | 19 | public LogEntryException(String message) { 20 | super(message); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/log/SyncListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.log; 10 | 11 | import org.xtreemfs.babudb.lsmdb.LSN; 12 | 13 | /** 14 | * Simple listener called after a LogEntry was synchronized or sent. 15 | * 16 | * @author bjko 17 | * @author flangner 18 | */ 19 | public interface SyncListener { 20 | 21 | /** Called after the LogEntry was synchronized to log-file. 22 | * @param lsn 23 | */ 24 | public void synced(LSN lsn); 25 | 26 | /** Called if the LogEntry could not be written/sent. 27 | */ 28 | public void failed(Exception ex); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/lsmdb/BabuDBInsertGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.lsmdb; 10 | 11 | import org.xtreemfs.babudb.api.database.DatabaseInsertGroup; 12 | 13 | 14 | /** 15 | * 16 | * @author bjko 17 | */ 18 | public class BabuDBInsertGroup implements DatabaseInsertGroup { 19 | 20 | private final InsertRecordGroup rec; 21 | 22 | BabuDBInsertGroup(LSMDatabase db) { 23 | this(db.getDatabaseId()); 24 | } 25 | 26 | BabuDBInsertGroup(int databaseId) { 27 | rec = new InsertRecordGroup(databaseId); 28 | } 29 | 30 | public InsertRecordGroup getRecord() { 31 | return rec; 32 | } 33 | 34 | /* (non-Javadoc) 35 | * @see org.xtreemfs.babudb.lsmdb.InsertGroup#addInsert(int, byte[], byte[]) 36 | */ 37 | @Override 38 | public void addInsert(int indexId, byte[] key, byte[] value) { 39 | rec.addInsert(indexId, key, value); 40 | } 41 | 42 | /* (non-Javadoc) 43 | * @see org.xtreemfs.babudb.lsmdb.InsertGroup#addDelete(int, byte[]) 44 | */ 45 | @Override 46 | public void addDelete(int indexId, byte[] key) { 47 | rec.addInsert(indexId, key, null); 48 | } 49 | 50 | public String toString() { 51 | return rec.toString(); 52 | } 53 | 54 | public static BabuDBInsertGroup createInsertGroup(int dbId) { 55 | return new BabuDBInsertGroup(dbId); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/lsmdb/LSMLookupInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | 6 | package org.xtreemfs.babudb.lsmdb; 7 | 8 | import java.util.Iterator; 9 | import java.util.Map.Entry; 10 | 11 | import org.xtreemfs.babudb.api.exception.BabuDBException; 12 | import org.xtreemfs.babudb.index.LSMTree; 13 | 14 | /** 15 | * 16 | * @author bjko 17 | */ 18 | public class LSMLookupInterface { 19 | 20 | private final LSMDatabase database; 21 | 22 | public LSMLookupInterface(LSMDatabase database) { 23 | this.database = database; 24 | } 25 | 26 | public byte[] lookup(int indexId, byte[] key) throws BabuDBException { 27 | LSMTree tree = database.getIndex(indexId); 28 | if (tree == null) 29 | throw new BabuDBException(BabuDBException.ErrorCode.NO_SUCH_INDEX, "index " + indexId + " does not exist"); 30 | return tree.lookup(key); 31 | } 32 | 33 | public byte[] lookup(int indexId, byte[] key, int snapId) throws BabuDBException { 34 | LSMTree tree = database.getIndex(indexId); 35 | if (tree == null) 36 | throw new BabuDBException(BabuDBException.ErrorCode.NO_SUCH_INDEX, "index " + indexId + " does not exist"); 37 | return tree.lookup(key,snapId); 38 | } 39 | 40 | public Iterator> prefixLookup(int indexId, byte[] startKey) throws BabuDBException { 41 | LSMTree tree = database.getIndex(indexId); 42 | if (tree == null) 43 | throw new BabuDBException(BabuDBException.ErrorCode.NO_SUCH_INDEX, "index does not exist"); 44 | return tree.prefixLookup(startKey); 45 | } 46 | 47 | public Iterator> prefixLookup(int indexId, byte[] startKey, int snapId) throws BabuDBException { 48 | LSMTree tree = database.getIndex(indexId); 49 | if (tree == null) 50 | throw new BabuDBException(BabuDBException.ErrorCode.NO_SUCH_INDEX, "index does not exist"); 51 | return tree.prefixLookup(startKey,snapId); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/lsmdb/LSN.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2008 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | 9 | package org.xtreemfs.babudb.lsmdb; 10 | 11 | /** 12 | * @author bjko 13 | * @author flangner 14 | */ 15 | public class LSN implements Comparable { 16 | 17 | private final int viewId; 18 | private final long sequenceNo; 19 | 20 | public LSN(int viewId, long sequenceNo) { 21 | this.viewId = viewId; 22 | this.sequenceNo = sequenceNo; 23 | } 24 | 25 | public LSN(byte[] lsn) throws NumberFormatException{ 26 | this(new String(lsn)); 27 | } 28 | 29 | /** 30 | *

Gets the LSN from given string representation.

31 | *

Pattern: "'viewId':'sequenceNo'"

32 | * 33 | * @param representation 34 | */ 35 | public LSN(String representation) throws NumberFormatException{ 36 | String[] rep = representation.split(":"); 37 | if (rep.length!=2) throw new NumberFormatException(representation+" is not a legal LSN string-representation."); 38 | 39 | viewId = Integer.parseInt(rep[0]); 40 | sequenceNo = Long.valueOf(rep[1]); 41 | } 42 | 43 | public int getViewId() { 44 | return viewId; 45 | } 46 | 47 | public long getSequenceNo() { 48 | return sequenceNo; 49 | } 50 | 51 | /* 52 | * (non-Javadoc) 53 | * @see java.lang.Comparable#compareTo(java.lang.Object) 54 | */ 55 | @Override 56 | public int compareTo(LSN o) { 57 | if (this.viewId > o.viewId) { 58 | return 1; 59 | } else if (this.viewId < o.viewId) { 60 | return -1; 61 | } else { 62 | if (this.sequenceNo > o.sequenceNo) { 63 | return 1; 64 | } else if (this.sequenceNo < o.sequenceNo) { 65 | return -1; 66 | } else 67 | return 0; 68 | } 69 | } 70 | 71 | /* 72 | * (non-Javadoc) 73 | * @see java.lang.Object#equals(java.lang.Object) 74 | */ 75 | @Override 76 | public boolean equals(Object obj) { 77 | if (obj instanceof LSN) 78 | return ((LSN) obj).viewId == viewId && ((LSN) obj).sequenceNo == sequenceNo; 79 | else 80 | return false; 81 | } 82 | 83 | /* 84 | * (non-Javadoc) 85 | * @see java.lang.Object#toString() 86 | */ 87 | @Override 88 | public String toString() { 89 | return viewId+":"+sequenceNo; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/sandbox/ReplicationLongrunTestConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.sandbox; 9 | 10 | /** 11 | * 12 | * @author flangner 13 | * 14 | */ 15 | 16 | public class ReplicationLongrunTestConfig { 17 | public static final boolean CLUSTER = false; 18 | 19 | // the interval to sleep, if any other event occurred before 20 | public final static int MIN_SLEEP_INTERVAL; 21 | public final static int MAX_SLEEP_INTERVAL; 22 | 23 | public final static int MAX_DOWN_TIME; 24 | 25 | public final static String PATH; 26 | 27 | public final static long MIN_SEQUENCENO; 28 | 29 | public final static long MAX_SEQUENCENO; 30 | 31 | static { 32 | if (CLUSTER){ 33 | MIN_SLEEP_INTERVAL = 20*60*1000; 34 | MAX_SLEEP_INTERVAL = 30*60*1000; 35 | MAX_DOWN_TIME = 5*60*1000; 36 | PATH = "/scratch/babuDB/data/"; 37 | MIN_SEQUENCENO = 1000L; 38 | MAX_SEQUENCENO = Integer.MAX_VALUE; 39 | }else{ 40 | MIN_SLEEP_INTERVAL = 5*1000; 41 | MAX_SLEEP_INTERVAL = 20*1000; 42 | MAX_DOWN_TIME = 30*1000; 43 | PATH = "/tmp/babuDB/"; 44 | MIN_SEQUENCENO = 10000L; 45 | MAX_SEQUENCENO = 20000L; 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/sandbox/SimpleDemo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | 6 | package org.xtreemfs.babudb.sandbox; 7 | 8 | import org.xtreemfs.babudb.BabuDBFactory; 9 | import org.xtreemfs.babudb.api.BabuDB; 10 | import org.xtreemfs.babudb.api.DatabaseManager; 11 | import org.xtreemfs.babudb.api.database.Database; 12 | import org.xtreemfs.babudb.api.database.DatabaseInsertGroup; 13 | import org.xtreemfs.babudb.api.exception.BabuDBException; 14 | import org.xtreemfs.babudb.config.BabuDBConfig; 15 | import org.xtreemfs.babudb.log.DiskLogger.SyncMode; 16 | 17 | public class SimpleDemo { 18 | 19 | public static void main(String[] args) throws InterruptedException { 20 | try { 21 | // start the database 22 | BabuDB databaseSystem = BabuDBFactory.createBabuDB(new BabuDBConfig("myDatabase/", "myDatabase/", 23 | 2, 1024 * 1024 * 16, 5 * 60, SyncMode.SYNC_WRITE, 0, 0, false, 16, 1024 * 1024 * 512)); 24 | DatabaseManager dbm = databaseSystem.getDatabaseManager(); 25 | 26 | // create a new database called myDB 27 | dbm.createDatabase("myDB", 2); 28 | Database db = dbm.getDatabase("myDB"); 29 | 30 | // create an insert group for atomic inserts 31 | DatabaseInsertGroup group = db.createInsertGroup(); 32 | 33 | // insert one key in each index 34 | group.addInsert(0, "Key1".getBytes(), "Value1".getBytes()); 35 | group.addInsert(1, "Key2".getBytes(), "Value2".getBytes()); 36 | 37 | // and execute group insert 38 | db.insert(group, null).get(); 39 | 40 | // now do a lookup 41 | byte[] result = db.lookup(0, "Key1".getBytes(), null).get(); 42 | 43 | // create a checkpoint for faster start-ups 44 | databaseSystem.getCheckpointer().checkpoint(); 45 | 46 | // shutdown database 47 | databaseSystem.shutdown(); 48 | } catch (BabuDBException ex) { 49 | ex.printStackTrace(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/snapshots/BabuDBView.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.snapshots; 9 | 10 | import org.xtreemfs.babudb.api.database.ResultSet; 11 | import org.xtreemfs.babudb.api.exception.BabuDBException; 12 | 13 | public interface BabuDBView { 14 | 15 | public byte[] directLookup(int indexId, byte[] key) throws BabuDBException; 16 | 17 | public ResultSet directPrefixLookup(int indexId, byte[] key, boolean ascending) 18 | throws BabuDBException; 19 | 20 | public ResultSet directRangeLookup(int indexId, byte[] from, byte[] to, 21 | boolean ascending) throws BabuDBException; 22 | 23 | public void shutdown() throws BabuDBException; 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/java/org/xtreemfs/babudb/snapshots/SnapshotConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.snapshots; 9 | 10 | import java.io.Serializable; 11 | 12 | /** 13 | * A set of configuration parameters for a new snapshot. 14 | * 15 | * @author stender 16 | * 17 | */ 18 | public interface SnapshotConfig extends Serializable { 19 | 20 | /** 21 | * Returns the name for the new snapshot. 22 | * 23 | * @return the name 24 | */ 25 | public String getName(); 26 | 27 | /** 28 | * Returns an array of indices to be included in the new snapshot. 29 | * 30 | * @return a set of indices 31 | */ 32 | public int[] getIndices(); 33 | 34 | /** 35 | * Returns an array of prefix keys that are supposed to be written to the 36 | * snapshot of the given index. This method is used to pre-select certain 37 | * key ranges, so that only parts of each index may have to be traversed 38 | * when writing the snapshot to disk. 39 | * 40 | * Implementations have to make sure that the array of prefix keys returned 41 | * for a given index map is sorted in ascending order, and that prefixes 42 | * from the array do not cover one another. 43 | * 44 | * @param index 45 | * the index 46 | * @return An array of byte arrays, where each byte array represents a key 47 | * prefix for the given index. If the prefix key is 48 | * null, the whole index will be traversed. 49 | */ 50 | public byte[][] getPrefixes(int index); 51 | 52 | /** 53 | * Checks if the given key in the given index is contained in the snapshot. 54 | * Note that this check will only be performed for keys that are covered by 55 | * (one of) the prefix keys returned by getPrefix(index). 56 | * 57 | * @param index 58 | * the index 59 | * @param key 60 | * the key 61 | * @return true, if the key is part of the snapshot, 62 | * false, otherwise 63 | */ 64 | public boolean containsKey(int index, byte[] key); 65 | 66 | } 67 | -------------------------------------------------------------------------------- /java/babudb-core/src/main/resources/config/babuDB.properties: -------------------------------------------------------------------------------- 1 | ##################################################################### 2 | # BabuDB configuration # 3 | ##################################################################### 4 | 5 | # optional debug level 6 | # (0 = emergency, 7 | # 1 = alert, 8 | # 2 = critical, 9 | # 3 = error, 10 | # 4 = warning, 11 | # 5 = notice, 12 | # 6 = info, 13 | # 7 = debug) 14 | #babudb.debug.level = 4 15 | 16 | # optional debug category 17 | #babudb.debug.category = all 18 | 19 | # name for the database configuration file 20 | #babudb.cfgFile = config.db 21 | 22 | # base directory to store database index snapshots in 23 | babudb.baseDir = /tmp/babuDB/base/ 24 | 25 | # directory in which the database logs are stored 26 | babudb.logDir = /tmp/babuDB/log/ 27 | 28 | # SyncMode the synchronization mode to use for the logFile 29 | # ASYNC - asynchronously write log entries (data is lost when system crashes). 30 | # FSYNC - executes an fsync on the logfile before acknowledging the operation. 31 | # FDATASYNC 32 | # SYNC_WRITE - synchronously writes the log entry to disk before ack. Does not 33 | # update the metadata. 34 | # SYNC_WRITE_METADATA - synchronously writes the log entry to disk and updates 35 | # the metadat before ack. 36 | babudb.sync = ASYNC 37 | 38 | # max queue length: if > 0, the queue for each worker is limited to maxQ 39 | #babudb.worker.maxQueueLength = 0 40 | 41 | # number of worker threads to use, if 0 requests will be inserted directly 42 | # without being queued. this may increase performance dramatically. 43 | #babudb.worker.numThreads = 1 44 | 45 | # a checkpoint is generated ,if maxLogfileSize is exceeded 46 | #babudb.maxLogfileSize = 1 47 | 48 | # interval between two checks in seconds, 0 disables auto checkPointing 49 | #babudb.checkInterval = 0 50 | 51 | # if set to a value > 0, operations are acknowledged immediately before 52 | # they are written to the disk log. The disk logger will do batch writes 53 | # and call fSync... every pseudoSyncWait seconds. This can be used to 54 | # increase performance and emulate PostgreSQL behavior. 55 | #babudb.pseudoSyncWait = 0 56 | 57 | # flag that determines whether the indices shall be compressed or not. 58 | #babudb.compression = false 59 | 60 | ##################################################################### 61 | # BabuDB plugins configuration # 62 | ##################################################################### 63 | 64 | #babudb.plugin.0 = replication/config/replication.properties -------------------------------------------------------------------------------- /java/babudb-core/src/main/resources/config/default-config.properties: -------------------------------------------------------------------------------- 1 | # optional debug level ( 2 | # 0 = emergency, 3 | # 1 = alert, 4 | # 2 = critical, 5 | # 3 = error, 6 | # 4 = warning, 7 | # 5 = notice, 8 | # 6 = info, 9 | # 7 = debug) 10 | babudb.debug.level = 4 11 | 12 | # optional debug category 13 | babudb.debug.category = all 14 | 15 | # base directory to store database index snapshots in 16 | babudb.baseDir = /tmp/babudb/database 17 | 18 | # directory in which the database logs are stored 19 | babudb.logDir = /tmp/babudb/db-log 20 | 21 | # SyncMode the synchronization mode to use for the logFile 22 | # ASYNC - asynchronously write log entries (data is lost when system crashes). 23 | # FSYNC - executes an fsync on the logfile before acknowledging the operation. 24 | # FDATASYNC 25 | # SYNC_WRITE - synchronously writes the log entry to disk before ack. Does not 26 | # update the metadata. 27 | # SYNC_WRITE_METADATA - synchronously writes the log entry to disk and updates 28 | # the metadata before ack. 29 | babudb.sync = ASYNC 30 | 31 | # max queue length: if > 0, the queue for each worker is limited to maxQ 32 | babudb.worker.maxQueueLength = 1000 33 | 34 | # number of worker threads to use 35 | babudb.worker.numThreads = 0 36 | 37 | # a checkpoint is generated ,if maxLogfileSize is exceeded 38 | babudb.maxLogfileSize = 16777216 39 | 40 | # interval between two checks in seconds, 0 disables auto checkPointing 41 | babudb.checkInterval = 300 42 | 43 | # if set to a value > 0, operations are acknowledged immediately before 44 | # they are written to the disk log. The disk logger will do batch writes 45 | # and call fSync... every pseudoSyncWait seconds. This can be used to 46 | # increase performance and emulate PostgreSQL behavior. 47 | babudb.pseudoSyncWait = 0 48 | 49 | # flag that determines whether the indices shall be compressed or not. 50 | babudb.compression = false 51 | 52 | # maximum number of key-value pairs per block 53 | babudb.maxNumRecordsPerBlock = 64 54 | 55 | # maximum size for a babudb on-disk index file 56 | babudb.maxBlockFileSize = 52428800 57 | 58 | # Disables memory-mapping of database files. Disabling mmap'ing may 59 | # reduce memory shortage at the cost of a slightly decreased read 60 | # performance. 61 | babudb.disableMmap = false 62 | 63 | # Sets a high watermark for the size of all databases after which 64 | # block files will no longer be mmap'ed. On 32-bit VMs, setting such 65 | # a limit is necessary to deal with databases in GB size. If set to 66 | # -1, no limit will be enforced. 67 | babudb.mmapLimit = -1 -------------------------------------------------------------------------------- /java/babudb-replication/.gitignore: -------------------------------------------------------------------------------- 1 | # protobuf generated code 2 | src/main/java/com/ 3 | src/main/java/org/xtreemfs/babudb/pbrpc/ 4 | -------------------------------------------------------------------------------- /java/babudb-replication/AUTHORS: -------------------------------------------------------------------------------- 1 | Jan Stender (Zuse Institute Berlin) 2 | Bjoern Kolbeck (Zuse Institute Berlin) 3 | Mikael Hoegqvist (Zuse Institute Berlin) 4 | Felix Hupfeld 5 | Felix Langner (Zuse Institute Berlin) -------------------------------------------------------------------------------- /java/babudb-replication/ChangeLog: -------------------------------------------------------------------------------- 1 | 2011-04-21: 2 | * Initial release of the replication plug-in. The plug-in provides transparent replication service with automatic master failover and request redirection 3 | 4 | 2011-05-19: 5 | * bugfix: state synchronization and fail-over now work properly in the event of a restart 6 | * bugfix: prefix lookups now return records in the correct order 7 | * bugfix: context objects are no longer lost on replicated db inserts 8 | * reduced logging output 9 | 10 | 2011-06-06: 11 | * adopted plug-in to the latest BabuDB changes 12 | * bugfix: buffer finalization for transactions now works properly 13 | * bugfix: fixed issue that caused a sending of wrongly encoded responses on a proxy make persistent operation 14 | 15 | 2011-08-26: 16 | * various bugfixes and robustness improvements 17 | * added configurable retry mechanism to make temporary replication failures transparent to applications 18 | * added information about the current replication master to monitoring interface 19 | 20 | 2011-10-12: 21 | * updated to latest Flease version 22 | * fixed a bug with the encoding of flease messages 23 | 24 | 2011-12-01: 25 | * enhanced log-output for time-drift detection, such that the relevant participants are shown -------------------------------------------------------------------------------- /java/babudb-replication/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 2 | Felix Hupfeld, Felix Langner, Zuse Institute Berlin 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above 10 | copyright notice, this list of conditions and the 11 | following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials 15 | provided with the distribution. 16 | * Neither the name of the Zuse Institute Berlin nor the 17 | names of its contributors may be used to endorse or promote 18 | products derived from this software without specific prior 19 | written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 | COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 31 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /java/babudb-replication/README: -------------------------------------------------------------------------------- 1 | For further details on how to use the BabuDB replication plug-in, please refer to the BabuDB Wiki: 2 | 3 | http://code.google.com/p/babudb/wiki/UsageReplicationForJava -------------------------------------------------------------------------------- /java/babudb-replication/src/main/interface/Common.proto: -------------------------------------------------------------------------------- 1 | option java_package="org.xtreemfs.babudb.pbrpc"; 2 | package org.xtreemfs.pbrpc; 3 | 4 | message emptyRequest { 5 | } 6 | 7 | message emptyResponse { 8 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/interface/GlobalTypes.proto: -------------------------------------------------------------------------------- 1 | package org.xtreemfs.pbrpc; 2 | option java_package="org.xtreemfs.babudb.pbrpc"; 3 | 4 | message ErrorCodeResponse { 5 | optional fixed32 error_code = 1[default = 0]; 6 | } 7 | 8 | message Timestamp { 9 | optional fixed32 error_code = 1[default = 0]; 10 | optional fixed64 value = 2[default = 0]; 11 | } 12 | 13 | message DBFileMetaData { 14 | required string file_name = 1; 15 | required fixed64 file_size = 2; 16 | } 17 | 18 | message DBFileMetaDatas { 19 | optional fixed32 error_code = 1[default = 0]; 20 | optional fixed32 max_chunk_size = 2[default = 5242880]; 21 | repeated DBFileMetaData db_file_metadatas = 3; 22 | } 23 | 24 | message LogEntry { 25 | required fixed64 length = 1; 26 | } 27 | 28 | message LogEntries { 29 | optional fixed32 error_code = 1[default = 0]; 30 | repeated LogEntry log_entries = 2; 31 | } 32 | 33 | message LSN { 34 | required fixed32 view_id = 1; 35 | required fixed64 sequence_no = 2; 36 | } 37 | 38 | message HeartbeatMessage { 39 | required fixed32 port = 1; 40 | required LSN lsn = 2; 41 | } 42 | 43 | message Chunk { 44 | required string file_name = 1; 45 | required fixed64 start = 2; 46 | required fixed64 end = 3; 47 | } 48 | 49 | message LSNRange { 50 | required LSN start = 1; 51 | required LSN end = 2; 52 | } 53 | 54 | message InetAddress { 55 | required string host = 1; 56 | required fixed32 port = 2; 57 | } 58 | 59 | message FLease { 60 | required string host = 1; 61 | required fixed32 port = 2; 62 | } 63 | 64 | message Databases { 65 | repeated Database database = 1; 66 | } 67 | 68 | message Database { 69 | optional fixed32 error_code = 1[default = 0]; 70 | required string database_name = 2; 71 | required fixed32 database_id = 3; 72 | } 73 | 74 | message DatabaseName { 75 | required string database_name = 1; 76 | } 77 | 78 | message DatabaseId { 79 | required fixed32 database_id = 1; 80 | } 81 | 82 | message Lookup { 83 | required string database_name = 1; 84 | required fixed32 index_id = 2; 85 | } 86 | 87 | message RangeLookup { 88 | required string database_name = 1; 89 | required fixed32 index_id = 2; 90 | required fixed32 from_length = 3; 91 | } 92 | 93 | message EntryMap { 94 | optional fixed32 error_code = 1[default = 0]; 95 | repeated fixed32 length = 2; 96 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/interface/PBRPC.proto: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2009-2010, Konrad-Zuse-Zentrum fuer Informationstechnik Berlin 3 | // 4 | // All rights reserved. 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are met: 8 | // 9 | // Redistributions of source code must retain the above copyright notice, this 10 | // list of conditions and the following disclaimer. 11 | // Redistributions in binary form must reproduce the above copyright notice, 12 | // this list of conditions and the following disclaimer in the documentation 13 | // and/or other materials provided with the distribution. 14 | // Neither the name of the Konrad-Zuse-Zentrum fuer Informationstechnik Berlin 15 | // nor the names of its contributors may be used to endorse or promote products 16 | // derived from this software without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | // POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // AUTHORS: Bjoern Kolbeck (ZIB) 31 | // 32 | 33 | 34 | option java_package="org.xtreemfs.babudb.pbrpc.generatedinterfaces"; 35 | package org.xtreemfs.pbrpc; 36 | import "google/protobuf/descriptor.proto"; 37 | 38 | extend google.protobuf.MethodOptions { 39 | optional fixed32 proc_id = 50001; 40 | optional bool data_in = 50004; 41 | optional bool data_out = 50003; 42 | } 43 | 44 | extend google.protobuf.ServiceOptions { 45 | optional fixed32 interface_id = 50002; 46 | } 47 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/interface/RemoteAccess.proto: -------------------------------------------------------------------------------- 1 | package org.xtreemfs.pbrpc; 2 | option java_package = "org.xtreemfs.babudb.pbrpc"; 3 | import "Common.proto"; 4 | import "GlobalTypes.proto"; 5 | import "PBRPC.proto"; 6 | 7 | service RemoteAccessService { 8 | option(interface_id)=10001; 9 | 10 | rpc makePersistent(emptyRequest) returns(Database) { 11 | option(proc_id)=1; 12 | option(data_in)=true; 13 | }; 14 | 15 | rpc getDatabaseByName(DatabaseName) returns(Database) { 16 | option(proc_id)=2; 17 | }; 18 | 19 | rpc getDatabaseById(DatabaseId) returns(Database) { 20 | option(proc_id)=3; 21 | }; 22 | 23 | rpc getDatabases(emptyRequest) returns(Databases) { 24 | option(proc_id)=4; 25 | }; 26 | 27 | rpc lookup(Lookup) returns(ErrorCodeResponse) { 28 | option(proc_id)=5; 29 | option(data_in)=true; 30 | option(data_out)=true; 31 | }; 32 | 33 | rpc plookup(Lookup) returns(EntryMap) { 34 | option(proc_id)=6; 35 | option(data_in)=true; 36 | option(data_out)=true; 37 | }; 38 | 39 | rpc plookupReverse(Lookup) returns(EntryMap) { 40 | option(proc_id)=7; 41 | option(data_in)=true; 42 | option(data_out)=true; 43 | }; 44 | 45 | rpc rlookup(RangeLookup) returns(EntryMap) { 46 | option(proc_id)=8; 47 | option(data_in)=true; 48 | option(data_out)=true; 49 | }; 50 | 51 | rpc rlookupReverse(RangeLookup) returns(EntryMap) { 52 | option(proc_id)=9; 53 | option(data_in)=true; 54 | option(data_out)=true; 55 | }; 56 | } 57 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/interface/Replication.proto: -------------------------------------------------------------------------------- 1 | package org.xtreemfs.pbrpc; 2 | option java_package = "org.xtreemfs.babudb.pbrpc"; 3 | import "Common.proto"; 4 | import "GlobalTypes.proto"; 5 | import "PBRPC.proto"; 6 | 7 | service ReplicationService { 8 | option(interface_id)=20001; 9 | 10 | rpc state(emptyRequest) returns(LSN) { 11 | option(proc_id)=1; 12 | }; 13 | 14 | rpc load(LSN) returns(DBFileMetaDatas) { 15 | option(proc_id)=2; 16 | }; 17 | 18 | rpc chunk(Chunk) returns(ErrorCodeResponse) { 19 | option(proc_id)=3; 20 | option(data_out)=true; 21 | }; 22 | 23 | rpc flease(FLease) returns(ErrorCodeResponse) { 24 | option(proc_id)=4; 25 | option(data_in)=true; 26 | }; 27 | 28 | rpc localTime(emptyRequest) returns(Timestamp) { 29 | option(proc_id)=5; 30 | }; 31 | 32 | rpc replica(LSNRange) returns(LogEntries) { 33 | option(proc_id)=6; 34 | option(data_out)=true; 35 | }; 36 | 37 | rpc heartbeat(HeartbeatMessage) returns(ErrorCodeResponse) { 38 | option(proc_id)=7; 39 | }; 40 | 41 | rpc replicate(LSN) returns(ErrorCodeResponse) { 42 | option(proc_id)=8; 43 | option(data_in)=true; 44 | }; 45 | 46 | rpc synchronize(HeartbeatMessage) returns(ErrorCodeResponse) { 47 | option(proc_id)=9; 48 | }; 49 | 50 | rpc volatileState(emptyRequest) returns(LSN) { 51 | option(proc_id)=10; 52 | }; 53 | } 54 | 55 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/config/DependencyConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.config; 9 | 10 | import java.io.IOException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | /** 15 | * Class to read the dependency paths from properties file. 16 | * 17 | * @author flangner 18 | * @since 04/01/2011 19 | */ 20 | public class DependencyConfig extends Config { 21 | 22 | private List paths = new ArrayList(); 23 | 24 | public DependencyConfig(String path) throws IOException { 25 | super(path); 26 | read(); 27 | } 28 | 29 | private void read() { 30 | int i = 0; 31 | String path; 32 | while ((path = readOptionalString("babudb.repl.dependency." + i++, null)) != null) { 33 | paths.add(path); 34 | } 35 | } 36 | 37 | public String[] getDependencyPaths() { 38 | return paths.toArray(new String[paths.size()]); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/FleaseMessageReceiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | import org.xtreemfs.foundation.flease.comm.FleaseMessage; 11 | 12 | /** 13 | * Receives {@link FleaseMessage}s. 14 | * 15 | * @author flangner 16 | * @since 04/15/2010 17 | */ 18 | public interface FleaseMessageReceiver { 19 | 20 | /** 21 | * Method to receive a {@link FleaseMessage}. 22 | * @param message 23 | */ 24 | public void receive(FleaseMessage message); 25 | } 26 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/Layer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | import org.xtreemfs.foundation.LifeCycleListener; 11 | import org.xtreemfs.foundation.LifeCycleThread; 12 | 13 | /** 14 | *

15 | * Abstraction of the transmission facilities used for the replication. 16 | * Includes interfaces for the layer above. 17 | *

18 | * 19 | * @author flangner 20 | * @since 04/13/2010 21 | */ 22 | public abstract class Layer { 23 | 24 | /** listener for life cycle events */ 25 | protected LifeCycleListener listener = null; 26 | 27 | /** 28 | * Sets the given {@link LifeCycleListener} for all {@link LifeCycleThread}s 29 | * of this {@link Layer}. 30 | * 31 | * @param listener - the {@link LifeCycleListener}. 32 | */ 33 | public synchronized void setLifeCycleListener(LifeCycleListener listener) { 34 | assert (listener != null); 35 | 36 | if (this.listener == null) { 37 | this.listener = listener; 38 | } 39 | } 40 | 41 | /** 42 | * Sets the given {@link LifeCycleListener} for all {@link LifeCycleThread}s 43 | * of this {@link Layer}. 44 | * -internal method- 45 | * Use setLifeCycleListener instead! 46 | * 47 | * @param listener - the {@link LifeCycleListener}. 48 | */ 49 | public abstract void _setLifeCycleListener(LifeCycleListener listener); 50 | 51 | /** 52 | * Start the services embedded into this layer. 53 | */ 54 | public abstract void start(); 55 | 56 | /** 57 | * Shutdown the services of this layer without waiting until they are not 58 | * running anymore. 59 | */ 60 | public abstract void asyncShutdown(); 61 | 62 | /** 63 | * Initializes shutdown sequence, synchronously waiting until its finished. 64 | */ 65 | public abstract void shutdown(); 66 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/LockableService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | /** 11 | * Interface that describes a replication service which may be locked by the replication controller. 12 | * 13 | * @author flangner 14 | * @since 02/16/2011 15 | */ 16 | public interface LockableService { 17 | 18 | /** 19 | * Ensures that the service is not used currently and may not be used throwing, 20 | * until service is unlocked again. A steady state is assured. 21 | * 22 | * @throws InterruptedException if waiting for pending requests to be finished has failed. 23 | */ 24 | public void lock() throws InterruptedException; 25 | 26 | /** 27 | * Method to unlock the service. Has to be locked before. 28 | */ 29 | public void unlock(); 30 | } 31 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/TopLayer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | import org.xtreemfs.babudb.api.BabuDB; 11 | import org.xtreemfs.babudb.replication.control.ControlLayerInterface; 12 | 13 | /** 14 | * Interface for {@link BabuDB} controlling the replication mechanism. 15 | * 16 | * @author flangner 17 | * @since 04/15/2010 18 | */ 19 | public abstract class TopLayer extends Layer implements ControlLayerInterface {} 20 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/control/ControlLayerInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.control; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | import org.xtreemfs.babudb.replication.BabuDBInterface; 13 | import org.xtreemfs.babudb.replication.FleaseMessageReceiver; 14 | import org.xtreemfs.babudb.replication.LockableService; 15 | import org.xtreemfs.babudb.replication.control.TimeDriftDetector.TimeDriftListener; 16 | 17 | /** 18 | * Interface between ControlLayer and {@link BabuDBInterface}. 19 | * 20 | * @author flangner 21 | * @since 04/15/2010 22 | */ 23 | public interface ControlLayerInterface extends TimeDriftListener, FleaseMessageReceiver, 24 | FleaseEventListener{ 25 | 26 | /** 27 | * Waits until a lease holder becomes available if necessary. 28 | * @param timeout - in ms to wait for a lease holder to become available. 29 | * May be 0 to wait forever. If timeout < 0 there will be no waiting. 30 | * 31 | * @return the address of the current lease holder. 32 | */ 33 | public InetSocketAddress getLeaseHolder(int timeout) throws InterruptedException; 34 | 35 | /** 36 | * @return the address of this server. 37 | */ 38 | public InetSocketAddress getThisAddress(); 39 | 40 | /** 41 | * Use only at initialization. Waits for the first failover to happen. 42 | * 43 | * @throws InterruptedException if waiting was interrupted. 44 | */ 45 | public void waitForInitialFailover() throws InterruptedException; 46 | 47 | /** 48 | * Method to register a {@link LockableService} to the control layer. 49 | * 50 | * @param service 51 | */ 52 | public void registerReplicationControl(LockableService service); 53 | 54 | /** 55 | * Locks replication only. 56 | * 57 | * @throws InterruptedException 58 | */ 59 | public void lockReplication() throws InterruptedException; 60 | 61 | /** 62 | * Unlocks the replication related services. 63 | */ 64 | public void unlockReplication(); 65 | 66 | /** 67 | * @return true, if there is currently a failover in progress, false otherwise. 68 | */ 69 | public boolean isFailoverInProgress(); 70 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/control/FleaseEventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.control; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | import org.xtreemfs.foundation.flease.Flease; 13 | 14 | /** 15 | * Interface for receiving filtered {@link Flease} events. 16 | * 17 | * @author flangner 18 | * @since 02/17/2011 19 | */ 20 | public interface FleaseEventListener { 21 | 22 | /** 23 | * Method to execute if a new leaseholder has to be announced. 24 | * 25 | * @param leaseholder 26 | */ 27 | public void updateLeaseHolder(InetSocketAddress leaseholder); 28 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/policy/MasterOnly.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | /* 9 | * AUTHORS: Felix Langner (ZIB) 10 | */ 11 | package org.xtreemfs.babudb.replication.policy; 12 | 13 | /** 14 | * Default replication policy. Any kind of operation has to be performed at the 15 | * master server in first case. This policy has the worst performance, but best 16 | * consistency for the data stored at the database. If you are not sure how the 17 | * queries for your database look like exactly, this is the only policy you 18 | * should use. 19 | * 20 | * @author flangner 21 | * @since 01/19/2011 22 | */ 23 | public final class MasterOnly implements Policy { 24 | 25 | /* (non-Javadoc) 26 | * @see org.xtreemfs.babudb.replication.policy.Policy#insertIsMasterRestricted() 27 | */ 28 | @Override 29 | public boolean insertIsMasterRestricted() { 30 | return true; 31 | } 32 | 33 | /* (non-Javadoc) 34 | * @see org.xtreemfs.babudb.replication.policy.Policy#snapshotManipultationIsMasterRestricted() 35 | */ 36 | @Override 37 | public boolean snapshotManipultationIsMasterRestricted() { 38 | return true; 39 | } 40 | 41 | /* (non-Javadoc) 42 | * @see org.xtreemfs.babudb.replication.policy.Policy#dbModificationIsMasterRestricted() 43 | */ 44 | @Override 45 | public boolean dbModificationIsMasterRestricted() { 46 | return true; 47 | } 48 | 49 | /* (non-Javadoc) 50 | * @see org.xtreemfs.babudb.replication.policy.Policy#lookUpIsMasterRestricted() 51 | */ 52 | @Override 53 | public boolean lookUpIsMasterRestricted() { 54 | return true; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/policy/Policy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | /* 9 | * AUTHORS: Felix Langner (ZIB) 10 | */ 11 | package org.xtreemfs.babudb.replication.policy; 12 | 13 | /** 14 | * Interface to implement replication policies. Decide for each kind of database 15 | * operation if it is necessary to perform in on the master only. 16 | * 17 | * @author flangner 18 | * @since 01/19/2011 19 | */ 20 | public interface Policy { 21 | 22 | /** 23 | * @return true, if user inserts are only allowed to be performed at the 24 | * replication master. false, if also slaves are allowed to perform 25 | * inserts directly (these are also replicated). 26 | */ 27 | public boolean insertIsMasterRestricted(); 28 | 29 | /** 30 | * @return true, if snapshot manipulations are only allowed to be performed 31 | * at the replication master. false, if also slaves are allowed to 32 | * perform snapshot operations directly (these are also replicated). 33 | */ 34 | public boolean snapshotManipultationIsMasterRestricted(); 35 | 36 | /** 37 | * @return true, if database manipulations are only allowed to be performed 38 | * at the replication master. false, if also slaves are allowed to 39 | * perform database manipulating operations directly (these are also 40 | * replicated). 41 | */ 42 | public boolean dbModificationIsMasterRestricted(); 43 | 44 | /** 45 | * @return true, if lookup operations are only allowed to be performed at 46 | * the replication master. false, if also slaves are allowed to 47 | * perform lookups directly. 48 | */ 49 | public boolean lookUpIsMasterRestricted(); 50 | } 51 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/policy/WriteRestriction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | /* 9 | * AUTHORS: Felix Langner (ZIB) 10 | */ 11 | package org.xtreemfs.babudb.replication.policy; 12 | 13 | /** 14 | * This policy allows lookups at the slaves. 15 | * Be careful consistency may suffer! 16 | * 17 | * @author flangner 18 | * @since 05/17/2011 19 | */ 20 | public final class WriteRestriction implements Policy { 21 | 22 | /* (non-Javadoc) 23 | * @see org.xtreemfs.babudb.replication.policy.Policy#insertIsMasterRestricted() 24 | */ 25 | @Override 26 | public boolean insertIsMasterRestricted() { 27 | return true; 28 | } 29 | 30 | /* (non-Javadoc) 31 | * @see org.xtreemfs.babudb.replication.policy.Policy#snapshotManipultationIsMasterRestricted() 32 | */ 33 | @Override 34 | public boolean snapshotManipultationIsMasterRestricted() { 35 | return true; 36 | } 37 | 38 | /* (non-Javadoc) 39 | * @see org.xtreemfs.babudb.replication.policy.Policy#dbModificationIsMasterRestricted() 40 | */ 41 | @Override 42 | public boolean dbModificationIsMasterRestricted() { 43 | return true; 44 | } 45 | 46 | /* (non-Javadoc) 47 | * @see org.xtreemfs.babudb.replication.policy.Policy#lookUpIsMasterRestricted() 48 | */ 49 | @Override 50 | public boolean lookUpIsMasterRestricted() { 51 | return false; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/RequestManagement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | import org.xtreemfs.babudb.lsmdb.LSN; 13 | import org.xtreemfs.babudb.replication.control.ControlLayerInterface; 14 | import org.xtreemfs.babudb.replication.service.ReplicationStage.BusyServerException; 15 | 16 | /** 17 | * Request-flow controlling methods of the {@link ReplicationStage}. 18 | * 19 | * @author flangner 20 | * @since 04/14/2010 21 | */ 22 | public interface RequestManagement { 23 | 24 | /** 25 | * Method to delegate dispatched replicate-requests from the replication logic itself. 26 | * 27 | * @param rq - the request 28 | * 29 | * @throws BusyServerException if queue-size has reached MAX_Q. 30 | */ 31 | public void enqueueOperation(Object[] rq) throws BusyServerException; 32 | 33 | /** 34 | * This method is used to synchronize a server with the master to establish a global stable 35 | * state. 36 | * 37 | * @param lastOnView 38 | * @param master 39 | * @param control 40 | * 41 | * @throws InterruptedException if creation of a stable state was interrupted by shutdown 42 | * or crash. 43 | */ 44 | public void createStableState(LSN lastOnView, InetSocketAddress master, ControlLayerInterface control) 45 | throws InterruptedException; 46 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/SlaveView.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service; 9 | 10 | import org.xtreemfs.babudb.lsmdb.LSN; 11 | import org.xtreemfs.babudb.replication.service.clients.MasterClient; 12 | 13 | /** 14 | * Interface to provide a slave view on the service layer. 15 | * 16 | * @author flangner 17 | * @since 04/14/2010 18 | */ 19 | public interface SlaveView { 20 | 21 | /** 22 | * Chooses a server from the given list which is at least that up-to-date to 23 | * synchronize with this server. 24 | * 25 | * @param progressAtLeast 26 | * @return a master client retrieved from the list, or null if none of the 27 | * given babuDBs has the required information. 28 | */ 29 | public MasterClient getSynchronizationPartner(LSN progressAtLeast); 30 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/StageRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service; 9 | 10 | import org.xtreemfs.babudb.log.LogEntry; 11 | import org.xtreemfs.babudb.lsmdb.LSN; 12 | import org.xtreemfs.foundation.buffer.BufferPool; 13 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 14 | 15 | /** 16 | * Wrapper for stage requests. 17 | * 18 | * @author flangner 19 | * @since 06/08/2009 20 | */ 21 | 22 | public class StageRequest implements Comparable{ 23 | 24 | private Object[] args; 25 | 26 | private final LSN lsn; 27 | 28 | /** 29 | * @param args - first argument has to be the {@link LSN} for ordering the requests. 30 | */ 31 | public StageRequest(Object[] args) { 32 | this.args = args; 33 | if (args != null && args[0] instanceof LSN) 34 | this.lsn = (LSN) args[0]; 35 | else 36 | this.lsn = null; 37 | } 38 | 39 | public Object[] getArgs() { 40 | return args; 41 | } 42 | 43 | public LSN getLSN() { 44 | return lsn; 45 | } 46 | 47 | /* 48 | * (non-Javadoc) 49 | * @see java.lang.Comparable#compareTo(java.lang.Object) 50 | */ 51 | @Override 52 | public int compareTo(StageRequest o) { 53 | if (lsn != null && o.lsn != null) 54 | return lsn.compareTo(o.lsn); 55 | else return 1; 56 | } 57 | 58 | /** 59 | * Frees all reusable buffers on the arguments list. 60 | */ 61 | public void free() { 62 | for (Object arg : args) { 63 | if (arg instanceof ReusableBuffer) { 64 | BufferPool.free((ReusableBuffer) arg); 65 | } else if (arg instanceof LogEntry) { 66 | ((LogEntry) arg).free(); 67 | } 68 | } 69 | } 70 | 71 | /* 72 | * (non-Javadoc) 73 | * @see java.lang.Object#toString() 74 | */ 75 | @Override 76 | public String toString() { 77 | String result = "LSN: "+lsn.toString() +"\n"; 78 | result += "ARGS: "+args.toString(); 79 | return result; 80 | } 81 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/accounting/LatestLSNUpdateListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.accounting; 9 | 10 | import org.xtreemfs.babudb.lsmdb.LSN; 11 | 12 | /** 13 | * Instance that has to be informed about latest {@link LSN} changes. 14 | * 15 | * @author flangner 16 | * @since 06/05/2009 17 | */ 18 | 19 | public abstract class LatestLSNUpdateListener implements Comparable { 20 | 21 | protected final LSN lsn; 22 | 23 | public LatestLSNUpdateListener(LSN lsn) { 24 | this.lsn = lsn; 25 | } 26 | 27 | /** 28 | * Function to call, if the latest {@link LSN} has been changed. 29 | * 30 | */ 31 | public abstract void upToDate(); 32 | 33 | /** 34 | * Function to call, if the listener is outdated before its {@link LSN} was reached. 35 | */ 36 | public abstract void failed(); 37 | 38 | /* 39 | * (non-Javadoc) 40 | * @see java.lang.Comparable#compareTo(java.lang.Object) 41 | */ 42 | @Override 43 | public int compareTo(LatestLSNUpdateListener o) { 44 | return lsn.compareTo(o.lsn); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/accounting/ParticipantsOverview.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.accounting; 9 | 10 | import java.net.InetSocketAddress; 11 | import java.util.List; 12 | 13 | import org.xtreemfs.babudb.replication.service.accounting.ParticipantsStates.UnknownParticipantException; 14 | import org.xtreemfs.babudb.replication.service.clients.ConditionClient; 15 | 16 | /** 17 | * Interface to the {@link ConditionClient}s of servers participating at the 18 | * replication. 19 | * 20 | * @author flangner 21 | * @since 04/13/2010 22 | */ 23 | public interface ParticipantsOverview { 24 | 25 | /** 26 | * @return a list of all {@link ConditionClient}s descending sorted by the last acknowledged 27 | * LSN of their services. 28 | */ 29 | public List getConditionClients(); 30 | 31 | /** 32 | * @param address 33 | * 34 | * @throws UnknownParticipantException if the address of at least one participant could not have 35 | * been resolved. 36 | * 37 | * @return a {@link ConditionClient} retrieved by the address of the server 38 | * it connects to. 39 | */ 40 | public ConditionClient getByAddress(InetSocketAddress address) throws UnknownParticipantException; 41 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/accounting/StatesManipulation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.accounting; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | import org.xtreemfs.babudb.lsmdb.LSN; 13 | import org.xtreemfs.babudb.replication.service.accounting.ParticipantsStates.UnknownParticipantException; 14 | import org.xtreemfs.babudb.replication.service.clients.ClientInterface; 15 | import org.xtreemfs.babudb.replication.service.clients.SlaveClient; 16 | 17 | /** 18 | *

19 | * Interface for the {@link ParticipantsStates} to manipulate them using 20 | * informations received via other components of the system. 21 | *

22 | * 23 | * @author flangner 24 | * @since 04/13/2010 25 | */ 26 | public interface StatesManipulation { 27 | 28 | /** 29 | *

30 | * Updates the state for the given participant using the receiveTime and 31 | * the acknowledgedLSN. 32 | *

33 | * 34 | * @param participant 35 | * @param acknowledgedLSN 36 | * @param receiveTime 37 | * 38 | * @throws UnknownParticipantException if the participant is not registered or it's address 39 | * could not have been determined. 40 | */ 41 | public void update(InetSocketAddress participant, LSN acknowledgedLSN, 42 | long receiveTime) throws UnknownParticipantException; 43 | 44 | /** 45 | *

46 | * Marks a slave manually as dead and decrements the open requests for the 47 | * slave. 48 | *

49 | * 50 | * @param slave 51 | */ 52 | public void markAsDead(ClientInterface slave); 53 | 54 | /** 55 | *

Decrements the open requests for the slave.

56 | * 57 | * @param slave 58 | */ 59 | public void requestFinished(SlaveClient slave); 60 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/clients/ClientInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.clients; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | /** 13 | * Default functionality of a client for a specified server. 14 | * 15 | * @author flangner 16 | * @since 04/13/2010 17 | */ 18 | public interface ClientInterface { 19 | 20 | /* (non-Javadoc) 21 | * @see java.lang.Object#toString() 22 | */ 23 | @Override 24 | public String toString(); 25 | 26 | /* (non-Javadoc) 27 | * @see java.lang.Object#equals(java.lang.Object) 28 | */ 29 | @Override 30 | public boolean equals(Object obj); 31 | 32 | /** 33 | * @return the address of the server this client addresses. 34 | */ 35 | public InetSocketAddress getDefaultServerAddress(); 36 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/clients/MasterClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.clients; 9 | 10 | import org.xtreemfs.babudb.log.LogEntry; 11 | import org.xtreemfs.babudb.lsmdb.LSN; 12 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.DBFileMetaDatas; 13 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.ErrorCodeResponse; 14 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.LogEntries; 15 | import org.xtreemfs.babudb.replication.service.logic.LoadLogic.DBFileMetaDataSet; 16 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 17 | 18 | /** 19 | * Client to access services of a server that has an active master lease. 20 | * 21 | * @author flangner 22 | * @since 04/12/2010 23 | */ 24 | public interface MasterClient extends ConditionClient { 25 | 26 | /** 27 | * Requests a list of serialized {@link LogEntry}s inclusive between the 28 | * given {@link LSN}s start and end at the master. 29 | * 30 | * @param start 31 | * @param end 32 | * @return the {@link ClientResponseFuture} to receive a list of serialized 33 | * LogEntries. 34 | */ 35 | public ClientResponseFuture replica(LSN start, LSN end); 36 | 37 | /** 38 | * Requests the chunk data with the given chunk details at the master. 39 | * 40 | * @param fileName 41 | * @param start 42 | * @param end 43 | * @return the {@link ClientResponseFuture} for receiving Chunk-data. 44 | */ 45 | public ClientResponseFuture chunk( 46 | String fileName, long start, long end); 47 | 48 | /** 49 | * Requests the DBFileMetadata of the master. 50 | * 51 | * @param lsn - of the latest written {@link LogEntry}. 52 | * @return the {@link ClientResponseFuture} receiving a 53 | * {@link DBFileMetaDataSet}. 54 | */ 55 | public ClientResponseFuture load(LSN lsn); 56 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/clients/SlaveClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.clients; 9 | 10 | import org.xtreemfs.babudb.lsmdb.LSN; 11 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.ErrorCodeResponse; 12 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 13 | 14 | /** 15 | * Client to access services of a server that is currently slave for a master. 16 | * 17 | * @author flangner 18 | * @since 04/12/2010 19 | */ 20 | public interface SlaveClient extends ClientInterface { 21 | 22 | /** 23 | * The slave is requested to replicate the given LogEntry identified by its 24 | * {@link org.xtreemfs.babudb.lsmdb.LSN}. The buffer will be freed afterwards. 25 | * 26 | * @param lsn 27 | * @param data 28 | * @return the {@link ClientResponseFuture}. 29 | */ 30 | public ClientResponseFuture replicate(LSN lsn, ReusableBuffer data); 31 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/logic/LogicID.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.logic; 9 | 10 | /** 11 | * @author flangner 12 | * @since 06/08/2009 13 | */ 14 | 15 | -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/operations/LocalTimeOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.operations; 9 | 10 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.Timestamp; 11 | import org.xtreemfs.babudb.pbrpc.ReplicationServiceConstants; 12 | import org.xtreemfs.babudb.replication.transmission.dispatcher.Operation; 13 | import org.xtreemfs.babudb.replication.transmission.dispatcher.Request; 14 | import org.xtreemfs.foundation.TimeSync; 15 | import org.xtreemfs.foundation.logging.Logging; 16 | 17 | import com.google.protobuf.Message; 18 | 19 | /** 20 | * {@link Operation} to answer a local-time request. 21 | * 22 | * @since 03/30/2010 23 | * @author flangner 24 | */ 25 | 26 | public class LocalTimeOperation extends Operation { 27 | 28 | /* 29 | * (non-Javadoc) 30 | * @see org.xtreemfs.babudb.replication.service.operations.Operation# 31 | * getProcedureId() 32 | */ 33 | @Override 34 | public int getProcedureId() { 35 | return ReplicationServiceConstants.PROC_ID_LOCALTIME; 36 | } 37 | 38 | /* (non-Javadoc) 39 | * @see org.xtreemfs.babudb.replication.transmission.dispatcher.Operation#getDefaultRequest() 40 | */ 41 | @Override 42 | public Message getDefaultRequest() { 43 | return null; 44 | } 45 | 46 | /* (non-Javadoc) 47 | * @see org.xtreemfs.babudb.replication.transmission.dispatcher.Operation# 48 | * processRequest(org.xtreemfs.babudb.replication.transmission.dispatcher.Request) 49 | */ 50 | @Override 51 | public void processRequest(Request rq) { 52 | long time = TimeSync.getGlobalTime(); 53 | Logging.logMessage(Logging.LEVEL_DEBUG, this, "LocalTimeOperation:" + 54 | " reporting %d to %s.", time, rq.getSenderAddress().toString()); 55 | 56 | rq.sendSuccess(Timestamp.newBuilder().setValue(time).build()); 57 | } 58 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/service/operations/VolatileStateOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.service.operations; 9 | 10 | import org.xtreemfs.babudb.pbrpc.GlobalTypes.LSN; 11 | import org.xtreemfs.babudb.pbrpc.ReplicationServiceConstants; 12 | import org.xtreemfs.babudb.replication.BabuDBInterface; 13 | import org.xtreemfs.babudb.replication.transmission.dispatcher.Operation; 14 | import org.xtreemfs.babudb.replication.transmission.dispatcher.Request; 15 | import org.xtreemfs.foundation.logging.Logging; 16 | 17 | import com.google.protobuf.Message; 18 | 19 | /** 20 | * {@link Operation} to request the latest LSN at a {@link BabuDB} server. 21 | * 22 | * @since 02/18/2011 23 | * @author flangner 24 | */ 25 | 26 | public class VolatileStateOperation extends Operation { 27 | 28 | private final BabuDBInterface dbInterface; 29 | 30 | public VolatileStateOperation(BabuDBInterface dbInterface) { 31 | this.dbInterface = dbInterface; 32 | } 33 | 34 | /* 35 | * (non-Javadoc) 36 | * @see org.xtreemfs.babudb.replication.service.operations.Operation#getProcedureId() 37 | */ 38 | @Override 39 | public int getProcedureId() { 40 | return ReplicationServiceConstants.PROC_ID_VOLATILESTATE; 41 | } 42 | 43 | /* (non-Javadoc) 44 | * @see org.xtreemfs.babudb.replication.transmission.dispatcher.Operation#getDefaultRequest() 45 | */ 46 | @Override 47 | public Message getDefaultRequest() { 48 | return LSN.getDefaultInstance(); 49 | } 50 | 51 | /* (non-Javadoc) 52 | * @see org.xtreemfs.babudb.replication.transmission.dispatcher.Operation# 53 | * processRequest(org.xtreemfs.babudb.replication.transmission.dispatcher.Request) 54 | */ 55 | @Override 56 | public void processRequest(Request rq) { 57 | 58 | org.xtreemfs.babudb.lsmdb.LSN state = dbInterface.getState(); 59 | Logging.logMessage(Logging.LEVEL_INFO, this, "StateOperation:" + 60 | " reporting %s to %s.", state.toString(), 61 | rq.getSenderAddress().toString()); 62 | 63 | rq.sendSuccess(LSN.newBuilder().setViewId(state.getViewId()) 64 | .setSequenceNo(state.getSequenceNo()).build()); 65 | } 66 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/transmission/TransmissionToServiceInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | /* 9 | * AUTHORS: Felix Langner (ZIB) 10 | */ 11 | package org.xtreemfs.babudb.replication.transmission; 12 | 13 | import org.xtreemfs.babudb.replication.service.ServiceLayer; 14 | import org.xtreemfs.babudb.replication.transmission.client.ClientFactory; 15 | import org.xtreemfs.babudb.replication.transmission.dispatcher.RequestHandler; 16 | 17 | /** 18 | * The interface for the {@link ServiceLayer} to access methods of the 19 | * {@link TransmissionLayer}. 20 | * 21 | * @author flangner 22 | * @since 04/14/2010 23 | */ 24 | public interface TransmissionToServiceInterface extends ClientFactory { 25 | 26 | /** 27 | * @return the {@link FileIOInterface}. 28 | */ 29 | public FileIOInterface getFileIOInterface(); 30 | 31 | /** 32 | * Adds the given request handler to the request dispatcher. Request handler 33 | * process the request logically depending on the semantic of the interface 34 | * they represent. Request handler are identified by the ID of such an 35 | * interface and will replace any already existing one if they have the same 36 | * ID. 37 | * 38 | * @param requestHandler - identified by its interface id. 39 | */ 40 | public void addRequestHandler(RequestHandler requestHandler); 41 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/transmission/client/ClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 - 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.transmission.client; 9 | 10 | import java.net.InetSocketAddress; 11 | 12 | import org.xtreemfs.babudb.pbrpc.ReplicationServiceClient; 13 | import org.xtreemfs.babudb.replication.proxy.DatabaseManagerProxy; 14 | import org.xtreemfs.babudb.replication.proxy.ProxyAccessClient; 15 | 16 | /** 17 | * Interface for {@link ReplicationServiceClient}-generating objects. 18 | * 19 | * @author flangner 20 | * @since 04/13/2010 21 | */ 22 | public interface ClientFactory { 23 | 24 | /** 25 | * @param receiver 26 | * @return the {@link ProxyAccessClientAdapter}, an abstraction from the 27 | * underlying RPC architecture. 28 | */ 29 | public ReplicationClientAdapter getClient(InetSocketAddress receiver); 30 | 31 | /** 32 | * @param dbManProxy 33 | * 34 | * @return a generic proxy-client instance as abstraction from the 35 | * underlying RPC architecture. 36 | */ 37 | public ProxyAccessClient getProxyClient(DatabaseManagerProxy dbManProxy); 38 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/main/java/org/xtreemfs/babudb/replication/transmission/dispatcher/RequestControl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication.transmission.dispatcher; 9 | 10 | /** 11 | * Methods to control they way how requests will be handled on arrival. 12 | * 13 | * @author flangner 14 | * @since 04/05/2011 15 | */ 16 | public interface RequestControl { 17 | 18 | /** 19 | * Method to prevent the handler from processing requests right away. Instead they will 20 | * be buffered and processed when the queue is drained. 21 | */ 22 | public void enableQueuing(); 23 | 24 | /** 25 | * Remove and process any of the requests that are at the queue. Queuing will be disabled 26 | * afterwards. 27 | */ 28 | public void processQueue(); 29 | } 30 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/java/org/xtreemfs/babudb/mock/Mock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.mock; 9 | 10 | /** 11 | * @author flangner 12 | * @since 03/18/2011 13 | */ 14 | public interface Mock { 15 | 16 | public interface MockListener { 17 | 18 | public void eventHappened(String event); 19 | 20 | public void eventFailed(String event); 21 | } 22 | 23 | public void awaitEvent(MockListener listener, String event); 24 | } -------------------------------------------------------------------------------- /java/babudb-replication/src/test/java/org/xtreemfs/babudb/mock/RequestHandlerMock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.mock; 9 | 10 | import java.util.Map; 11 | 12 | import org.xtreemfs.babudb.replication.transmission.dispatcher.Operation; 13 | import org.xtreemfs.babudb.replication.transmission.dispatcher.RequestHandler; 14 | 15 | /** 16 | * Mock for request handler logic. 17 | * 18 | * @author flangner 19 | * @since 02/25/2011 20 | */ 21 | public class RequestHandlerMock extends RequestHandler { 22 | 23 | private final int interfaceID; 24 | 25 | public RequestHandlerMock(int maxQ, int interfaceID, Map ops) { 26 | super(maxQ); 27 | this.interfaceID = interfaceID; 28 | this.operations.putAll(ops); 29 | } 30 | 31 | /* (non-Javadoc) 32 | * @see org.xtreemfs.babudb.replication.transmission.dispatcher.RequestHandler# 33 | * getInterfaceID() 34 | */ 35 | @Override 36 | public int getInterfaceID() { 37 | return interfaceID; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/java/org/xtreemfs/babudb/replication/ReusableBufferUsageTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | import org.junit.Test; 11 | import org.xtreemfs.foundation.buffer.BufferPool; 12 | import org.xtreemfs.foundation.buffer.ReusableBuffer; 13 | 14 | import static junit.framework.Assert.*; 15 | 16 | 17 | /** 18 | * @author flangner 19 | * @since 05/03/2011 20 | */ 21 | public class ReusableBufferUsageTest { 22 | 23 | /** 24 | * @throws Exception 25 | */ 26 | @Test 27 | public void testExtendReusableBuffer() throws Exception { 28 | 29 | byte[] key = "testKey".getBytes(); 30 | byte[] value = "testValue".getBytes(); 31 | 32 | // store data to the buffer 33 | ReusableBuffer testBuffer = BufferPool.allocate(key.length + value.length); 34 | 35 | testBuffer.put(key); 36 | testBuffer.put(value); 37 | testBuffer.flip(); 38 | 39 | // retrieve data from the buffer 40 | byte[] rKey = new byte[key.length]; 41 | byte[] rValue = new byte[value.length]; 42 | 43 | testBuffer.get(rKey, 0, key.length); 44 | assertEquals(new String(key), new String(rKey)); 45 | 46 | testBuffer.get(rValue, 0, value.length); 47 | assertEquals(new String(value), new String (rValue)); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/java/org/xtreemfs/babudb/replication/TestParameters.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, Jan Stender, Bjoern Kolbeck, Mikael Hoegqvist, 3 | * Felix Hupfeld, Felix Langner, Zuse Institute Berlin 4 | * 5 | * Licensed under the BSD License, see LICENSE file for details. 6 | * 7 | */ 8 | package org.xtreemfs.babudb.replication; 9 | 10 | import org.xtreemfs.babudb.config.BabuDBConfig; 11 | import org.xtreemfs.babudb.config.ConfigBuilder; 12 | import org.xtreemfs.babudb.config.ReplicationConfig; 13 | import org.xtreemfs.babudb.log.DiskLogger.SyncMode; 14 | 15 | /** 16 | * Default parameters for the replication TestSuite. 17 | * 18 | * @author flangner 19 | * @since 02/25/2011 20 | */ 21 | public final class TestParameters { 22 | 23 | private TestParameters() { } 24 | 25 | public final static int MAX_PARTICIPANTS = 20; 26 | public final static int MIN_PARTICIPANTS = 1; 27 | public final static int MAX_Q = 100; 28 | public final static int RQ_TIMEOUT = ReplicationConfig.REQUEST_TIMEOUT; 29 | public final static int CON_TIMEOUT = ReplicationConfig.CONNECTION_TIMEOUT; 30 | public final static int TIMESYNC_GLOBAL = 3 * 1000; 31 | public final static int TIMESYNC_LOCAL = 3 * 1000; 32 | 33 | public final static BabuDBConfig conf0 = new ConfigBuilder() 34 | .setLogAppendSyncMode(SyncMode.FSYNC) 35 | .setDataPath("/tmp/babudb0/base", "/tmp/babudb0/log") 36 | .addPlugin(TestParameters.class.getResource("/config/replication_server0.test").getPath()) 37 | .build(); 38 | 39 | public final static BabuDBConfig conf1 = new ConfigBuilder() 40 | .setLogAppendSyncMode(SyncMode.FSYNC) 41 | .setDataPath("/tmp/babudb1/base", "/tmp/babudb1/log") 42 | .addPlugin(TestParameters.class.getResource("/config/replication_server1.test").getPath()) 43 | .build(); 44 | 45 | public final static BabuDBConfig conf2 = new ConfigBuilder() 46 | .setLogAppendSyncMode(SyncMode.FSYNC) 47 | .setDataPath("/tmp/babudb2/base", "/tmp/babudb2/log") 48 | .addPlugin(TestParameters.class.getResource("/config/replication_server2.test").getPath()) 49 | .build(); 50 | } 51 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/resources/config/replication.properties: -------------------------------------------------------------------------------- 1 | ##################################################################### 2 | # BabuDB replication plugin configuration # 3 | ##################################################################### 4 | 5 | plugin.jar = target/babudb-replication-0.6.0.jar 6 | 7 | # DB backup directory - needed for the initial loading of the BabuDB from the 8 | # master in replication context 9 | babudb.repl.backupDir = /tmp/babuDB-backup/ 10 | 11 | # number of servers that at least have to be up to date 12 | babudb.repl.sync.n = 0 13 | 14 | # choose here one of the predefined or a user implemented policy for handling 15 | # database requests: 16 | # 17 | # MasterOnly - will redirect any kind of request to the master. 18 | # WriteRestriction - will only permit lookup operations at the slaves. Any other kind of request is 19 | # redirected to the master. 20 | # NoRestriction - will allow any kind of request to be performed at the local BabuDB instance. 21 | # 22 | # default setting is MasterOnly. 23 | #babudb.repl.policy = MasterOnly 24 | 25 | # it is possible to set the local address and port of this server explicitly. if not it will be 26 | # chosen from the list of participants added right hereafter (default). 27 | #babudb.repl.localhost = localhost 28 | #babudb.repl.localport = 35667 29 | 30 | # participants of the replication including the local address (may be missing, if localhost was 31 | # defined explicitly) 32 | babudb.repl.participant.0 = localhost 33 | babudb.repl.participant.0.port = 35667 34 | babudb.repl.participant.1 = anotherHost.com 35 | babudb.repl.participant.1.port = 35666 36 | 37 | # local time renew in milliseconds 38 | #babudb.localTimeRenew = 3000 39 | 40 | # specify whether SSL is required 41 | #babudb.ssl.enabled = false 42 | 43 | # server credentials for SSL handshakes 44 | #babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12 45 | #babudb.ssl.service_creds.pw = xtreemfs 46 | #babudb.ssl.service_creds.container = pkcs12 47 | 48 | # trusted certificates for SSL handshakes 49 | #babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/xosrootca.jks 50 | #babudb.ssl.trusted_certs.pw = xtreemfs 51 | #babudb.ssl.trusted_certs.container = jks 52 | 53 | #babudb.ssl.authenticationWithoutEncryption = false 54 | 55 | # chunk size, for initial load of file chunks 56 | #babudb.repl.chunkSize = 5242880 57 | 58 | # decides whether redirects should be handled by the user-application or not 59 | #babudb.repl.redirectIsVisible = false 60 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/resources/config/replication_server0.test: -------------------------------------------------------------------------------- 1 | ##################################################################### 2 | # BabuDB replication configuration # 3 | ##################################################################### 4 | 5 | plugin.jar = target/babudb-replication-0.6.0.jar 6 | 7 | # DB backup directory - needed for the initial loading of the BabuDB from the 8 | # master in replication context 9 | babudb.repl.backupDir = /tmp/babuDB-backup0/ 10 | 11 | # number of servers that at least have to be up to date 12 | babudb.repl.sync.n = 3 13 | 14 | # choose here one of the predefined or a user implemented policy for handling 15 | # database requests: 16 | # 17 | # MasterOnly - will redirect any kind of request to the master. 18 | # WriteRestriction - will only permit lookup operations at the slaves. Any other kind of request is 19 | # redirected to the master. 20 | # NoRestriction - will allow any kind of request to be performed at the local BabuDB instance. 21 | # 22 | # default setting is MasterOnly. 23 | babudb.repl.policy = WriteRestriction 24 | 25 | # it is possible to set the local address and port of this server explicitly. if not it will be 26 | # chosen from the list of participants added right hereafter (default). 27 | babudb.repl.localhost = localhost 28 | babudb.repl.localport = 35666 29 | 30 | # participants of the replication including the local address (may be missing, if localhost was 31 | # defined explicitly) 32 | babudb.repl.participant.0 = localhost 33 | babudb.repl.participant.0.port = 35667 34 | babudb.repl.participant.1 = 127.0.0.1 35 | babudb.repl.participant.1.port = 35668 36 | 37 | # local time renew in milliseconds 38 | #babudb.localTimeRenew = 3000 39 | 40 | # specify whether SSL is required 41 | babudb.ssl.enabled = false 42 | 43 | # server credentials for SSL handshakes 44 | #babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12 45 | #babudb.ssl.service_creds.pw = xtreemfs 46 | #babudb.ssl.service_creds.container = pkcs12 47 | 48 | # trusted certificates for SSL handshakes 49 | #babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/xosrootca.jks 50 | #babudb.ssl.trusted_certs.pw = xtreemfs 51 | #babudb.ssl.trusted_certs.container = jks 52 | 53 | #babudb.ssl.authenticationWithoutEncryption = false 54 | 55 | # chunk size, for initial load of file chunks 56 | #babudb.repl.chunkSize = 5242880 57 | 58 | # decides whether redirects should be handled by the user-application or not 59 | #babudb.repl.redirectIsVisible = false 60 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/resources/config/replication_server1.test: -------------------------------------------------------------------------------- 1 | ##################################################################### 2 | # BabuDB replication configuration # 3 | ##################################################################### 4 | 5 | plugin.jar = target/babudb-replication-0.6.0.jar 6 | 7 | # DB backup directory - needed for the initial loading of the BabuDB from the 8 | # master in replication context 9 | babudb.repl.backupDir = /tmp/babuDB-backup1/ 10 | 11 | # number of servers that at least have to be up to date 12 | babudb.repl.sync.n = 3 13 | 14 | # choose here one of the predefined or a user implemented policy for handling 15 | # database requests: 16 | # 17 | # MasterOnly - will redirect any kind of request to the master. 18 | # WriteRestriction - will only permit lookup operations at the slaves. Any other kind of request is 19 | # redirected to the master. 20 | # NoRestriction - will allow any kind of request to be performed at the local BabuDB instance. 21 | # 22 | # default setting is MasterOnly. 23 | babudb.repl.policy = WriteRestriction 24 | 25 | # it is possible to set the local address and port of this server explicitly. if not it will be 26 | # chosen from the list of participants added right hereafter (default). 27 | babudb.repl.localhost = 127.0.0.1 28 | babudb.repl.localport = 35667 29 | 30 | # participants of the replication including the local address (may be missing, if localhost was 31 | # defined explicitly) 32 | babudb.repl.participant.0 = localhost 33 | babudb.repl.participant.0.port = 35666 34 | babudb.repl.participant.1 = 127.0.0.1 35 | babudb.repl.participant.1.port = 35668 36 | 37 | # local time renew in milliseconds 38 | #babudb.localTimeRenew = 3000 39 | 40 | # specify whether SSL is required 41 | babudb.ssl.enabled = false 42 | 43 | # server credentials for SSL handshakes 44 | #babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12 45 | #babudb.ssl.service_creds.pw = xtreemfs 46 | #babudb.ssl.service_creds.container = pkcs12 47 | 48 | # trusted certificates for SSL handshakes 49 | #babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/xosrootca.jks 50 | #babudb.ssl.trusted_certs.pw = xtreemfs 51 | #babudb.ssl.trusted_certs.container = jks 52 | 53 | #babudb.ssl.authenticationWithoutEncryption = false 54 | 55 | # chunk size, for initial load of file chunks 56 | #babudb.repl.chunkSize = 5242880 57 | 58 | # decides whether redirects should be handled by the user-application or not 59 | #babudb.repl.redirectIsVisible = false 60 | -------------------------------------------------------------------------------- /java/babudb-replication/src/test/resources/config/replication_server2.test: -------------------------------------------------------------------------------- 1 | ##################################################################### 2 | # BabuDB replication configuration # 3 | ##################################################################### 4 | 5 | plugin.jar = target/babudb-replication-0.6.0.jar 6 | 7 | # DB backup directory - needed for the initial loading of the BabuDB from the 8 | # master in replication context 9 | babudb.repl.backupDir = /tmp/babuDB-backup2/ 10 | 11 | # number of servers that at least have to be up to date 12 | babudb.repl.sync.n = 3 13 | 14 | # choose here one of the predefined or a user implemented policy for handling 15 | # database requests: 16 | # 17 | # MasterOnly - will redirect any kind of request to the master. 18 | # WriteRestriction - will only permit lookup operations at the slaves. Any other kind of request is 19 | # redirected to the master. 20 | # NoRestriction - will allow any kind of request to be performed at the local BabuDB instance. 21 | # 22 | # default setting is MasterOnly. 23 | babudb.repl.policy = WriteRestriction 24 | 25 | # it is possible to set the local address and port of this server explicitly. if not it will be 26 | # chosen from the list of participants added right hereafter (default). 27 | babudb.repl.localhost = localhost 28 | babudb.repl.localport = 35668 29 | 30 | # participants of the replication including the local address (may be missing, if localhost was 31 | # defined explicitly) 32 | babudb.repl.participant.0 = localhost 33 | babudb.repl.participant.0.port = 35666 34 | babudb.repl.participant.1 = 127.0.0.1 35 | babudb.repl.participant.1.port = 35667 36 | 37 | # local time renew in milliseconds 38 | #babudb.localTimeRenew = 3000 39 | 40 | # specify whether SSL is required 41 | babudb.ssl.enabled = false 42 | 43 | # server credentials for SSL handshakes 44 | #babudb.ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12 45 | #babudb.ssl.service_creds.pw = xtreemfs 46 | #babudb.ssl.service_creds.container = pkcs12 47 | 48 | # trusted certificates for SSL handshakes 49 | #babudb.ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/xosrootca.jks 50 | #babudb.ssl.trusted_certs.pw = xtreemfs 51 | #babudb.ssl.trusted_certs.container = jks 52 | 53 | #babudb.ssl.authenticationWithoutEncryption = false 54 | 55 | # chunk size, for initial load of file chunks 56 | #babudb.repl.chunkSize = 5242880 57 | 58 | # decides whether redirects should be handled by the user-application or not 59 | #babudb.repl.redirectIsVisible = false 60 | --------------------------------------------------------------------------------