├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── LICENSE ├── README.MD ├── bench_tpcc.cpp ├── bench_ycsb.cpp ├── benchmark ├── tpcc │ ├── Context.h │ ├── Database.h │ ├── Query.h │ ├── Random.h │ ├── Schema.h │ ├── Storage.h │ ├── Transaction.h │ └── Workload.h └── ycsb │ ├── Context.h │ ├── Database.h │ ├── Query.h │ ├── Random.h │ ├── Schema.h │ ├── Storage.h │ ├── Transaction.h │ └── Workload.h ├── common ├── BufferedFileWriter.h ├── BufferedReader.h ├── ClassOf.h ├── Encoder.h ├── FastSleep.h ├── FixedString.h ├── FunctionTraits.h ├── Hash.h ├── HashMap.h ├── LockfreeQueue.h ├── MVCCHashMap.h ├── Message.h ├── MessagePiece.h ├── Operation.h ├── Percentile.h ├── Random.h ├── Serialization.h ├── Socket.h ├── SpinLock.h ├── StringPiece.h ├── Time.cpp ├── Time.h └── Zipf.h ├── compile.sh ├── core ├── Context.h ├── ControlMessage.h ├── Coordinator.h ├── Defs.h ├── Delay.h ├── Dispatcher.h ├── Executor.h ├── Macros.h ├── Manager.h ├── Partitioner.h ├── SchemaDef.h ├── Table.h ├── Worker.h └── factory │ └── WorkerFactory.h ├── loc.sh ├── protocol ├── Aria │ ├── Aria.h │ ├── AriaExecutor.h │ ├── AriaHelper.h │ ├── AriaManager.h │ ├── AriaMessage.h │ ├── AriaRWKey.h │ └── AriaTransaction.h ├── AriaFB │ ├── AriaFB.h │ ├── AriaFBExecutor.h │ ├── AriaFBHelper.h │ ├── AriaFBManager.h │ ├── AriaFBMessage.h │ ├── AriaFBRWKey.h │ └── AriaFBTransaction.h ├── Bohm │ ├── Bohm.h │ ├── BohmExecutor.h │ ├── BohmHelper.h │ ├── BohmManager.h │ ├── BohmMessage.h │ ├── BohmPartitioner.h │ ├── BohmRWKey.h │ └── BohmTransaction.h ├── Calvin │ ├── Calvin.h │ ├── CalvinExecutor.h │ ├── CalvinHelper.h │ ├── CalvinManager.h │ ├── CalvinMessage.h │ ├── CalvinPartitioner.h │ ├── CalvinRWKey.h │ └── CalvinTransaction.h ├── Pwv │ ├── PwvExecutor.h │ ├── PwvHelper.h │ ├── PwvManager.h │ ├── PwvRWKey.h │ ├── PwvStatement.h │ ├── PwvTransaction.h │ └── PwvWorkload.h └── TwoPL │ ├── TwoPL.h │ ├── TwoPLExecutor.h │ ├── TwoPLHelper.h │ ├── TwoPLMessage.h │ ├── TwoPLRWKey.h │ └── TwoPLTransaction.h └── scripts ├── aws_barrier.py ├── aws_dist_tpcc.py ├── aws_dist_ycsb.py ├── aws_hstore.py ├── aws_reordering.py ├── aws_reordering_pb.py ├── aws_scalability.py ├── aws_tpcc.py ├── aws_tpcc_pb.py ├── pretty_print.py ├── scp_script.py ├── scp_zip.py └── utility.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all 2 | /* 3 | cmake/* 4 | 5 | 6 | ## Unignore the following extensions 7 | !*.h 8 | !*.cc 9 | !*.cpp 10 | !*.sh 11 | !*.MD 12 | !*.py 13 | 14 | # Unignore the following files 15 | !*CMakeLists.txt 16 | !.travis.yml 17 | !.gitignore 18 | 19 | # Unignore source code directories 20 | !benchmark/ 21 | !cmake 22 | !common/ 23 | !core/ 24 | !database/ 25 | !protocol/ 26 | !scripts/ 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: require 2 | language: generic 3 | compiler: gcc 4 | dist: xenial 5 | 6 | before_install: 7 | # C++14 8 | - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 9 | - sudo apt-get update -qq 10 | 11 | install: 12 | - sudo apt-get install make cmake libjemalloc-dev libboost-dev libgoogle-glog-dev 13 | # C++14 14 | - sudo apt-get install -qq g++-5 15 | # g++ should call g++-5 16 | - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 90 17 | 18 | script: 19 | - ./compile.sh -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.2) 2 | project(aria) 3 | 4 | #set(CMAKE_BUILD_TYPE Debug) 5 | #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wall -Wno-long-long -Wno-unused-variable -Wno-variadic-macros -pedantic") 6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wall -Wno-long-long -Wno-unused-variable -Wno-variadic-macros -pedantic -O2") 7 | set(CMAKE_CXX_STANDARD 14) 8 | 9 | find_library(jemalloc_lib jemalloc) # jemalloc 5.0 10 | 11 | # additional target to perform clang-format run, requires clang-format 12 | 13 | # get all project files 14 | file(GLOB_RECURSE ALL_SOURCE_FILES benchmark/*.h common/*.h core/*.h protocol/*.h bench*.cpp) 15 | 16 | add_custom_target( 17 | format 18 | COMMAND clang-format 19 | -style=LLVM 20 | -i 21 | -sort-includes 22 | ${ALL_SOURCE_FILES} 23 | ) 24 | 25 | include_directories(${CMAKE_SOURCE_DIR}) 26 | 27 | file(GLOB_RECURSE COMMON_SOURCE_FILES common/*.cpp) 28 | add_library(common STATIC ${COMMON_SOURCE_FILES}) 29 | 30 | if(APPLE) 31 | find_package(glog REQUIRED) 32 | find_package(gflags REQUIRED) 33 | target_link_libraries(common ${jemalloc_lib} glog::glog gflags) 34 | else() 35 | target_link_libraries(common ${jemalloc_lib} glog gflags) 36 | endif() 37 | 38 | add_executable(bench_tpcc bench_tpcc.cpp) 39 | target_link_libraries(bench_tpcc common) 40 | 41 | add_executable(bench_ycsb bench_ycsb.cpp) 42 | target_link_libraries(bench_ycsb common) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yi Lu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | **Yi Lu**, Xiangyao Yu, Lei Cao, Samuel Madden 2 | 3 | [Aria: A Fast and Practical Deterministic OLTP Database]() 4 | 5 | *Proc. of the VLDB Endowment (PVLDB), Volume 13, Tokyo, Japan, 2020.* 6 | 7 | [![Build Status](https://travis-ci.org/luyi0619/aria.svg?branch=master)](https://travis-ci.org/luyi0619/aria) 8 | 9 | # Dependencies 10 | 11 | ```sh 12 | sudo apt-get update 13 | sudo apt-get install -y zip make cmake g++ libjemalloc-dev libboost-dev libgoogle-glog-dev 14 | ``` 15 | 16 | # Download 17 | 18 | ```sh 19 | git clone https://github.com/luyi0619/aria.git 20 | ``` 21 | 22 | # Build 23 | 24 | ``` 25 | ./compile.sh 26 | ``` -------------------------------------------------------------------------------- /bench_tpcc.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark/tpcc/Database.h" 2 | #include "core/Coordinator.h" 3 | #include "core/Macros.h" 4 | 5 | DEFINE_bool(operation_replication, false, "use operation replication"); 6 | DEFINE_string(query, "neworder", "tpcc query, mixed, neworder, payment"); 7 | DEFINE_int32(neworder_dist, 10, "new order distributed."); 8 | DEFINE_int32(payment_dist, 15, "payment distributed."); 9 | DEFINE_int32(n_district, 10, "no. of districts in a warehouse"); 10 | DEFINE_bool(write_to_w_ytd, true, "by default, we run standard tpc-c."); 11 | DEFINE_bool(payment_look_up, false, "look up C_ID on secondary index."); 12 | 13 | int main(int argc, char *argv[]) { 14 | 15 | google::InitGoogleLogging(argv[0]); 16 | google::InstallFailureSignalHandler(); 17 | google::ParseCommandLineFlags(&argc, &argv, true); 18 | 19 | aria::tpcc::Context context; 20 | SETUP_CONTEXT(context); 21 | 22 | context.operation_replication = FLAGS_operation_replication; 23 | 24 | if (FLAGS_query == "mixed") { 25 | context.workloadType = aria::tpcc::TPCCWorkloadType::MIXED; 26 | } else if (FLAGS_query == "neworder") { 27 | context.workloadType = aria::tpcc::TPCCWorkloadType::NEW_ORDER_ONLY; 28 | } else if (FLAGS_query == "payment") { 29 | context.workloadType = aria::tpcc::TPCCWorkloadType::PAYMENT_ONLY; 30 | } else { 31 | CHECK(false); 32 | } 33 | 34 | context.newOrderCrossPartitionProbability = FLAGS_neworder_dist; 35 | context.paymentCrossPartitionProbability = FLAGS_payment_dist; 36 | context.n_district = FLAGS_n_district; 37 | context.write_to_w_ytd = FLAGS_write_to_w_ytd; 38 | context.payment_look_up = FLAGS_payment_look_up; 39 | 40 | aria::tpcc::Database db; 41 | db.initialize(context); 42 | 43 | aria::Coordinator c(FLAGS_id, db, context); 44 | c.connectToPeers(); 45 | c.start(); 46 | return 0; 47 | } -------------------------------------------------------------------------------- /bench_ycsb.cpp: -------------------------------------------------------------------------------- 1 | #include "benchmark/ycsb/Database.h" 2 | #include "core/Coordinator.h" 3 | #include "core/Macros.h" 4 | 5 | DEFINE_int32(read_write_ratio, 80, "read write ratio"); 6 | DEFINE_int32(read_only_ratio, 0, "read only transaction ratio"); 7 | DEFINE_int32(cross_ratio, 0, "cross partition transaction ratio"); 8 | DEFINE_int32(keys, 200000, "keys in a partition."); 9 | DEFINE_double(zipf, 0, "skew factor"); 10 | DEFINE_string(skew_pattern, "both", "skew pattern: both, read, write"); 11 | DEFINE_bool(two_partitions, false, "dist transactions access two partitions."); 12 | DEFINE_bool(pwv_ycsb_star, false, "ycsb keys dependency."); 13 | DEFINE_bool(global_key_space, false, "ycsb global key space."); 14 | 15 | int main(int argc, char *argv[]) { 16 | 17 | google::InitGoogleLogging(argv[0]); 18 | google::InstallFailureSignalHandler(); 19 | google::ParseCommandLineFlags(&argc, &argv, true); 20 | 21 | aria::ycsb::Context context; 22 | SETUP_CONTEXT(context); 23 | 24 | if (FLAGS_skew_pattern == "both") { 25 | context.skewPattern = aria::ycsb::YCSBSkewPattern::BOTH; 26 | } else if (FLAGS_skew_pattern == "read") { 27 | context.skewPattern = aria::ycsb::YCSBSkewPattern::READ; 28 | } else if (FLAGS_skew_pattern == "write") { 29 | context.skewPattern = aria::ycsb::YCSBSkewPattern::WRITE; 30 | } else { 31 | CHECK(false); 32 | } 33 | 34 | context.readWriteRatio = FLAGS_read_write_ratio; 35 | context.readOnlyTransaction = FLAGS_read_only_ratio; 36 | context.crossPartitionProbability = FLAGS_cross_ratio; 37 | context.keysPerPartition = FLAGS_keys; 38 | context.two_partitions = FLAGS_two_partitions; 39 | context.pwv_ycsb_star = FLAGS_pwv_ycsb_star; 40 | context.global_key_space = FLAGS_global_key_space; 41 | 42 | if (FLAGS_zipf > 0) { 43 | context.isUniform = false; 44 | if (context.global_key_space) { 45 | aria::Zipf::globalZipf().init( 46 | context.keysPerPartition * context.partition_num, FLAGS_zipf); 47 | } else { 48 | aria::Zipf::globalZipf().init(context.keysPerPartition, FLAGS_zipf); 49 | } 50 | } 51 | 52 | aria::ycsb::Database db; 53 | db.initialize(context); 54 | 55 | aria::Coordinator c(FLAGS_id, db, context); 56 | c.connectToPeers(); 57 | c.start(); 58 | return 0; 59 | } -------------------------------------------------------------------------------- /benchmark/tpcc/Context.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/19/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Context.h" 8 | 9 | namespace aria { 10 | namespace tpcc { 11 | 12 | enum class TPCCWorkloadType { NEW_ORDER_ONLY, PAYMENT_ONLY, MIXED }; 13 | 14 | class Context : public aria::Context { 15 | public: 16 | TPCCWorkloadType workloadType = TPCCWorkloadType::NEW_ORDER_ONLY; 17 | 18 | Context get_single_partition_context() const { 19 | Context c = *this; 20 | c.newOrderCrossPartitionProbability = 0; 21 | c.paymentCrossPartitionProbability = 0; 22 | c.operation_replication = this->operation_replication; 23 | c.star_sync_in_single_master_phase = false; 24 | return c; 25 | } 26 | 27 | Context get_cross_partition_context() const { 28 | Context c = *this; 29 | c.newOrderCrossPartitionProbability = 100; 30 | c.paymentCrossPartitionProbability = 100; 31 | c.operation_replication = false; 32 | c.star_sync_in_single_master_phase = this->star_sync_in_single_master_phase; 33 | return c; 34 | } 35 | 36 | int n_district = 10; 37 | int newOrderCrossPartitionProbability = 10; // out of 100 38 | int paymentCrossPartitionProbability = 15; // out of 100 39 | 40 | bool write_to_w_ytd = true; 41 | bool payment_look_up = false; 42 | }; 43 | } // namespace tpcc 44 | } // namespace aria 45 | -------------------------------------------------------------------------------- /benchmark/tpcc/Random.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "common/Random.h" 11 | 12 | namespace aria { 13 | namespace tpcc { 14 | class Random : public aria::Random { 15 | public: 16 | using aria::Random::Random; 17 | 18 | uint64_t non_uniform_distribution(uint64_t A, uint64_t x, uint64_t y) { 19 | return (uniform_dist(0, A) | uniform_dist(x, y)) % (y - x + 1) + x; 20 | } 21 | 22 | std::string n_string(std::size_t min_len, std::size_t max_len) { 23 | auto len = uniform_dist(min_len, max_len); 24 | return rand_str(len, numeric()); 25 | } 26 | 27 | std::string rand_zip() { 28 | auto zip = n_string(4, 4); 29 | // append "11111" 30 | for (int i = 0; i < 5; i++) { 31 | zip += '1'; 32 | } 33 | return zip; 34 | } 35 | 36 | std::string rand_last_name(int n) { 37 | const auto &last_names = customer_last_names(); 38 | const auto &s1 = last_names[n / 100]; 39 | const auto &s2 = last_names[n / 10 % 10]; 40 | const auto &s3 = last_names[n % 10]; 41 | return s1 + s2 + s3; 42 | } 43 | 44 | private: 45 | static const std::vector &customer_last_names() { 46 | static std::vector last_names = { 47 | "BAR", "OUGHT", "ABLE", "PRI", "PRES", 48 | "ESE", "ANTI", "CALLY", "ATION", "EING"}; 49 | return last_names; 50 | } 51 | 52 | static const std::string &numeric() { 53 | static std::string numeric_ = "0123456789"; 54 | return numeric_; 55 | }; 56 | }; 57 | } // namespace tpcc 58 | } // namespace aria 59 | -------------------------------------------------------------------------------- /benchmark/tpcc/Storage.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/12/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "benchmark/tpcc/Schema.h" 8 | 9 | namespace aria { 10 | namespace tpcc { 11 | struct Storage { 12 | warehouse::key warehouse_key; 13 | warehouse::value warehouse_value; 14 | 15 | district::key district_key; 16 | district::value district_value; 17 | 18 | customer_name_idx::key customer_name_idx_key; 19 | customer_name_idx::value customer_name_idx_value; 20 | 21 | customer::key customer_key; 22 | customer::value customer_value; 23 | 24 | item::key item_keys[15]; 25 | item::value item_values[15]; 26 | 27 | stock::key stock_keys[15]; 28 | stock::value stock_values[15]; 29 | 30 | new_order::key new_order_key; 31 | 32 | order::key order_key; 33 | order::value order_value; 34 | 35 | order_line::key order_line_keys[15]; 36 | order_line::value order_line_values[15]; 37 | 38 | history::key h_key; 39 | history::value h_value; 40 | }; 41 | } // namespace tpcc 42 | } // namespace aria -------------------------------------------------------------------------------- /benchmark/tpcc/Workload.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/24/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "benchmark/tpcc/Context.h" 8 | #include "benchmark/tpcc/Database.h" 9 | #include "benchmark/tpcc/Random.h" 10 | #include "benchmark/tpcc/Storage.h" 11 | #include "benchmark/tpcc/Transaction.h" 12 | #include "core/Partitioner.h" 13 | 14 | namespace aria { 15 | 16 | namespace tpcc { 17 | 18 | template class Workload { 19 | public: 20 | using TransactionType = Transaction; 21 | using DatabaseType = Database; 22 | using ContextType = Context; 23 | using RandomType = Random; 24 | using StorageType = Storage; 25 | 26 | Workload(std::size_t coordinator_id, DatabaseType &db, RandomType &random, 27 | Partitioner &partitioner) 28 | : coordinator_id(coordinator_id), db(db), random(random), 29 | partitioner(partitioner) {} 30 | 31 | std::unique_ptr next_transaction(const ContextType &context, 32 | std::size_t partition_id, 33 | StorageType &storage) { 34 | 35 | int x = random.uniform_dist(1, 100); 36 | std::unique_ptr p; 37 | 38 | if (context.workloadType == TPCCWorkloadType::MIXED) { 39 | if (x <= 50) { 40 | p = std::make_unique>( 41 | coordinator_id, partition_id, db, context, random, partitioner, 42 | storage); 43 | } else { 44 | p = std::make_unique>(coordinator_id, partition_id, 45 | db, context, random, 46 | partitioner, storage); 47 | } 48 | } else if (context.workloadType == TPCCWorkloadType::NEW_ORDER_ONLY) { 49 | p = std::make_unique>(coordinator_id, partition_id, 50 | db, context, random, 51 | partitioner, storage); 52 | } else { 53 | p = std::make_unique>(coordinator_id, partition_id, 54 | db, context, random, 55 | partitioner, storage); 56 | } 57 | 58 | return p; 59 | } 60 | 61 | private: 62 | std::size_t coordinator_id; 63 | DatabaseType &db; 64 | RandomType &random; 65 | Partitioner &partitioner; 66 | }; 67 | 68 | } // namespace tpcc 69 | } // namespace aria 70 | -------------------------------------------------------------------------------- /benchmark/ycsb/Context.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/19/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Context.h" 8 | 9 | #include 10 | 11 | namespace aria { 12 | namespace ycsb { 13 | 14 | enum class PartitionStrategy { RANGE, ROUND_ROBIN }; 15 | 16 | enum class YCSBSkewPattern { BOTH, READ, WRITE }; 17 | 18 | class Context : public aria::Context { 19 | public: 20 | std::size_t getPartitionID(std::size_t key) const { 21 | DCHECK(key >= 0 && key < partition_num * keysPerPartition); 22 | 23 | if (strategy == PartitionStrategy::ROUND_ROBIN) { 24 | return key % partition_num; 25 | } else { 26 | return key / keysPerPartition; 27 | } 28 | } 29 | 30 | std::size_t getGlobalKeyID(std::size_t key, std::size_t partitionID) const { 31 | DCHECK(key >= 0 && key < keysPerPartition && partitionID >= 0 && 32 | partitionID < partition_num); 33 | 34 | if (strategy == PartitionStrategy::ROUND_ROBIN) { 35 | return key * partition_num + partitionID; 36 | } else { 37 | return partitionID * keysPerPartition + key; 38 | } 39 | } 40 | 41 | Context get_single_partition_context() const { 42 | Context c = *this; 43 | c.crossPartitionProbability = 0; 44 | c.operation_replication = this->operation_replication; 45 | c.star_sync_in_single_master_phase = false; 46 | return c; 47 | } 48 | 49 | Context get_cross_partition_context() const { 50 | Context c = *this; 51 | c.crossPartitionProbability = 100; 52 | c.operation_replication = false; 53 | c.star_sync_in_single_master_phase = this->star_sync_in_single_master_phase; 54 | return c; 55 | } 56 | 57 | public: 58 | YCSBSkewPattern skewPattern = YCSBSkewPattern::BOTH; 59 | 60 | int readWriteRatio = 0; // out of 100 61 | int readOnlyTransaction = 0; // out of 100 62 | int crossPartitionProbability = 0; // out of 100 63 | 64 | std::size_t keysPerTransaction = 10; 65 | std::size_t keysPerPartition = 200000; 66 | 67 | bool isUniform = true; 68 | bool two_partitions = false; 69 | bool global_key_space = false; 70 | 71 | PartitionStrategy strategy = PartitionStrategy::ROUND_ROBIN; 72 | }; 73 | } // namespace ycsb 74 | } // namespace aria 75 | -------------------------------------------------------------------------------- /benchmark/ycsb/Random.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "common/Random.h" 11 | 12 | namespace aria { 13 | namespace ycsb { 14 | class Random : public aria::Random { 15 | public: 16 | using aria::Random::Random; 17 | 18 | std::string rand_str(std::size_t length) { 19 | auto &characters_ = characters(); 20 | auto characters_len = characters_.length(); 21 | std::string result; 22 | for (auto i = 0u; i < length; i++) { 23 | int k = uniform_dist(0, characters_len - 1); 24 | result += characters_[k]; 25 | } 26 | return result; 27 | } 28 | 29 | private: 30 | static const std::string &characters() { 31 | static std::string characters_ = 32 | "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; 33 | return characters_; 34 | }; 35 | }; 36 | } // namespace ycsb 37 | } // namespace aria 38 | -------------------------------------------------------------------------------- /benchmark/ycsb/Schema.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/15/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/ClassOf.h" 8 | #include "common/FixedString.h" 9 | #include "common/Hash.h" 10 | #include "common/Serialization.h" 11 | #include "core/SchemaDef.h" 12 | 13 | namespace aria { 14 | namespace ycsb { 15 | static constexpr auto __BASE_COUNTER__ = __COUNTER__ + 1; 16 | static constexpr auto YCSB_FIELD_SIZE = 10; 17 | } // namespace ycsb 18 | } // namespace aria 19 | 20 | #undef NAMESPACE_FIELDS 21 | #define NAMESPACE_FIELDS(x) x(aria) x(ycsb) 22 | 23 | #define YCSB_KEY_FIELDS(x, y) x(int32_t, Y_KEY) 24 | #define YCSB_VALUE_FIELDS(x, y) \ 25 | x(FixedString, Y_F01) \ 26 | y(FixedString, Y_F02) \ 27 | y(FixedString, Y_F03) \ 28 | y(FixedString, Y_F04) \ 29 | y(FixedString, Y_F05) \ 30 | y(FixedString, Y_F06) \ 31 | y(FixedString, Y_F07) \ 32 | y(FixedString, Y_F08) \ 33 | y(FixedString, Y_F09) \ 34 | y(FixedString, Y_F10) 35 | 36 | DO_STRUCT(ycsb, YCSB_KEY_FIELDS, YCSB_VALUE_FIELDS, NAMESPACE_FIELDS) 37 | 38 | namespace aria { 39 | 40 | template <> class Serializer { 41 | public: 42 | std::string operator()(const ycsb::ycsb::value &v) { 43 | return Serializer()(v.Y_F01) + 44 | Serializer()(v.Y_F02) + 45 | Serializer()(v.Y_F03) + 46 | Serializer()(v.Y_F04) + 47 | Serializer()(v.Y_F05) + 48 | Serializer()(v.Y_F06) + 49 | Serializer()(v.Y_F07) + 50 | Serializer()(v.Y_F08) + 51 | Serializer()(v.Y_F09) + 52 | Serializer()(v.Y_F10); 53 | } 54 | }; 55 | 56 | template <> class Deserializer { 57 | public: 58 | std::size_t operator()(StringPiece str, ycsb::ycsb::value &result) const { 59 | 60 | std::size_t sz = Deserializer()(str, result.Y_F01); 61 | str.remove_prefix(sz); 62 | Deserializer()(str, result.Y_F02); 63 | str.remove_prefix(sz); 64 | Deserializer()(str, result.Y_F03); 65 | str.remove_prefix(sz); 66 | Deserializer()(str, result.Y_F04); 67 | str.remove_prefix(sz); 68 | Deserializer()(str, result.Y_F05); 69 | str.remove_prefix(sz); 70 | Deserializer()(str, result.Y_F06); 71 | str.remove_prefix(sz); 72 | Deserializer()(str, result.Y_F07); 73 | str.remove_prefix(sz); 74 | Deserializer()(str, result.Y_F08); 75 | str.remove_prefix(sz); 76 | Deserializer()(str, result.Y_F09); 77 | str.remove_prefix(sz); 78 | Deserializer()(str, result.Y_F10); 79 | str.remove_prefix(sz); 80 | return sz * 10; 81 | } 82 | }; 83 | 84 | template <> class ClassOf { 85 | public: 86 | static constexpr std::size_t size() { 87 | return ClassOf::size() * 10; 88 | } 89 | }; 90 | 91 | } // namespace aria -------------------------------------------------------------------------------- /benchmark/ycsb/Storage.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/12/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "benchmark/ycsb/Schema.h" 8 | 9 | namespace aria { 10 | 11 | namespace ycsb { 12 | struct Storage { 13 | ycsb::key ycsb_keys[YCSB_FIELD_SIZE]; 14 | ycsb::value ycsb_values[YCSB_FIELD_SIZE]; 15 | }; 16 | 17 | } // namespace ycsb 18 | } // namespace aria -------------------------------------------------------------------------------- /benchmark/ycsb/Transaction.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/22/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "glog/logging.h" 8 | 9 | #include "benchmark/ycsb/Database.h" 10 | #include "benchmark/ycsb/Query.h" 11 | #include "benchmark/ycsb/Schema.h" 12 | #include "benchmark/ycsb/Storage.h" 13 | #include "common/Operation.h" 14 | #include "core/Defs.h" 15 | #include "core/Partitioner.h" 16 | #include "core/Table.h" 17 | 18 | namespace aria { 19 | namespace ycsb { 20 | 21 | template class ReadModifyWrite : public Transaction { 22 | 23 | public: 24 | using DatabaseType = Database; 25 | using ContextType = typename DatabaseType::ContextType; 26 | using RandomType = typename DatabaseType::RandomType; 27 | using StorageType = Storage; 28 | 29 | static constexpr std::size_t keys_num = 10; 30 | 31 | ReadModifyWrite(std::size_t coordinator_id, std::size_t partition_id, 32 | DatabaseType &db, const ContextType &context, 33 | RandomType &random, Partitioner &partitioner, 34 | Storage &storage) 35 | : Transaction(coordinator_id, partition_id, partitioner), db(db), 36 | context(context), random(random), storage(storage), 37 | partition_id(partition_id), 38 | query(makeYCSBQuery()(context, partition_id, random)) {} 39 | 40 | virtual ~ReadModifyWrite() override = default; 41 | 42 | TransactionResult execute(std::size_t worker_id) override { 43 | 44 | DCHECK(context.keysPerTransaction == keys_num); 45 | 46 | int ycsbTableID = ycsb::tableID; 47 | 48 | for (auto i = 0u; i < keys_num; i++) { 49 | auto key = query.Y_KEY[i]; 50 | storage.ycsb_keys[i].Y_KEY = key; 51 | if (query.UPDATE[i]) { 52 | this->search_for_update(ycsbTableID, context.getPartitionID(key), 53 | storage.ycsb_keys[i], storage.ycsb_values[i]); 54 | } else { 55 | this->search_for_read(ycsbTableID, context.getPartitionID(key), 56 | storage.ycsb_keys[i], storage.ycsb_values[i]); 57 | } 58 | } 59 | 60 | if (this->process_requests(worker_id)) { 61 | return TransactionResult::ABORT; 62 | } 63 | 64 | for (auto i = 0u; i < keys_num; i++) { 65 | auto key = query.Y_KEY[i]; 66 | if (query.UPDATE[i]) { 67 | 68 | if (this->execution_phase) { 69 | RandomType local_random; 70 | storage.ycsb_values[i].Y_F01.assign( 71 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 72 | storage.ycsb_values[i].Y_F02.assign( 73 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 74 | storage.ycsb_values[i].Y_F03.assign( 75 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 76 | storage.ycsb_values[i].Y_F04.assign( 77 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 78 | storage.ycsb_values[i].Y_F05.assign( 79 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 80 | storage.ycsb_values[i].Y_F06.assign( 81 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 82 | storage.ycsb_values[i].Y_F07.assign( 83 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 84 | storage.ycsb_values[i].Y_F08.assign( 85 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 86 | storage.ycsb_values[i].Y_F09.assign( 87 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 88 | storage.ycsb_values[i].Y_F10.assign( 89 | local_random.a_string(YCSB_FIELD_SIZE, YCSB_FIELD_SIZE)); 90 | } 91 | 92 | this->update(ycsbTableID, context.getPartitionID(key), 93 | storage.ycsb_keys[i], storage.ycsb_values[i]); 94 | } 95 | } 96 | 97 | return TransactionResult::READY_TO_COMMIT; 98 | } 99 | 100 | void reset_query() override { 101 | query = makeYCSBQuery()(context, partition_id, random); 102 | } 103 | 104 | private: 105 | DatabaseType &db; 106 | const ContextType &context; 107 | RandomType &random; 108 | Storage &storage; 109 | std::size_t partition_id; 110 | YCSBQuery query; 111 | }; 112 | } // namespace ycsb 113 | 114 | } // namespace aria 115 | -------------------------------------------------------------------------------- /benchmark/ycsb/Workload.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/25/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "benchmark/ycsb/Context.h" 8 | #include "benchmark/ycsb/Database.h" 9 | #include "benchmark/ycsb/Random.h" 10 | #include "benchmark/ycsb/Storage.h" 11 | #include "benchmark/ycsb/Transaction.h" 12 | #include "core/Partitioner.h" 13 | 14 | namespace aria { 15 | 16 | namespace ycsb { 17 | 18 | template class Workload { 19 | public: 20 | using TransactionType = Transaction; 21 | using DatabaseType = Database; 22 | using ContextType = Context; 23 | using RandomType = Random; 24 | using StorageType = Storage; 25 | 26 | Workload(std::size_t coordinator_id, DatabaseType &db, RandomType &random, 27 | Partitioner &partitioner) 28 | : coordinator_id(coordinator_id), db(db), random(random), 29 | partitioner(partitioner) {} 30 | 31 | std::unique_ptr next_transaction(const ContextType &context, 32 | std::size_t partition_id, 33 | StorageType &storage) { 34 | 35 | std::unique_ptr p = 36 | std::make_unique>( 37 | coordinator_id, partition_id, db, context, random, partitioner, 38 | storage); 39 | 40 | return p; 41 | } 42 | 43 | private: 44 | std::size_t coordinator_id; 45 | DatabaseType &db; 46 | RandomType &random; 47 | Partitioner &partitioner; 48 | }; 49 | 50 | } // namespace ycsb 51 | } // namespace aria 52 | -------------------------------------------------------------------------------- /common/BufferedFileWriter.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 3/21/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | class BufferedFileWriter { 14 | 15 | public: 16 | BufferedFileWriter(const char *filename) { 17 | fd = open(filename, O_WRONLY | O_CREAT, 18 | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); 19 | CHECK(fd >= 0); 20 | bytes_total = 0; 21 | } 22 | 23 | void write(const char *str, long size) { 24 | 25 | if (bytes_total + size < BUFFER_SIZE) { 26 | memcpy(buffer + bytes_total, str, size); 27 | bytes_total += size; 28 | return; 29 | } 30 | 31 | auto copy_size = BUFFER_SIZE - bytes_total; 32 | 33 | memcpy(buffer + bytes_total, str, copy_size); 34 | bytes_total += copy_size; 35 | flush(); 36 | 37 | str += copy_size; 38 | size -= copy_size; 39 | 40 | if (size >= BUFFER_SIZE) { 41 | int err = ::write(fd, str, size); 42 | CHECK(err >= 0); 43 | bytes_total = 0; 44 | } else { 45 | memcpy(buffer, str, size); 46 | bytes_total += size; 47 | } 48 | } 49 | 50 | void flush() { 51 | DCHECK(fd >= 0); 52 | if (bytes_total > 0) { 53 | int err = ::write(fd, buffer, bytes_total); 54 | CHECK(err >= 0); 55 | } 56 | bytes_total = 0; 57 | } 58 | 59 | void close() { 60 | flush(); 61 | int err = ::close(fd); 62 | CHECK(err == 0); 63 | } 64 | 65 | public: 66 | static constexpr uint32_t BUFFER_SIZE = 1024 * 1024 * 4; // 4MB 67 | 68 | private: 69 | int fd; 70 | char buffer[BUFFER_SIZE]; 71 | std::size_t bytes_total; 72 | }; -------------------------------------------------------------------------------- /common/BufferedReader.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/30/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Message.h" 8 | #include "common/Socket.h" 9 | 10 | #include 11 | 12 | namespace aria { 13 | class BufferedReader { 14 | public: 15 | BufferedReader(Socket &socket) 16 | : socket(&socket), bytes_read(0), bytes_total(0) {} 17 | 18 | // BufferedReader is not copyable 19 | BufferedReader(const BufferedReader &) = delete; 20 | 21 | BufferedReader &operator=(const BufferedReader &) = delete; 22 | 23 | // BufferedReader is movable 24 | 25 | BufferedReader(BufferedReader &&that) 26 | : socket(that.socket), bytes_read(that.bytes_read), 27 | bytes_total(that.bytes_total) { 28 | that.socket = nullptr; 29 | that.bytes_read = 0; 30 | that.bytes_total = 0; 31 | } 32 | 33 | BufferedReader &operator=(BufferedReader &&that) { 34 | socket = that.socket; 35 | bytes_read = that.bytes_read; 36 | bytes_total = that.bytes_total; 37 | 38 | that.socket = nullptr; 39 | that.bytes_read = 0; 40 | that.bytes_total = 0; 41 | return *this; 42 | } 43 | 44 | std::unique_ptr next_message() { 45 | DCHECK(socket != nullptr); 46 | 47 | fetch_message(); 48 | if (!has_message()) { 49 | return nullptr; 50 | } 51 | 52 | // read header and deadbeef; 53 | auto header = 54 | *reinterpret_cast(buffer + bytes_read); 55 | auto deadbeef = *reinterpret_cast( 56 | buffer + bytes_read + sizeof(header)); 57 | 58 | // check deadbeaf 59 | DCHECK(deadbeef == Message::DEADBEEF); 60 | auto message = std::make_unique(); 61 | auto length = Message::get_message_length(header); 62 | message->resize(length); 63 | 64 | // copy the data 65 | DCHECK(bytes_read + length <= bytes_total); 66 | std::memcpy(message->get_raw_ptr(), buffer + bytes_read, length); 67 | bytes_read += length; 68 | DCHECK(bytes_read <= bytes_total); 69 | 70 | return message; 71 | } 72 | 73 | private: 74 | void fetch_message() { 75 | DCHECK(socket != nullptr); 76 | 77 | // return if there is a message left 78 | if (has_message()) { 79 | return; 80 | } 81 | 82 | // copy left bytes 83 | DCHECK(bytes_read <= bytes_total); 84 | auto bytes_left = bytes_total - bytes_read; 85 | bytes_total = 0; 86 | 87 | if (bytes_left > 0 && bytes_read > 0) { 88 | 89 | if (bytes_left <= bytes_read) { // non overlapping 90 | std::memcpy(buffer, buffer + bytes_read, bytes_left); 91 | } else { 92 | for (auto i = 0u; i < bytes_left; i++) { 93 | buffer[i] = buffer[i + bytes_read]; 94 | } 95 | } 96 | } 97 | bytes_total += bytes_left; 98 | bytes_read = 0; 99 | 100 | // read new message 101 | 102 | auto bytes_received = 103 | socket->read_async(buffer + bytes_total, BUFFER_SIZE - bytes_total); 104 | 105 | if (bytes_received > 0) { 106 | // successful read 107 | bytes_total += bytes_received; 108 | } 109 | } 110 | 111 | bool has_message() { 112 | // check if the buffer has a message header 113 | if (bytes_read + Message::get_prefix_size() > bytes_total) { 114 | return false; 115 | } 116 | 117 | // read header and deadbeef; 118 | auto header = 119 | *reinterpret_cast(buffer + bytes_read); 120 | auto deadbeef = *reinterpret_cast( 121 | buffer + bytes_read + sizeof(header)); 122 | 123 | // check deadbeaf 124 | DCHECK(deadbeef == Message::DEADBEEF); 125 | 126 | // check if the buffer has a message 127 | return bytes_read + Message::get_message_length(header) <= bytes_total; 128 | } 129 | 130 | public: 131 | static constexpr uint32_t BUFFER_SIZE = 1024 * 1024 * 4; // 4MB 132 | 133 | private: 134 | Socket *socket; 135 | char buffer[BUFFER_SIZE]; 136 | std::size_t bytes_read, bytes_total; 137 | }; 138 | } // namespace aria 139 | -------------------------------------------------------------------------------- /common/ClassOf.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/5/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | template class ClassOf { 11 | public: 12 | static constexpr std::size_t size() { return sizeof(T); } 13 | }; 14 | } // namespace aria -------------------------------------------------------------------------------- /common/Encoder.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/17/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "Serialization.h" 11 | #include "StringPiece.h" 12 | 13 | namespace aria { 14 | class Encoder { 15 | public: 16 | Encoder(std::string &bytes) : bytes(bytes) {} 17 | 18 | template friend Encoder &operator<<(Encoder &enc, const T &rhs); 19 | 20 | StringPiece toStringPiece() { 21 | return StringPiece(bytes.data(), bytes.size()); 22 | } 23 | 24 | void write_n_bytes(const void *ptr, std::size_t size) { 25 | bytes.append(static_cast(ptr), size); 26 | } 27 | 28 | std::size_t size() { return bytes.size(); } 29 | 30 | private: 31 | std::string &bytes; 32 | }; 33 | 34 | template Encoder &operator<<(Encoder &enc, const T &rhs) { 35 | Serializer serializer; 36 | enc.bytes += serializer(rhs); 37 | return enc; 38 | } 39 | 40 | class Decoder { 41 | public: 42 | Decoder(StringPiece bytes) : bytes(bytes) {} 43 | 44 | template friend Decoder &operator>>(Decoder &dec, T &rhs); 45 | 46 | void read_n_bytes(void *ptr, std::size_t size) { 47 | DCHECK(bytes.size() >= size); 48 | std::memcpy(ptr, bytes.data(), size); 49 | bytes.remove_prefix(size); 50 | } 51 | 52 | std::size_t size() { return bytes.size(); } 53 | 54 | private: 55 | StringPiece bytes; 56 | }; 57 | 58 | template Decoder &operator>>(Decoder &dec, T &rhs) { 59 | Deserializer deserializer; 60 | std::size_t size = deserializer(dec.bytes, rhs); 61 | dec.bytes.remove_prefix(size); 62 | return dec; 63 | } 64 | } // namespace aria 65 | -------------------------------------------------------------------------------- /common/FastSleep.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | class FastSleep { 11 | 12 | public: 13 | // in microseconds 14 | static int64_t sleep_for(int64_t length) { 15 | 16 | std::chrono::steady_clock::time_point start, end; 17 | start = std::chrono::steady_clock::now(); 18 | std::chrono::steady_clock::duration d; 19 | 20 | // use microseconds for accuracy. 21 | int64_t nano_length = length * 1000; 22 | 23 | do { 24 | nop(); 25 | end = std::chrono::steady_clock::now(); 26 | d = std::chrono::duration_cast(end - start); 27 | } while (d.count() < nano_length); 28 | 29 | return d.count() / 1000; 30 | } 31 | 32 | private: 33 | static void nop() { asm("nop"); } 34 | }; 35 | -------------------------------------------------------------------------------- /common/FixedString.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/13/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "ClassOf.h" 12 | #include "Hash.h" 13 | #include "Serialization.h" 14 | #include "StringPiece.h" 15 | 16 | namespace aria { 17 | 18 | template class FixedString { 19 | public: 20 | static_assert(N > 0, "string length should be positive."); 21 | 22 | using size_type = std::size_t; 23 | 24 | FixedString() { assign(""); } 25 | 26 | FixedString(const char *str) { assign(std::string(str)); } 27 | 28 | FixedString(const std::string &str) { assign(str); } 29 | 30 | int compare(const FixedString &that) const { 31 | 32 | for (auto i = 0u; i < N; i++) { 33 | if (data_[i] < that.data_[i]) { 34 | return -1; 35 | } 36 | 37 | if (data_[i] > that.data_[i]) { 38 | return 1; 39 | } 40 | } 41 | return 0; 42 | } 43 | 44 | bool operator<(const FixedString &that) const { return compare(that) < 0; } 45 | 46 | bool operator<=(const FixedString &that) const { return compare(that) <= 0; } 47 | 48 | bool operator>(const FixedString &that) const { return compare(that) > 0; } 49 | 50 | bool operator>=(const FixedString &that) const { return compare(that) >= 0; } 51 | 52 | bool operator==(const FixedString &that) const { return compare(that) == 0; } 53 | 54 | bool operator!=(const FixedString &that) const { return compare(that) != 0; } 55 | 56 | FixedString &assign(const std::string &str) { 57 | return assign(str, str.length()); 58 | } 59 | 60 | FixedString &assign(const std::string &str, size_type length) { 61 | DCHECK(length <= str.length()); 62 | DCHECK(length <= N); 63 | std::copy(str.begin(), str.begin() + length, data_.begin()); 64 | DCHECK(data_.begin() + length <= data_.end() - 1); 65 | std::fill(data_.begin() + length, data_.end() - 1, ' '); 66 | data_[N] = 0; 67 | return *this; 68 | } 69 | 70 | const char *c_str() { return &data_[0]; } 71 | 72 | std::size_t hash_code() const { 73 | std::hash h; 74 | std::size_t hashCode = 0; 75 | for (auto i = 0u; i < N; i++) { 76 | hashCode = aria::hash_combine(hashCode, h(data_[i])); 77 | } 78 | return hashCode; 79 | } 80 | 81 | constexpr size_type length() const { return N; } 82 | 83 | constexpr size_type size() const { return N; } 84 | 85 | std::string toString() const { 86 | std::string str; 87 | // the last char is \0 88 | std::copy(data_.begin(), data_.end() - 1, std::back_inserter(str)); 89 | DCHECK(str.length() == N); 90 | return str; 91 | } 92 | 93 | private: 94 | std::array data_; 95 | }; 96 | 97 | template 98 | inline std::basic_ostream &operator<<(std::basic_ostream &os, 99 | const FixedString &str) { 100 | os << str.toString(); 101 | return os; 102 | } 103 | 104 | template class Serializer> { 105 | public: 106 | std::string operator()(const FixedString &v) { return v.toString(); } 107 | }; 108 | 109 | template class Deserializer> { 110 | public: 111 | std::size_t operator()(StringPiece str, FixedString &result) const { 112 | result.assign(str.data(), N); 113 | return N; 114 | } 115 | }; 116 | 117 | template class ClassOf> { 118 | public: 119 | static constexpr std::size_t size() { return N; } 120 | }; 121 | 122 | } // namespace aria 123 | 124 | namespace std { 125 | template struct hash> { 126 | std::size_t operator()(const aria::FixedString &k) const { 127 | return k.hash_code(); 128 | } 129 | }; 130 | } // namespace std 131 | -------------------------------------------------------------------------------- /common/FunctionTraits.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | template 11 | struct FunctionTraits : public FunctionTraits {}; 12 | 13 | template 14 | struct FunctionTraits { 15 | enum { arity = sizeof...(Args) }; 16 | 17 | typedef ReturnType return_type; 18 | 19 | template struct arg { 20 | typedef typename std::tuple_element>::type type; 21 | }; 22 | }; 23 | 24 | template 25 | using ReturnType = typename FunctionTraits::return_type; 26 | 27 | template 28 | using Argument0 = typename FunctionTraits::template arg<0>::type; 29 | 30 | template 31 | using Argument1 = typename FunctionTraits::template arg<1>::type; 32 | 33 | } // namespace aria 34 | -------------------------------------------------------------------------------- /common/Hash.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/13/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | template 12 | inline std::size_t hash_combine(const T &v1, const T &v2) { 13 | return v2 ^ (v1 + 0x9e3779b9 + (v2 << 6) + (v2 >> 2)); 14 | } 15 | 16 | template inline std::size_t hash(const T &v) { 17 | return std::hash()(v); 18 | } 19 | 20 | template 21 | inline std::size_t hash(const T &v, Rest... rest) { 22 | std::hash h; 23 | return hash_combine(h(v), hash(rest...)); 24 | } 25 | 26 | } // namespace aria 27 | -------------------------------------------------------------------------------- /common/HashMap.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "SpinLock.h" 8 | #include 9 | #include 10 | #include 11 | 12 | namespace aria { 13 | 14 | template class HashMap { 15 | 16 | public: 17 | using HashMapType = std::unordered_map; 18 | using HasherType = typename HashMapType::hasher; 19 | 20 | public: 21 | bool remove(const KeyType &key) { 22 | return apply( 23 | [&key](HashMapType &map) { 24 | auto it = map.find(key); 25 | if (it == map.end()) { 26 | return false; 27 | } else { 28 | map.erase(it); 29 | return true; 30 | } 31 | }, 32 | bucket_number(key)); 33 | } 34 | 35 | bool contains(const KeyType &key) { 36 | return apply( 37 | [&key](const HashMapType &map) { return map.find(key) != map.end(); }, 38 | bucket_number(key)); 39 | } 40 | 41 | bool insert(const KeyType &key, const ValueType &value) { 42 | return apply( 43 | [&key, &value](HashMapType &map) { 44 | if (map.find(key) != map.end()) { 45 | return false; 46 | } 47 | map[key] = value; 48 | return true; 49 | }, 50 | bucket_number(key)); 51 | } 52 | 53 | ValueType &operator[](const KeyType &key) { 54 | return apply_ref( 55 | [&key](HashMapType &map) -> ValueType & { return map[key]; }, 56 | bucket_number(key)); 57 | } 58 | 59 | std::size_t size() { 60 | return fold(0, [](std::size_t totalSize, const HashMapType &map) { 61 | return totalSize + map.size(); 62 | }); 63 | } 64 | 65 | void clear() { 66 | map([](HashMapType &map) { map.clear(); }); 67 | } 68 | 69 | private: 70 | template 71 | auto &apply_ref(ApplyFunc applyFunc, std::size_t i) { 72 | DCHECK(i < N) << "index " << i << " is greater than " << N; 73 | locks[i].lock(); 74 | auto &result = applyFunc(maps[i]); 75 | locks[i].unlock(); 76 | return result; 77 | } 78 | 79 | template auto apply(ApplyFunc applyFunc, std::size_t i) { 80 | DCHECK(i < N) << "index " << i << " is greater than " << N; 81 | locks[i].lock(); 82 | auto result = applyFunc(maps[i]); 83 | locks[i].unlock(); 84 | return result; 85 | } 86 | 87 | template void map(MapFunc mapFunc) { 88 | for (auto i = 0u; i < N; i++) { 89 | locks[i].lock(); 90 | mapFunc(maps[i]); 91 | locks[i].unlock(); 92 | } 93 | } 94 | 95 | template 96 | auto fold(const T &firstValue, FoldFunc foldFunc) { 97 | T finalValue = firstValue; 98 | for (auto i = 0u; i < N; i++) { 99 | locks[i].lock(); 100 | finalValue = foldFunc(finalValue, maps[i]); 101 | locks[i].unlock(); 102 | } 103 | return finalValue; 104 | } 105 | 106 | auto bucket_number(const KeyType &key) { return hasher(key) % N; } 107 | 108 | private: 109 | HasherType hasher; 110 | HashMapType maps[N]; 111 | SpinLock locks[N]; 112 | }; 113 | 114 | } // namespace aria 115 | -------------------------------------------------------------------------------- /common/LockfreeQueue.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/29/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "glog/logging.h" 8 | #include 9 | 10 | namespace aria { 11 | 12 | /* 13 | * boost::lockfree::spsc_queue does not support move only objects, e.g., 14 | * std::unique_ptr. As a result, only Message* can be pushed into 15 | * MessageQueue. Usage: std::unique_ptr ptr; MessageQueue q; 16 | * q.push(ptr.release()); 17 | * 18 | * std::unique_ptr ptr1(q.front()); 19 | * q.pop(); 20 | * 21 | */ 22 | 23 | template 24 | class LockfreeQueue 25 | : public boost::lockfree::spsc_queue> { 26 | public: 27 | using element_type = T; 28 | using base_type = 29 | boost::lockfree::spsc_queue>; 30 | 31 | void push(const T &value) { 32 | while (base_type::write_available() == 0) { 33 | nop_pause(); 34 | } 35 | bool ok = base_type::push(value); 36 | CHECK(ok); 37 | } 38 | 39 | void wait_till_non_empty() { 40 | while (base_type::empty()) { 41 | nop_pause(); 42 | } 43 | } 44 | 45 | auto capacity() { return N; } 46 | 47 | private: 48 | void nop_pause() { __asm volatile("pause" : :); } 49 | }; 50 | } // namespace aria 51 | -------------------------------------------------------------------------------- /common/MessagePiece.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/30/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/StringPiece.h" 8 | 9 | namespace aria { 10 | 11 | /* 12 | * MessagePiece header format 13 | * 14 | * | Message type (12 => 4048) | Message length (31 => 2147483648) | table id (5 15 | * => 32) | partition id (16 => 65536) | 16 | * 17 | * Note that, the header is included in the message length. 18 | */ 19 | 20 | class MessagePiece { 21 | 22 | public: 23 | using header_type = uint64_t; 24 | 25 | MessagePiece(const MessagePiece &messagePiece) 26 | : stringPiece(messagePiece.stringPiece) {} 27 | 28 | MessagePiece(const StringPiece &stringPiece) : stringPiece(stringPiece) {} 29 | 30 | uint64_t get_message_type() const { 31 | return (get_header() >> MESSAGE_TYPE_OFFSET) & MESSAGE_TYPE_MASK; 32 | } 33 | 34 | uint64_t get_message_length() const { 35 | return (get_header() >> MESSAGE_LENGTH_OFFSET) & MESSAGE_LENGTH_MASK; 36 | } 37 | 38 | uint64_t get_table_id() const { 39 | return (get_header() >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 40 | } 41 | 42 | uint64_t get_partition_id() const { 43 | return (get_header() >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 44 | } 45 | 46 | StringPiece toStringPiece() { 47 | return StringPiece(stringPiece.data() + get_header_size(), 48 | get_message_length() - get_header_size()); 49 | } 50 | 51 | bool operator==(const MessagePiece &that) const { 52 | return stringPiece == that.stringPiece; 53 | } 54 | 55 | bool operator!=(const MessagePiece &that) const { 56 | return stringPiece != that.stringPiece; 57 | } 58 | 59 | private: 60 | uint64_t get_header() const { 61 | return *reinterpret_cast(stringPiece.data()); 62 | } 63 | 64 | public: 65 | StringPiece stringPiece; 66 | 67 | public: 68 | static uint64_t get_header_size() { return sizeof(header_type); } 69 | 70 | static uint64_t construct_message_piece_header(uint64_t message_type, 71 | uint64_t message_length, 72 | std::size_t table_id, 73 | std::size_t partition_id) { 74 | DCHECK(message_type < (1ul << 12)); 75 | DCHECK(message_length < (1ul << 31)); 76 | DCHECK(table_id < (1ul << 5)); 77 | DCHECK(partition_id < (1ul << 16)); 78 | 79 | return (message_type << MESSAGE_TYPE_OFFSET) + 80 | (message_length << MESSAGE_LENGTH_OFFSET) + 81 | (table_id << TABLE_ID_OFFSET) + 82 | (partition_id << PARTITION_ID_OFFSET); 83 | } 84 | 85 | static constexpr uint64_t get_message_length(uint64_t header) { 86 | return (header >> MESSAGE_LENGTH_OFFSET) & MESSAGE_LENGTH_MASK; 87 | } 88 | 89 | public: 90 | static constexpr uint64_t MESSAGE_TYPE_MASK = 0xfff; 91 | static constexpr uint64_t MESSAGE_TYPE_OFFSET = 52; 92 | static constexpr uint64_t MESSAGE_LENGTH_MASK = 0x7fffffff; 93 | static constexpr uint64_t MESSAGE_LENGTH_OFFSET = 21; 94 | static constexpr uint64_t TABLE_ID_MASK = 0x1f; 95 | static constexpr uint64_t TABLE_ID_OFFSET = 16; 96 | static constexpr uint64_t PARTITION_ID_MASK = 0xffff; 97 | static constexpr uint64_t PARTITION_ID_OFFSET = 0; 98 | }; 99 | } // namespace aria 100 | -------------------------------------------------------------------------------- /common/Operation.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/17/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Encoder.h" 8 | #include 9 | 10 | namespace aria { 11 | 12 | class Operation { 13 | 14 | public: 15 | Operation() : tid(0), partition_id(0) {} 16 | 17 | void clear() { 18 | tid = 0; 19 | partition_id = 0; 20 | data.clear(); 21 | } 22 | 23 | void set_tid(uint64_t id) { tid = id; } 24 | 25 | uint64_t get_tid() const { return tid; } 26 | 27 | public: 28 | uint64_t tid; 29 | std::size_t partition_id; 30 | std::string data; 31 | }; 32 | } // namespace aria -------------------------------------------------------------------------------- /common/Percentile.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/29/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | // The nearest-rank method 13 | // https://en.wikipedia.org/wiki/Percentile 14 | 15 | namespace aria { 16 | 17 | template class Percentile { 18 | public: 19 | using element_type = T; 20 | 21 | void add(const element_type &value) { 22 | isSorted_ = false; 23 | data_.push_back(value); 24 | } 25 | 26 | void add(const std::vector &v) { 27 | isSorted_ = false; 28 | std::copy(v.begin(), v.end(), std::back_inserter(data_)); 29 | } 30 | 31 | void clear() { 32 | isSorted_ = true; 33 | data_.clear(); 34 | } 35 | 36 | auto size() { return data_.size(); } 37 | 38 | element_type nth(double n) { 39 | if (data_.size() == 0) { 40 | return 0; 41 | } 42 | checkSort(); 43 | DCHECK(n > 0 && n <= 100); 44 | auto sz = size(); 45 | auto i = static_cast(ceil(n / 100 * sz)) - 1; 46 | DCHECK(i >= 0 && i < size()); 47 | return data_[i]; 48 | } 49 | 50 | void save_cdf(const std::string &path) { 51 | if (data_.size() == 0) { 52 | return; 53 | } 54 | checkSort(); 55 | 56 | if (path.empty()) { 57 | return; 58 | } 59 | 60 | std::ofstream cdf; 61 | cdf.open(path); 62 | 63 | cdf << "value\tcdf" << std::endl; 64 | 65 | // output ~ 1k rows 66 | auto step_size = std::max(1, int(data_.size() * 0.99 / 1000)); 67 | 68 | std::vector cdf_result; 69 | 70 | for (auto i = 0u; i < 0.99 * data_.size(); i += step_size) { 71 | cdf_result.push_back(data_[i]); 72 | } 73 | 74 | for (auto i = 0u; i < cdf_result.size(); i++) { 75 | cdf << cdf_result[i] << "\t" << 1.0 * (i + 1) / cdf_result.size() 76 | << std::endl; 77 | } 78 | 79 | cdf.close(); 80 | } 81 | 82 | private: 83 | void checkSort() { 84 | if (!isSorted_) { 85 | std::sort(data_.begin(), data_.end()); 86 | isSorted_ = true; 87 | } 88 | } 89 | 90 | private: 91 | bool isSorted_ = true; 92 | std::vector data_; 93 | }; 94 | } // namespace aria -------------------------------------------------------------------------------- /common/Random.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class Random { 12 | public: 13 | Random(uint64_t seed = 0) { init_seed(seed); } 14 | 15 | void init_seed(uint64_t seed) { 16 | seed_ = (seed ^ 0x5DEECE66DULL) & ((1ULL << 48) - 1); 17 | } 18 | 19 | void set_seed(uint64_t seed) { seed_ = seed; } 20 | 21 | uint64_t get_seed() { return seed_; } 22 | 23 | uint64_t next() { return ((uint64_t)next(32) << 32) + next(32); } 24 | 25 | uint64_t next(unsigned int bits) { 26 | seed_ = (seed_ * 0x5DEECE66DULL + 0xBULL) & ((1ULL << 48) - 1); 27 | return (seed_ >> (48 - bits)); 28 | } 29 | 30 | /* [0.0, 1.0) */ 31 | double next_double() { 32 | return (((uint64_t)next(26) << 27) + next(27)) / (double)(1ULL << 53); 33 | } 34 | 35 | uint64_t uniform_dist(uint64_t a, uint64_t b) { 36 | if (a == b) 37 | return a; 38 | return next() % (b - a + 1) + a; 39 | } 40 | 41 | std::string rand_str(std::size_t length, const std::string &str) { 42 | std::string result; 43 | auto str_len = str.length(); 44 | for (auto i = 0u; i < length; i++) { 45 | int k = uniform_dist(0, str_len - 1); 46 | result += str[k]; 47 | } 48 | return result; 49 | } 50 | 51 | std::string a_string(std::size_t min_len, std::size_t max_len) { 52 | auto len = uniform_dist(min_len, max_len); 53 | return rand_str(len, alpha()); 54 | } 55 | 56 | private: 57 | static const std::string &alpha() { 58 | static std::string alpha_ = 59 | "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; 60 | return alpha_; 61 | }; 62 | 63 | uint64_t seed_; 64 | }; 65 | } // namespace aria 66 | -------------------------------------------------------------------------------- /common/Serialization.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/17/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "StringPiece.h" 11 | 12 | namespace aria { 13 | template class Serializer { 14 | public: 15 | std::string operator()(const T &v) { 16 | std::string result(sizeof(T), 0); 17 | memcpy(&result[0], &v, sizeof(T)); 18 | return result; 19 | } 20 | }; 21 | 22 | template class Deserializer { 23 | public: 24 | std::size_t operator()(StringPiece str, T &result) const { 25 | std::memcpy(&result, str.data(), sizeof(T)); 26 | return sizeof(T); 27 | } 28 | }; 29 | 30 | template <> class Serializer { 31 | public: 32 | std::string operator()(const std::string &v) { 33 | return Serializer()(v.size()) + v; 34 | } 35 | }; 36 | 37 | template <> class Deserializer { 38 | public: 39 | std::size_t operator()(StringPiece str, std::string &result) const { 40 | std::string::size_type string_length; 41 | std::size_t size = 42 | Deserializer()(str, string_length); 43 | size += string_length; 44 | str.remove_prefix(sizeof(string_length)); 45 | result = std::string(str.begin(), str.begin() + string_length); 46 | return size; 47 | } 48 | }; 49 | 50 | } // namespace aria 51 | -------------------------------------------------------------------------------- /common/Socket.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/24/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace aria { 18 | 19 | class Socket { 20 | 21 | public: 22 | Socket() : quick_ack(false) { 23 | fd = socket(AF_INET, SOCK_STREAM, 0); 24 | DCHECK(fd >= 0); 25 | } 26 | 27 | Socket(int fd) : quick_ack(false), fd(fd) {} 28 | 29 | // Socket is not copyable 30 | Socket(const Socket &) = delete; 31 | 32 | Socket &operator=(const Socket &) = delete; 33 | 34 | // Socket is movable 35 | Socket(Socket &&that) { 36 | quick_ack = that.quick_ack; 37 | 38 | DCHECK(that.fd >= 0); 39 | fd = that.fd; 40 | that.fd = -1; 41 | } 42 | 43 | Socket &operator=(Socket &&that) { 44 | quick_ack = that.quick_ack; 45 | 46 | DCHECK(that.fd >= 0); 47 | fd = that.fd; 48 | that.fd = -1; 49 | return *this; 50 | } 51 | 52 | int connect(const char *addr, int port) { 53 | DCHECK(fd >= 0); 54 | sockaddr_in serv = make_endpoint(addr, port); 55 | return ::connect(fd, (const sockaddr *)(&serv), sizeof(serv)); 56 | } 57 | 58 | void disable_nagle_algorithm() { 59 | DCHECK(fd >= 0); 60 | // disable Nagle's algorithm 61 | int flag = 1; 62 | int res = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(int)); 63 | CHECK(res >= 0); 64 | } 65 | 66 | void set_quick_ack_flag(bool quick_ack) { this->quick_ack = quick_ack; } 67 | 68 | void try_quick_ack() { 69 | #ifndef __APPLE__ 70 | if (quick_ack) { 71 | DCHECK(fd >= 0); 72 | int flag = 1; 73 | int res = setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(int)); 74 | CHECK(res >= 0); 75 | } 76 | #endif 77 | } 78 | 79 | int close() { 80 | DCHECK(fd >= 0); 81 | return ::close(fd); 82 | } 83 | 84 | long read_n_bytes(char *buf, long size) { 85 | DCHECK(fd >= 0); 86 | long n = 0; 87 | while (n < size) { 88 | long bytes_read = read(buf + n, size - n); 89 | if (bytes_read == 0) { 90 | CHECK(n == 0); // no partial reading is support 91 | return 0; // remote socket is closed. 92 | } 93 | n += bytes_read; 94 | } 95 | return n; 96 | } 97 | 98 | long read_n_bytes_async(char *buf, long size) { 99 | DCHECK(fd >= 0); 100 | long n = 0; 101 | while (n < size) { 102 | long bytes_read = read_async(buf + n, size - n); 103 | if (bytes_read == -1) { // non blocking 104 | CHECK(errno == EWOULDBLOCK || errno == EAGAIN); 105 | if (n == 0) 106 | return -1; 107 | else 108 | continue; 109 | } 110 | if (bytes_read == 0) { 111 | CHECK(n == 0); // no partial reading is support 112 | return 0; // remote socket is closed. 113 | } 114 | n += bytes_read; 115 | } 116 | return n; 117 | } 118 | 119 | long write_n_bytes(const char *buf, long size) { 120 | DCHECK(fd >= 0); 121 | long n = 0; 122 | while (n < size) { 123 | long bytes_written = write(buf + n, size - n); 124 | n += bytes_written; 125 | } 126 | return n; 127 | } 128 | 129 | template long write_number(const T &n) { 130 | DCHECK(fd >= 0); 131 | return write_n_bytes(reinterpret_cast(&n), sizeof(T)); 132 | } 133 | 134 | template long read_number(T &n) { 135 | DCHECK(fd >= 0); 136 | return read_n_bytes(reinterpret_cast(&n), sizeof(T)); 137 | } 138 | 139 | template long read_number_async(T &n) { 140 | DCHECK(fd >= 0); 141 | return read_n_bytes_async(reinterpret_cast(&n), sizeof(T)); 142 | } 143 | 144 | long read(char *buf, long size) { 145 | DCHECK(fd >= 0); 146 | if (size > 0) { 147 | long recv_size = recv(fd, buf, size, 0); 148 | try_quick_ack(); 149 | return recv_size; 150 | } 151 | return 0; 152 | } 153 | 154 | long read_async(char *buf, long size) { 155 | DCHECK(fd >= 0); 156 | if (size > 0) { 157 | long recv_size = recv(fd, buf, size, MSG_DONTWAIT); 158 | try_quick_ack(); 159 | return recv_size; 160 | } 161 | return 0; 162 | } 163 | 164 | long write(const char *buf, long size) { 165 | DCHECK(fd >= 0); 166 | if (size > 0) { 167 | return send(fd, buf, size, 0); 168 | } 169 | return 0; 170 | } 171 | 172 | static sockaddr_in make_endpoint(const char *addr, int port) { 173 | sockaddr_in serv; 174 | memset(&serv, 0, sizeof(serv)); 175 | 176 | serv.sin_family = AF_INET; 177 | serv.sin_addr.s_addr = inet_addr(addr); 178 | serv.sin_port = htons(port); // convert to big-endian order 179 | return serv; 180 | } 181 | 182 | private: 183 | bool quick_ack = false; 184 | int fd; 185 | }; 186 | 187 | class Listener { 188 | public: 189 | Listener(const char *addr, int port, int max_connections) { 190 | fd = socket(AF_INET, SOCK_STREAM, 0); 191 | CHECK(fd >= 0); 192 | bind(addr, port); 193 | listen(max_connections); 194 | } 195 | 196 | Socket accept() { 197 | int acc_fd = ::accept(fd, 0, 0); 198 | CHECK(acc_fd >= 0); 199 | return Socket(acc_fd); 200 | } 201 | 202 | int close() { return ::close(fd); } 203 | 204 | private: 205 | void bind(const char *addr, int port) { 206 | sockaddr_in serv = Socket::make_endpoint(addr, port); 207 | int ret = ::bind(fd, (sockaddr *)(&serv), sizeof(serv)); 208 | CHECK(ret >= 0); 209 | } 210 | 211 | void listen(int max_connections) { ::listen(fd, max_connections); } 212 | 213 | private: 214 | int fd; 215 | }; 216 | } // namespace aria 217 | -------------------------------------------------------------------------------- /common/SpinLock.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | namespace aria { 11 | class SpinLock { 12 | public: 13 | // constructors 14 | SpinLock() = default; 15 | 16 | SpinLock(const SpinLock &) = delete; // non construction-copyable 17 | SpinLock &operator=(const SpinLock &) = delete; // non copyable 18 | 19 | // Modifiers 20 | void lock() { 21 | while (lock_.test_and_set(std::memory_order_acquire)) 22 | ; 23 | } 24 | 25 | void unlock() { lock_.clear(std::memory_order_release); } 26 | 27 | // friend declaration 28 | friend std::ostream &operator<<(std::ostream &, const SpinLock &); 29 | 30 | private: 31 | std::atomic_flag lock_ = ATOMIC_FLAG_INIT; 32 | }; 33 | } // namespace aria 34 | -------------------------------------------------------------------------------- /common/StringPiece.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/28/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | /* 13 | * StringPiece is adapted from 14 | * https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/stubs/stringpiece.h 15 | */ 16 | 17 | namespace aria { 18 | 19 | class StringPiece { 20 | 21 | public: 22 | using iterator = const char *; 23 | using size_type = std::size_t; 24 | 25 | StringPiece() : data_(nullptr), length_(0) {} 26 | 27 | StringPiece(const char *str) : data_(str), length_(0) { 28 | if (data_ != nullptr) { 29 | length_ = strlen(data_); 30 | } 31 | } 32 | 33 | StringPiece(const char *str, size_type length) 34 | : data_(str), length_(length) {} 35 | 36 | StringPiece(const std::string &str) 37 | : data_(str.data()), length_(str.length()) {} 38 | 39 | StringPiece(const StringPiece &that) 40 | : data_(that.data_), length_(that.length_) {} 41 | 42 | const char *data() const { return data_; } 43 | 44 | size_type size() const { return length_; } 45 | 46 | size_type length() const { return length_; } 47 | 48 | bool empty() const { return length_ == 0; } 49 | 50 | void clear() { 51 | data_ = nullptr; 52 | length_ = 0; 53 | } 54 | 55 | void set(const char *data, size_type length) { 56 | data_ = data; 57 | length_ = length; 58 | } 59 | 60 | void set(const char *data) { 61 | data_ = data; 62 | if (data_ == nullptr) { 63 | length_ = 0; 64 | } else { 65 | length_ = strlen(data); 66 | } 67 | } 68 | 69 | char operator[](size_type i) const { 70 | DCHECK(i < length_); 71 | return data_[i]; 72 | } 73 | 74 | void remove_prefix(size_type len) { 75 | DCHECK(len <= length_); 76 | data_ += len; 77 | length_ -= len; 78 | } 79 | 80 | void remove_suffix(size_type len) { 81 | DCHECK(len <= length_); 82 | length_ -= len; 83 | } 84 | 85 | int compare(const StringPiece &that) const { 86 | size_type minSize = length_ < that.length_ ? length_ : that.length_; 87 | int r = strncmp(data_, that.data_, minSize); 88 | if (r < 0) 89 | return -1; 90 | if (r > 0) 91 | return 1; 92 | if (length_ < that.length_) 93 | return -1; 94 | if (length_ > that.length_) 95 | return 1; 96 | return 0; 97 | } 98 | 99 | bool operator<(const StringPiece &that) const { return compare(that) < 0; } 100 | 101 | bool operator<=(const StringPiece &that) const { return compare(that) <= 0; } 102 | 103 | bool operator>(const StringPiece &that) const { return compare(that) > 0; } 104 | 105 | bool operator>=(const StringPiece &that) const { return compare(that) >= 0; } 106 | 107 | bool operator==(const StringPiece &that) const { return compare(that) == 0; } 108 | 109 | bool operator!=(const StringPiece &that) const { return compare(that) != 0; } 110 | 111 | std::string toString() const { 112 | if (data_ == nullptr) 113 | return std::string(); 114 | else 115 | return std::string(data_, length_); 116 | } 117 | 118 | iterator begin() const { return data_; } 119 | 120 | iterator end() const { return data_ + length_; } 121 | 122 | private: 123 | const char *data_; 124 | size_t length_; 125 | }; 126 | } // namespace aria -------------------------------------------------------------------------------- /common/Time.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/22/18. 3 | // 4 | 5 | #include "common/Time.h" 6 | 7 | namespace aria { 8 | std::chrono::steady_clock::time_point Time::startTime = std::chrono::steady_clock::now(); 9 | } -------------------------------------------------------------------------------- /common/Time.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/22/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class Time { 12 | public: 13 | static uint64_t now() { 14 | auto now = std::chrono::steady_clock::now(); 15 | return std::chrono::duration_cast(now - startTime) 16 | .count(); 17 | } 18 | 19 | static std::chrono::steady_clock::time_point startTime; 20 | }; 21 | 22 | } // namespace aria 23 | -------------------------------------------------------------------------------- /common/Zipf.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/19/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | namespace aria { 11 | 12 | class Zipf { 13 | public: 14 | void init(int n, double theta) { 15 | hasInit = true; 16 | 17 | n_ = n; 18 | theta_ = theta; 19 | alpha_ = 1.0 / (1.0 - theta_); 20 | zetan_ = zeta(n_); 21 | eta_ = (1.0 - std::pow(2.0 / n_, 1.0 - theta_)) / (1.0 - zeta(2) / zetan_); 22 | } 23 | 24 | int value(double u) { 25 | CHECK(hasInit); 26 | 27 | double uz = u * zetan_; 28 | int v; 29 | if (uz < 1) { 30 | v = 0; 31 | } else if (uz < 1 + std::pow(0.5, theta_)) { 32 | v = 1; 33 | } else { 34 | v = static_cast(n_ * std::pow(eta_ * u - eta_ + 1, alpha_)); 35 | } 36 | DCHECK(v >= 0 && v < n_); 37 | return v; 38 | } 39 | 40 | static Zipf &globalZipf() { 41 | static Zipf z; 42 | return z; 43 | } 44 | 45 | private: 46 | double zeta(int n) { 47 | DCHECK(hasInit); 48 | 49 | double sum = 0; 50 | 51 | for (auto i = 1; i <= n; i++) { 52 | sum += std::pow(1.0 / i, theta_); 53 | } 54 | 55 | return sum; 56 | } 57 | 58 | bool hasInit = false; 59 | 60 | int n_; 61 | double theta_; 62 | double alpha_; 63 | double zetan_; 64 | double eta_; 65 | }; 66 | } // namespace aria 67 | -------------------------------------------------------------------------------- /compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf CMakeFiles/ CMakeCache.txt goo* 4 | cmake -DCMAKE_BUILD_TYPE=Release . 5 | make -j 6 | -------------------------------------------------------------------------------- /core/Context.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/19/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace aria { 12 | class Context { 13 | 14 | public: 15 | void set_star_partitioner() { 16 | if (protocol != "Star") { 17 | return; 18 | } 19 | if (coordinator_id == 0) { 20 | partitioner = "StarS"; 21 | } else { 22 | partitioner = "StarC"; 23 | } 24 | } 25 | 26 | public: 27 | std::size_t coordinator_id = 0; 28 | std::size_t partition_num = 0; 29 | std::size_t worker_num = 0; 30 | std::size_t coordinator_num = 0; 31 | std::size_t io_thread_num = 1; 32 | std::string protocol; 33 | std::string replica_group; 34 | std::string lock_manager; 35 | std::size_t batch_size = 240; // star, calvin, dbx batch size 36 | std::size_t batch_flush = 10; 37 | std::size_t group_time = 40; // ms 38 | std::size_t sleep_time = 50; // us 39 | std::string partitioner; 40 | std::size_t delay_time = 0; 41 | std::string log_path; 42 | std::string cdf_path; 43 | std::size_t cpu_core_id = 0; 44 | 45 | std::size_t durable_write_cost = 0; 46 | 47 | bool tcp_no_delay = true; 48 | bool tcp_quick_ack = false; 49 | 50 | bool cpu_affinity = true; 51 | 52 | bool sleep_on_retry = true; 53 | 54 | bool exact_group_commit = false; 55 | 56 | bool mvcc = false; 57 | bool bohm_local = false; 58 | bool bohm_single_spin = false; 59 | 60 | bool read_on_replica = false; 61 | bool local_validation = false; 62 | bool rts_sync = false; 63 | bool star_sync_in_single_master_phase = false; 64 | bool star_dynamic_batch_size = true; 65 | bool parallel_locking_and_validation = true; 66 | 67 | bool same_batch = false; // calvin and bohm 68 | 69 | bool aria_read_only_optmization = true; 70 | bool aria_reordering_optmization = true; 71 | bool aria_snapshot_isolation = false; 72 | 73 | std::size_t ariaFB_lock_manager; 74 | 75 | bool pwv_ycsb_star = false; 76 | 77 | bool operation_replication = false; 78 | 79 | std::vector peers; 80 | }; 81 | } // namespace aria 82 | -------------------------------------------------------------------------------- /core/ControlMessage.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/6/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Encoder.h" 8 | #include "common/Message.h" 9 | #include "common/MessagePiece.h" 10 | 11 | #include 12 | 13 | namespace aria { 14 | 15 | enum class ControlMessage { STATISTICS, VECTOR, SIGNAL, ACK, STOP, NFIELDS }; 16 | 17 | class ControlMessageFactory { 18 | 19 | public: 20 | static std::size_t new_statistics_message(Message &message, double value) { 21 | /* 22 | * The structure of a statistics message: (statistics value : double) 23 | * 24 | */ 25 | 26 | // the message is not associated with a table or a partition, use 0. 27 | auto message_size = MessagePiece::get_header_size() + sizeof(double); 28 | auto message_piece_header = MessagePiece::construct_message_piece_header( 29 | static_cast(ControlMessage::STATISTICS), message_size, 0, 0); 30 | 31 | Encoder encoder(message.data); 32 | encoder << message_piece_header; 33 | encoder << value; 34 | message.flush(); 35 | return message_size; 36 | } 37 | 38 | static std::size_t new_vector_message(Message &message, 39 | const std::vector &v) { 40 | /* 41 | * The structure of a vector message: (vector value : v) 42 | * 43 | */ 44 | 45 | // the message is not associated with a table or a partition, use 0. 46 | auto message_size = MessagePiece::get_header_size() + 47 | sizeof(decltype(v.size())) + sizeof(int) * v.size(); 48 | auto message_piece_header = MessagePiece::construct_message_piece_header( 49 | static_cast(ControlMessage::VECTOR), message_size, 0, 0); 50 | 51 | Encoder encoder(message.data); 52 | encoder << message_piece_header; 53 | encoder << v.size(); 54 | for (auto i = 0u; i < v.size(); i++) { 55 | encoder << v[i]; 56 | } 57 | message.flush(); 58 | return message_size; 59 | } 60 | 61 | static std::size_t new_signal_message(Message &message, uint32_t value) { 62 | 63 | /* 64 | * The structure of a signal message: (signal value : uint32_t) 65 | */ 66 | 67 | // the message is not associated with a table or a partition, use 0. 68 | auto message_size = MessagePiece::get_header_size() + sizeof(uint32_t); 69 | auto message_piece_header = MessagePiece::construct_message_piece_header( 70 | static_cast(ControlMessage::SIGNAL), message_size, 0, 0); 71 | 72 | Encoder encoder(message.data); 73 | encoder << message_piece_header; 74 | encoder << value; 75 | message.flush(); 76 | return message_size; 77 | } 78 | 79 | static std::size_t new_ack_message(Message &message) { 80 | /* 81 | * The structure of an ack message: () 82 | */ 83 | 84 | auto message_size = MessagePiece::get_header_size(); 85 | auto message_piece_header = MessagePiece::construct_message_piece_header( 86 | static_cast(ControlMessage::ACK), message_size, 0, 0); 87 | Encoder encoder(message.data); 88 | encoder << message_piece_header; 89 | message.flush(); 90 | return message_size; 91 | } 92 | 93 | static std::size_t new_stop_message(Message &message) { 94 | /* 95 | * The structure of a stop message: () 96 | */ 97 | 98 | auto message_size = MessagePiece::get_header_size(); 99 | auto message_piece_header = MessagePiece::construct_message_piece_header( 100 | static_cast(ControlMessage::STOP), message_size, 0, 0); 101 | Encoder encoder(message.data); 102 | encoder << message_piece_header; 103 | message.flush(); 104 | return message_size; 105 | } 106 | }; 107 | 108 | } // namespace aria 109 | -------------------------------------------------------------------------------- /core/Defs.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/10/18. 3 | // 4 | 5 | #pragma once 6 | 7 | namespace aria { 8 | 9 | enum class ExecutorStatus { 10 | START, 11 | CLEANUP, 12 | C_PHASE, 13 | S_PHASE, 14 | Analysis, 15 | Execute, 16 | Aria_READ, 17 | Aria_COMMIT, 18 | AriaFB_READ, 19 | AriaFB_COMMIT, 20 | AriaFB_Fallback_Prepare, 21 | AriaFB_Fallback, 22 | Bohm_Analysis, 23 | Bohm_Insert, 24 | Bohm_Execute, 25 | Bohm_GC, 26 | Pwv_Analysis, 27 | Pwv_Execute, 28 | STOP, 29 | EXIT 30 | }; 31 | 32 | enum class TransactionResult { COMMIT, READY_TO_COMMIT, ABORT, ABORT_NORETRY }; 33 | 34 | } // namespace aria 35 | -------------------------------------------------------------------------------- /core/Delay.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/28/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | // delay in us from the sender side 12 | 13 | class Delay { 14 | 15 | public: 16 | Delay(std::size_t coordinator_id, std::size_t coordinator_num) { 17 | DCHECK(coordinator_id < coordinator_num); 18 | this->coordinator_id = coordinator_id; 19 | this->coordinator_num = coordinator_num; 20 | } 21 | 22 | virtual ~Delay() = default; 23 | 24 | virtual int64_t message_delay() const = 0; 25 | 26 | virtual bool delay_enabled() const = 0; 27 | 28 | protected: 29 | std::size_t coordinator_id; 30 | std::size_t coordinator_num; 31 | }; 32 | 33 | class SameDelay : public Delay { 34 | 35 | public: 36 | SameDelay(std::size_t coordinator_id, std::size_t coordinator_num, 37 | int64_t delay_time) 38 | : Delay(coordinator_id, coordinator_num), delay_time(delay_time) { 39 | DCHECK(delay_time >= 0); 40 | } 41 | 42 | virtual ~SameDelay() = default; 43 | 44 | int64_t message_delay() const override { return delay_time; } 45 | 46 | bool delay_enabled() const override { return delay_time != 0; } 47 | 48 | protected: 49 | int64_t delay_time; 50 | }; 51 | 52 | } // namespace aria -------------------------------------------------------------------------------- /core/Dispatcher.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 8/29/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/BufferedReader.h" 8 | #include "common/LockfreeQueue.h" 9 | #include "common/Message.h" 10 | #include "common/Socket.h" 11 | #include "core/ControlMessage.h" 12 | #include "core/Worker.h" 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace aria { 19 | class IncomingDispatcher { 20 | 21 | public: 22 | IncomingDispatcher(std::size_t id, std::size_t group_id, 23 | std::size_t io_thread_num, std::vector &sockets, 24 | const std::vector> &workers, 25 | LockfreeQueue &coordinator_queue, 26 | std::atomic &stopFlag) 27 | : id(id), group_id(group_id), io_thread_num(io_thread_num), 28 | network_size(0), workers(workers), coordinator_queue(coordinator_queue), 29 | stopFlag(stopFlag) { 30 | 31 | for (auto i = 0u; i < sockets.size(); i++) { 32 | buffered_readers.emplace_back(sockets[i]); 33 | } 34 | } 35 | 36 | void start() { 37 | auto numCoordinators = buffered_readers.size(); 38 | auto numWorkers = workers.size(); 39 | 40 | // single node test mode 41 | if (numCoordinators == 1) { 42 | return; 43 | } 44 | 45 | LOG(INFO) << "Incoming Dispatcher started, numCoordinators = " 46 | << numCoordinators << ", numWorkers = " << numWorkers 47 | << ", group id = " << group_id; 48 | 49 | while (!stopFlag.load()) { 50 | 51 | for (auto i = 0u; i < numCoordinators; i++) { 52 | if (i == id) { 53 | continue; 54 | } 55 | 56 | auto message = buffered_readers[i].next_message(); 57 | 58 | if (message == nullptr) { 59 | std::this_thread::yield(); 60 | continue; 61 | } 62 | 63 | network_size += message->get_message_length(); 64 | 65 | // check coordinator message 66 | if (is_coordinator_message(message.get())) { 67 | coordinator_queue.push(message.release()); 68 | CHECK(group_id == 0); 69 | continue; 70 | } 71 | 72 | auto workerId = message->get_worker_id(); 73 | CHECK(workerId % io_thread_num == group_id); 74 | // release the unique ptr 75 | workers[workerId]->push_message(message.release()); 76 | DCHECK(message == nullptr); 77 | } 78 | } 79 | 80 | LOG(INFO) << "Incoming Dispatcher exits, network size: " << network_size; 81 | } 82 | 83 | bool is_coordinator_message(Message *message) { 84 | return (*(message->begin())).get_message_type() == 85 | static_cast(ControlMessage::STATISTICS); 86 | } 87 | 88 | std::unique_ptr fetchMessage(Socket &socket) { return nullptr; } 89 | 90 | private: 91 | std::size_t id; 92 | std::size_t group_id; 93 | std::size_t io_thread_num; 94 | std::size_t network_size; 95 | std::vector buffered_readers; 96 | std::vector> workers; 97 | LockfreeQueue &coordinator_queue; 98 | std::atomic &stopFlag; 99 | }; 100 | 101 | class OutgoingDispatcher { 102 | public: 103 | OutgoingDispatcher(std::size_t id, std::size_t group_id, 104 | std::size_t io_thread_num, std::vector &sockets, 105 | const std::vector> &workers, 106 | LockfreeQueue &coordinator_queue, 107 | std::atomic &stopFlag) 108 | : id(id), group_id(group_id), io_thread_num(io_thread_num), 109 | network_size(0), sockets(sockets), workers(workers), 110 | coordinator_queue(coordinator_queue), stopFlag(stopFlag) {} 111 | 112 | void start() { 113 | 114 | auto numCoordinators = sockets.size(); 115 | auto numWorkers = workers.size(); 116 | // single node test mode 117 | if (numCoordinators == 1) { 118 | return; 119 | } 120 | 121 | LOG(INFO) << "Outgoing Dispatcher started, numCoordinators = " 122 | << numCoordinators << ", numWorkers = " << numWorkers 123 | << ", group id = " << group_id; 124 | 125 | while (!stopFlag.load()) { 126 | 127 | // check coordinator 128 | 129 | if (group_id == 0 && !coordinator_queue.empty()) { 130 | std::unique_ptr message(coordinator_queue.front()); 131 | bool ok = coordinator_queue.pop(); 132 | CHECK(ok); 133 | sendMessage(message.get()); 134 | } 135 | 136 | for (auto i = group_id; i < numWorkers; i += io_thread_num) { 137 | dispatchMessage(workers[i]); 138 | } 139 | std::this_thread::yield(); 140 | } 141 | 142 | LOG(INFO) << "Outgoing Dispatcher exits, network size: " << network_size; 143 | } 144 | 145 | void sendMessage(Message *message) { 146 | auto dest_node_id = message->get_dest_node_id(); 147 | DCHECK(dest_node_id >= 0 && dest_node_id < sockets.size() && 148 | dest_node_id != id); 149 | DCHECK(message->get_message_length() == message->data.length()) 150 | << message->get_message_length() << " " << message->data.length(); 151 | 152 | sockets[dest_node_id].write_n_bytes(message->get_raw_ptr(), 153 | message->get_message_length()); 154 | 155 | network_size += message->get_message_length(); 156 | } 157 | 158 | void dispatchMessage(const std::shared_ptr &worker) { 159 | 160 | Message *raw_message = worker->pop_message(); 161 | if (raw_message == nullptr) { 162 | return; 163 | } 164 | // wrap the message with a unique pointer. 165 | std::unique_ptr message(raw_message); 166 | // send the message 167 | sendMessage(message.get()); 168 | } 169 | 170 | private: 171 | std::size_t id; 172 | std::size_t group_id; 173 | std::size_t io_thread_num; 174 | std::size_t network_size; 175 | std::vector &sockets; 176 | std::vector> workers; 177 | LockfreeQueue &coordinator_queue; 178 | std::atomic &stopFlag; 179 | }; 180 | 181 | } // namespace aria 182 | -------------------------------------------------------------------------------- /core/Macros.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 3/18/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "glog/logging.h" 8 | #include 9 | 10 | DEFINE_string(servers, "127.0.0.1:10010", 11 | "semicolon-separated list of servers"); 12 | DEFINE_int32(id, 0, "coordinator id"); 13 | DEFINE_int32(threads, 1, "the number of threads"); 14 | DEFINE_int32(io, 1, "the number of i/o threads"); 15 | DEFINE_int32(partition_num, 1, "the number of partitions"); 16 | DEFINE_string(partitioner, "hash", "database partitioner (hash, hash2, pb)"); 17 | DEFINE_bool(sleep_on_retry, true, "sleep when retry aborted transactions"); 18 | DEFINE_int32(batch_size, 100, "star or calvin batch size"); 19 | DEFINE_int32(group_time, 10, "group commit frequency"); 20 | DEFINE_int32(batch_flush, 50, "batch flush"); 21 | DEFINE_int32(sleep_time, 1000, "retry sleep time"); 22 | DEFINE_string(protocol, "aria", "transaction protocol"); 23 | DEFINE_string(replica_group, "1,3", "calvin replica group"); 24 | DEFINE_string(lock_manager, "1,1", "calvin lock manager"); 25 | DEFINE_bool(read_on_replica, false, "read from replicas"); 26 | DEFINE_bool(local_validation, false, "local validation"); 27 | DEFINE_bool(rts_sync, false, "rts sync"); 28 | DEFINE_bool(star_sync, false, "synchronous write in the single-master phase"); 29 | DEFINE_bool(star_dynamic_batch_size, true, "dynamic batch size"); 30 | DEFINE_bool(plv, true, "parallel locking and validation"); 31 | DEFINE_bool(same_batch, false, 32 | "always run the same batch of txns in calvin and bohm."); 33 | DEFINE_bool(aria_read_only, true, "aria read only optimization"); 34 | DEFINE_bool(aria_reordering, true, "aria reordering optimization"); 35 | DEFINE_bool(aria_si, false, "aria snapshot isolation"); 36 | DEFINE_int32(delay, 0, "delay time in us."); 37 | DEFINE_string(cdf_path, "", "path to cdf"); 38 | DEFINE_string(log_path, "", "path to disk logging."); 39 | DEFINE_bool(tcp_no_delay, true, "TCP Nagle algorithm, true: disable nagle"); 40 | DEFINE_bool(tcp_quick_ack, false, "TCP quick ack mode, true: enable quick ack"); 41 | DEFINE_bool(cpu_affinity, true, "pinning each thread to a separate core"); 42 | DEFINE_int32(cpu_core_id, 0, "cpu core id"); 43 | DEFINE_int32(durable_write_cost, 0, 44 | "the cost of durable write in microseconds"); 45 | DEFINE_bool(exact_group_commit, false, "dynamically adjust group time."); 46 | DEFINE_bool(mvcc, false, "use mvcc storage for BOHM."); 47 | DEFINE_bool(bohm_local, false, "locality optimization for Bohm."); 48 | DEFINE_bool(bohm_single_spin, false, "spin optimization for Bohm."); 49 | DEFINE_int32(ariaFB_lock_manager, 0, 50 | "# of lock manager in aria's fallback mode."); 51 | 52 | #define SETUP_CONTEXT(context) \ 53 | boost::algorithm::split(context.peers, FLAGS_servers, \ 54 | boost::is_any_of(";")); \ 55 | context.coordinator_num = context.peers.size(); \ 56 | context.coordinator_id = FLAGS_id; \ 57 | context.worker_num = FLAGS_threads; \ 58 | context.io_thread_num = FLAGS_io; \ 59 | context.partition_num = FLAGS_partition_num; \ 60 | context.partitioner = FLAGS_partitioner; \ 61 | context.sleep_on_retry = FLAGS_sleep_on_retry; \ 62 | context.batch_size = FLAGS_batch_size; \ 63 | context.group_time = FLAGS_group_time; \ 64 | context.batch_flush = FLAGS_batch_flush; \ 65 | context.sleep_time = FLAGS_sleep_time; \ 66 | context.protocol = FLAGS_protocol; \ 67 | context.replica_group = FLAGS_replica_group; \ 68 | context.lock_manager = FLAGS_lock_manager; \ 69 | context.read_on_replica = FLAGS_read_on_replica; \ 70 | context.local_validation = FLAGS_local_validation; \ 71 | context.rts_sync = FLAGS_rts_sync; \ 72 | context.star_sync_in_single_master_phase = FLAGS_star_sync; \ 73 | context.star_dynamic_batch_size = FLAGS_star_dynamic_batch_size; \ 74 | context.parallel_locking_and_validation = FLAGS_plv; \ 75 | context.same_batch = FLAGS_same_batch; \ 76 | context.aria_read_only_optmization = FLAGS_aria_read_only; \ 77 | context.aria_reordering_optmization = FLAGS_aria_reordering; \ 78 | context.aria_snapshot_isolation = FLAGS_aria_si; \ 79 | context.delay_time = FLAGS_delay; \ 80 | context.log_path = FLAGS_log_path; \ 81 | context.cdf_path = FLAGS_cdf_path; \ 82 | context.tcp_no_delay = FLAGS_tcp_no_delay; \ 83 | context.tcp_quick_ack = FLAGS_tcp_quick_ack; \ 84 | context.cpu_affinity = FLAGS_cpu_affinity; \ 85 | context.cpu_core_id = FLAGS_cpu_core_id; \ 86 | context.durable_write_cost = FLAGS_durable_write_cost; \ 87 | context.exact_group_commit = FLAGS_exact_group_commit; \ 88 | context.mvcc = FLAGS_mvcc; \ 89 | context.bohm_local = FLAGS_bohm_local; \ 90 | context.bohm_single_spin = FLAGS_bohm_single_spin; \ 91 | context.ariaFB_lock_manager = FLAGS_ariaFB_lock_manager; \ 92 | CHECK(context.coordinator_num == 1 || context.bohm_single_spin == false) \ 93 | << "bohm_single_spin must be used in single-node mode."; \ 94 | CHECK((context.mvcc ^ (context.protocol == "Bohm")) == 0) \ 95 | << "MVCC must be used in Bohm."; \ 96 | context.set_star_partitioner(); 97 | -------------------------------------------------------------------------------- /core/SchemaDef.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/15/18. 3 | // 4 | 5 | #pragma once 6 | 7 | // macros for code generation 8 | 9 | #define APPLY_X_AND_Y(x, y) x(y, y) 10 | 11 | #define NAMESPACE_OPEN(name) namespace name { 12 | 13 | #define NAMESPACE_CLOSE(name) } 14 | 15 | #define NAMESPACE_EXPAND(name) name:: 16 | 17 | #define STRUCT_PARAM_FIRST_X(type, name) type name 18 | 19 | #define STRUCT_PARAM_REST_X(type, name) , type name 20 | 21 | #define STRUCT_INITLIST_FIRST_X(type, name) name(name) 22 | 23 | #define STRUCT_INITLIST_REST_X(type, name) , name(name) 24 | 25 | #define STRUCT_HASH_FIRST_X(type, name) k.name 26 | 27 | #define STRUCT_HASH_REST_X(type, name) , k.name 28 | 29 | #define STRUCT_LAYOUT_X(type, name) type name; 30 | 31 | #define STRUCT_EQ_X(type, name) \ 32 | if (this->name != other.name) \ 33 | return false; 34 | 35 | #define STRUCT_FIELDPOS_X(type, name) name##_field, 36 | 37 | // the main macro 38 | #define DO_STRUCT(name, keyfields, valuefields, namespacefields) \ 39 | namespacefields(NAMESPACE_OPEN) struct name { \ 40 | struct key { \ 41 | key() = default; \ 42 | key(keyfields(STRUCT_PARAM_FIRST_X, STRUCT_PARAM_REST_X)) \ 43 | : keyfields(STRUCT_INITLIST_FIRST_X, STRUCT_INITLIST_REST_X) {} \ 44 | APPLY_X_AND_Y(keyfields, STRUCT_LAYOUT_X) \ 45 | bool operator==(const struct key &other) const { \ 46 | APPLY_X_AND_Y(keyfields, STRUCT_EQ_X) \ 47 | return true; \ 48 | } \ 49 | bool operator!=(const struct key &other) const { \ 50 | return !operator==(other); \ 51 | } \ 52 | enum { APPLY_X_AND_Y(keyfields, STRUCT_FIELDPOS_X) NFIELDS }; \ 53 | }; \ 54 | struct value { \ 55 | value() = default; \ 56 | value(valuefields(STRUCT_PARAM_FIRST_X, STRUCT_PARAM_REST_X)) \ 57 | : valuefields(STRUCT_INITLIST_FIRST_X, STRUCT_INITLIST_REST_X) {} \ 58 | APPLY_X_AND_Y(valuefields, STRUCT_LAYOUT_X) \ 59 | bool operator==(const struct value &other) const { \ 60 | APPLY_X_AND_Y(valuefields, STRUCT_EQ_X) \ 61 | return true; \ 62 | } \ 63 | bool operator!=(const struct value &other) const { \ 64 | return !operator==(other); \ 65 | } \ 66 | enum { APPLY_X_AND_Y(valuefields, STRUCT_FIELDPOS_X) NFIELDS }; \ 67 | }; \ 68 | static constexpr std::size_t tableID = __COUNTER__ - __BASE_COUNTER__; \ 69 | }; \ 70 | namespacefields(NAMESPACE_CLOSE) namespace std { \ 71 | template <> struct hash { \ 72 | std::size_t operator()(const namespacefields(NAMESPACE_EXPAND) \ 73 | name::key &k) const { \ 74 | return aria::hash(keyfields(STRUCT_HASH_FIRST_X, STRUCT_HASH_REST_X)); \ 75 | } \ 76 | }; \ 77 | } 78 | -------------------------------------------------------------------------------- /core/Worker.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 7/22/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/LockfreeQueue.h" 8 | #include "common/Message.h" 9 | #include 10 | #include 11 | #include 12 | 13 | namespace aria { 14 | 15 | class Worker { 16 | public: 17 | Worker(std::size_t coordinator_id, std::size_t id) 18 | : coordinator_id(coordinator_id), id(id) { 19 | n_commit.store(0); 20 | n_abort_no_retry.store(0); 21 | n_abort_lock.store(0); 22 | n_abort_read_validation.store(0); 23 | n_local.store(0); 24 | n_si_in_serializable.store(0); 25 | n_network_size.store(0); 26 | } 27 | 28 | virtual ~Worker() = default; 29 | 30 | virtual void start() = 0; 31 | 32 | virtual void onExit() {} 33 | 34 | virtual void push_message(Message *message) = 0; 35 | 36 | virtual Message *pop_message() = 0; 37 | 38 | public: 39 | std::size_t coordinator_id; 40 | std::size_t id; 41 | std::atomic n_commit, n_abort_no_retry, n_abort_lock, 42 | n_abort_read_validation, n_local, n_si_in_serializable, n_network_size; 43 | }; 44 | 45 | } // namespace aria 46 | -------------------------------------------------------------------------------- /loc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cloc benchmark common core protocol *.cpp 4 | -------------------------------------------------------------------------------- /protocol/Aria/Aria.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Partitioner.h" 8 | #include "core/Table.h" 9 | #include "protocol/Aria/AriaHelper.h" 10 | #include "protocol/Aria/AriaMessage.h" 11 | #include "protocol/Aria/AriaTransaction.h" 12 | 13 | namespace aria { 14 | 15 | template class Aria { 16 | public: 17 | using DatabaseType = Database; 18 | using MetaDataType = std::atomic; 19 | using ContextType = typename DatabaseType::ContextType; 20 | using MessageType = AriaMessage; 21 | using TransactionType = AriaTransaction; 22 | 23 | using MessageFactoryType = AriaMessageFactory; 24 | using MessageHandlerType = AriaMessageHandler; 25 | 26 | Aria(DatabaseType &db, const ContextType &context, Partitioner &partitioner) 27 | : db(db), context(context), partitioner(partitioner) {} 28 | 29 | void abort(TransactionType &txn, 30 | std::vector> &messages) { 31 | // nothing needs to be done 32 | } 33 | 34 | bool commit(TransactionType &txn, 35 | std::vector> &messages) { 36 | 37 | auto &writeSet = txn.writeSet; 38 | for (auto i = 0u; i < writeSet.size(); i++) { 39 | auto &writeKey = writeSet[i]; 40 | auto tableId = writeKey.get_table_id(); 41 | auto partitionId = writeKey.get_partition_id(); 42 | auto table = db.find_table(tableId, partitionId); 43 | 44 | if (partitioner.has_master_partition(partitionId)) { 45 | auto key = writeKey.get_key(); 46 | auto value = writeKey.get_value(); 47 | table->update(key, value); 48 | } else { 49 | auto coordinatorID = partitioner.master_coordinator(partitionId); 50 | txn.network_size += MessageFactoryType::new_write_message( 51 | *messages[coordinatorID], *table, writeKey.get_key(), 52 | writeKey.get_value()); 53 | } 54 | } 55 | 56 | return true; 57 | } 58 | 59 | private: 60 | DatabaseType &db; 61 | const ContextType &context; 62 | Partitioner &partitioner; 63 | }; 64 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Aria/AriaHelper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "core/Table.h" 12 | 13 | #include "glog/logging.h" 14 | #include "protocol/Aria/AriaRWKey.h" 15 | 16 | namespace aria { 17 | 18 | class AriaHelper { 19 | 20 | public: 21 | using MetaDataType = std::atomic; 22 | 23 | static uint64_t read(const std::tuple &row, 24 | void *dest, std::size_t size) { 25 | MetaDataType &tid = *std::get<0>(row); 26 | void *src = std::get<1>(row); 27 | std::memcpy(dest, src, size); 28 | return tid.load(); 29 | } 30 | 31 | static void 32 | set_key_tid(AriaRWKey &key, 33 | const std::tuple *, void *> &row) { 34 | key.set_tid(std::get<0>(row)); 35 | } 36 | 37 | static std::atomic &get_metadata(ITable *table, 38 | const AriaRWKey &key) { 39 | auto tid = key.get_tid(); 40 | if (!tid) { 41 | tid = &table->search_metadata(key.get_key()); 42 | } 43 | return *tid; 44 | } 45 | 46 | static bool reserve_read(std::atomic &a, uint64_t epoch, 47 | uint32_t tid) { 48 | uint64_t old_value, new_value; 49 | do { 50 | old_value = a.load(); 51 | uint64_t old_epoch = get_epoch(old_value); 52 | uint64_t old_rts = get_rts(old_value); 53 | 54 | CHECK(epoch >= old_epoch); 55 | if (epoch > old_epoch) { 56 | new_value = set_epoch(0, epoch); 57 | new_value = set_rts(new_value, tid); 58 | } else { 59 | 60 | if (old_rts < tid && old_rts != 0) { 61 | return false; 62 | } 63 | // keep wts 64 | new_value = old_value; 65 | new_value = set_rts(new_value, tid); 66 | } 67 | } while (!a.compare_exchange_weak(old_value, new_value)); 68 | return true; 69 | } 70 | 71 | static bool reserve_write(std::atomic &a, uint64_t epoch, 72 | uint32_t tid) { 73 | uint64_t old_value, new_value; 74 | do { 75 | old_value = a.load(); 76 | uint64_t old_epoch = get_epoch(old_value); 77 | uint64_t old_wts = get_wts(old_value); 78 | 79 | CHECK(epoch >= old_epoch); 80 | if (epoch > old_epoch) { 81 | new_value = set_epoch(0, epoch); 82 | new_value = set_wts(new_value, tid); 83 | } else { 84 | 85 | if (old_wts < tid && old_wts != 0) { 86 | return false; 87 | } 88 | // keep rts 89 | new_value = old_value; 90 | new_value = set_wts(new_value, tid); 91 | } 92 | } while (!a.compare_exchange_weak(old_value, new_value)); 93 | return true; 94 | } 95 | 96 | static uint64_t get_epoch(uint64_t value) { 97 | return (value >> EPOCH_OFFSET) & EPOCH_MASK; 98 | } 99 | 100 | static uint64_t set_epoch(uint64_t value, uint64_t epoch) { 101 | DCHECK(epoch < (1ull << 24)); 102 | return (value & (~(EPOCH_MASK << EPOCH_OFFSET))) | (epoch << EPOCH_OFFSET); 103 | } 104 | 105 | static uint64_t get_rts(uint64_t value) { 106 | return (value >> RTS_OFFSET) & RTS_MASK; 107 | } 108 | 109 | static uint64_t set_rts(uint64_t value, uint64_t rts) { 110 | DCHECK(rts < (1ull << 20)); 111 | return (value & (~(RTS_MASK << RTS_OFFSET))) | (rts << RTS_OFFSET); 112 | } 113 | 114 | static uint64_t get_wts(uint64_t value) { 115 | return (value >> WTS_OFFSET) & WTS_MASK; 116 | } 117 | 118 | static uint64_t set_wts(uint64_t value, uint64_t wts) { 119 | DCHECK(wts < (1ull << 20)); 120 | return (value & (~(WTS_MASK << WTS_OFFSET))) | (wts << WTS_OFFSET); 121 | } 122 | 123 | public: 124 | /* 125 | * [epoch (24) | read-rts (20) | write-wts (20)] 126 | * 127 | */ 128 | 129 | static constexpr int EPOCH_OFFSET = 40; 130 | static constexpr uint64_t EPOCH_MASK = 0xffffffull; 131 | 132 | static constexpr int RTS_OFFSET = 20; 133 | static constexpr uint64_t RTS_MASK = 0xfffffull; 134 | 135 | static constexpr int WTS_OFFSET = 0; 136 | static constexpr uint64_t WTS_MASK = 0xfffffull; 137 | }; 138 | 139 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Aria/AriaManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Manager.h" 8 | #include "core/Partitioner.h" 9 | #include "protocol/Aria/Aria.h" 10 | #include "protocol/Aria/AriaExecutor.h" 11 | #include "protocol/Aria/AriaHelper.h" 12 | #include "protocol/Aria/AriaTransaction.h" 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | namespace aria { 19 | 20 | template class AriaManager : public aria::Manager { 21 | public: 22 | using base_type = aria::Manager; 23 | 24 | using WorkloadType = Workload; 25 | using DatabaseType = typename WorkloadType::DatabaseType; 26 | using StorageType = typename WorkloadType::StorageType; 27 | 28 | using TransactionType = AriaTransaction; 29 | static_assert(std::is_same::value, 31 | "Transaction types do not match."); 32 | using ContextType = typename DatabaseType::ContextType; 33 | using RandomType = typename DatabaseType::RandomType; 34 | 35 | AriaManager(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 36 | const ContextType &context, std::atomic &stopFlag) 37 | : base_type(coordinator_id, id, context, stopFlag), db(db), epoch(0) { 38 | 39 | storages.resize(context.batch_size); 40 | transactions.resize(context.batch_size); 41 | } 42 | 43 | void coordinator_start() override { 44 | 45 | std::size_t n_workers = context.worker_num; 46 | std::size_t n_coordinators = context.coordinator_num; 47 | 48 | while (!stopFlag.load()) { 49 | 50 | // the coordinator on each machine first moves the aborted transactions 51 | // from the last batch earlier to the next batch and set remaining 52 | // transaction slots to null. 53 | 54 | // then, each worker threads generates a transaction using the same seed. 55 | epoch.fetch_add(1); 56 | cleanup_batch(); 57 | 58 | // LOG(INFO) << "Seed: " << random.get_seed(); 59 | n_started_workers.store(0); 60 | n_completed_workers.store(0); 61 | signal_worker(ExecutorStatus::Aria_READ); 62 | wait_all_workers_start(); 63 | wait_all_workers_finish(); 64 | broadcast_stop(); 65 | wait4_stop(n_coordinators - 1); 66 | n_completed_workers.store(0); 67 | set_worker_status(ExecutorStatus::STOP); 68 | wait_all_workers_finish(); 69 | // wait for all machines until they finish the Aria_READ phase. 70 | wait4_ack(); 71 | 72 | // Allow each worker to commit transactions 73 | n_started_workers.store(0); 74 | n_completed_workers.store(0); 75 | signal_worker(ExecutorStatus::Aria_COMMIT); 76 | wait_all_workers_start(); 77 | wait_all_workers_finish(); 78 | broadcast_stop(); 79 | wait4_stop(n_coordinators - 1); 80 | n_completed_workers.store(0); 81 | set_worker_status(ExecutorStatus::STOP); 82 | wait_all_workers_finish(); 83 | // wait for all machines until they finish the Aria_COMMIT phase. 84 | wait4_ack(); 85 | } 86 | 87 | signal_worker(ExecutorStatus::EXIT); 88 | } 89 | 90 | void non_coordinator_start() override { 91 | 92 | std::size_t n_workers = context.worker_num; 93 | std::size_t n_coordinators = context.coordinator_num; 94 | 95 | for (;;) { 96 | // LOG(INFO) << "Seed: " << random.get_seed(); 97 | ExecutorStatus status = wait4_signal(); 98 | if (status == ExecutorStatus::EXIT) { 99 | set_worker_status(ExecutorStatus::EXIT); 100 | break; 101 | } 102 | 103 | DCHECK(status == ExecutorStatus::Aria_READ); 104 | // the coordinator on each machine first moves the aborted transactions 105 | // from the last batch earlier to the next batch and set remaining 106 | // transaction slots to null. 107 | 108 | epoch.fetch_add(1); 109 | cleanup_batch(); 110 | 111 | n_started_workers.store(0); 112 | n_completed_workers.store(0); 113 | set_worker_status(ExecutorStatus::Aria_READ); 114 | wait_all_workers_start(); 115 | wait_all_workers_finish(); 116 | broadcast_stop(); 117 | wait4_stop(n_coordinators - 1); 118 | n_completed_workers.store(0); 119 | set_worker_status(ExecutorStatus::STOP); 120 | wait_all_workers_finish(); 121 | send_ack(); 122 | 123 | status = wait4_signal(); 124 | DCHECK(status == ExecutorStatus::Aria_COMMIT); 125 | n_started_workers.store(0); 126 | n_completed_workers.store(0); 127 | set_worker_status(ExecutorStatus::Aria_COMMIT); 128 | wait_all_workers_start(); 129 | wait_all_workers_finish(); 130 | broadcast_stop(); 131 | wait4_stop(n_coordinators - 1); 132 | n_completed_workers.store(0); 133 | set_worker_status(ExecutorStatus::STOP); 134 | wait_all_workers_finish(); 135 | send_ack(); 136 | } 137 | } 138 | 139 | void cleanup_batch() { 140 | std::size_t it = 0; 141 | for (auto i = 0u; i < transactions.size(); i++) { 142 | if (transactions[i] == nullptr) { 143 | break; 144 | } 145 | if (transactions[i]->abort_lock) { 146 | transactions[it++].swap(transactions[i]); 147 | } 148 | } 149 | total_abort.store(it); 150 | } 151 | 152 | public: 153 | RandomType random; 154 | DatabaseType &db; 155 | std::atomic epoch; 156 | std::vector storages; 157 | std::vector> transactions; 158 | std::atomic total_abort; 159 | }; 160 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Aria/AriaRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class AriaRWKey { 12 | public: 13 | // local index read bit 14 | 15 | void set_local_index_read_bit() { 16 | clear_local_index_read_bit(); 17 | bitvec |= LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET; 18 | } 19 | 20 | void clear_local_index_read_bit() { 21 | bitvec &= ~(LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET); 22 | } 23 | 24 | uint32_t get_local_index_read_bit() const { 25 | return (bitvec >> LOCAL_INDEX_READ_BIT_OFFSET) & LOCAL_INDEX_READ_BIT_MASK; 26 | } 27 | 28 | // read request bit 29 | 30 | void set_read_request_bit() { 31 | clear_read_request_bit(); 32 | bitvec |= READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET; 33 | } 34 | 35 | void clear_read_request_bit() { 36 | bitvec &= ~(READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET); 37 | } 38 | 39 | uint32_t get_read_request_bit() const { 40 | return (bitvec >> READ_REQUEST_BIT_OFFSET) & READ_REQUEST_BIT_MASK; 41 | } 42 | 43 | // table id 44 | 45 | void set_table_id(uint32_t table_id) { 46 | DCHECK(table_id < (1 << 5)); 47 | clear_table_id(); 48 | bitvec |= table_id << TABLE_ID_OFFSET; 49 | } 50 | 51 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 52 | 53 | uint32_t get_table_id() const { 54 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 55 | } 56 | // partition id 57 | 58 | void set_partition_id(uint32_t partition_id) { 59 | DCHECK(partition_id < (1 << 16)); 60 | clear_partition_id(); 61 | bitvec |= partition_id << PARTITION_ID_OFFSET; 62 | } 63 | 64 | void clear_partition_id() { 65 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 66 | } 67 | 68 | uint32_t get_partition_id() const { 69 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 70 | } 71 | 72 | // key 73 | void set_key(const void *key) { this->key = key; } 74 | 75 | const void *get_key() const { return key; } 76 | 77 | // value 78 | void set_value(void *value) { this->value = value; } 79 | 80 | void *get_value() const { return value; } 81 | 82 | void set_tid(std::atomic *tid) { this->tid = tid; }; 83 | 84 | std::atomic *get_tid() const { return tid; }; 85 | 86 | private: 87 | /* 88 | * A bitvec is a 32-bit word. 89 | * 90 | * [ table id (5) ] | partition id (16) | unused bit (9) | 91 | * read request bit (1) | local index read (1) ] 92 | * 93 | * local index read is set when the read is from a local read only index. 94 | */ 95 | 96 | uint32_t bitvec = 0; 97 | const void *key = nullptr; 98 | void *value = nullptr; 99 | std::atomic *tid = nullptr; 100 | 101 | public: 102 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 103 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 104 | 105 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 106 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 107 | 108 | static constexpr uint32_t READ_REQUEST_BIT_MASK = 0x1; 109 | static constexpr uint32_t READ_REQUEST_BIT_OFFSET = 1; 110 | 111 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_MASK = 0x1; 112 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_OFFSET = 0; 113 | }; 114 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Aria/AriaTransaction.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Operation.h" 8 | #include "core/Defs.h" 9 | #include "core/Partitioner.h" 10 | #include "core/Table.h" 11 | #include "protocol/Aria/AriaHelper.h" 12 | #include "protocol/Aria/AriaRWKey.h" 13 | #include 14 | #include 15 | #include 16 | 17 | namespace aria { 18 | 19 | class AriaTransaction { 20 | 21 | public: 22 | using MetaDataType = std::atomic; 23 | 24 | AriaTransaction(std::size_t coordinator_id, std::size_t partition_id, 25 | Partitioner &partitioner) 26 | : coordinator_id(coordinator_id), partition_id(partition_id), 27 | startTime(std::chrono::steady_clock::now()), partitioner(partitioner) { 28 | reset(); 29 | } 30 | 31 | virtual ~AriaTransaction() = default; 32 | 33 | void reset() { 34 | abort_lock = false; 35 | abort_no_retry = false; 36 | abort_read_validation = false; 37 | distributed_transaction = false; 38 | execution_phase = false; 39 | waw = false; 40 | war = false; 41 | raw = false; 42 | pendingResponses = 0; 43 | network_size = 0; 44 | operation.clear(); 45 | readSet.clear(); 46 | writeSet.clear(); 47 | } 48 | 49 | virtual TransactionResult execute(std::size_t worker_id) = 0; 50 | 51 | virtual void reset_query() = 0; 52 | 53 | template 54 | void search_local_index(std::size_t table_id, std::size_t partition_id, 55 | const KeyType &key, ValueType &value) { 56 | if (execution_phase) { 57 | return; 58 | } 59 | 60 | AriaRWKey readKey; 61 | 62 | readKey.set_table_id(table_id); 63 | readKey.set_partition_id(partition_id); 64 | 65 | readKey.set_key(&key); 66 | readKey.set_value(&value); 67 | 68 | readKey.set_local_index_read_bit(); 69 | readKey.set_read_request_bit(); 70 | 71 | add_to_read_set(readKey); 72 | } 73 | 74 | template 75 | void search_for_read(std::size_t table_id, std::size_t partition_id, 76 | const KeyType &key, ValueType &value) { 77 | if (execution_phase) { 78 | return; 79 | } 80 | AriaRWKey readKey; 81 | 82 | readKey.set_table_id(table_id); 83 | readKey.set_partition_id(partition_id); 84 | 85 | readKey.set_key(&key); 86 | readKey.set_value(&value); 87 | 88 | readKey.set_read_request_bit(); 89 | 90 | add_to_read_set(readKey); 91 | } 92 | 93 | template 94 | void search_for_update(std::size_t table_id, std::size_t partition_id, 95 | const KeyType &key, ValueType &value) { 96 | if (execution_phase) { 97 | return; 98 | } 99 | AriaRWKey readKey; 100 | 101 | readKey.set_table_id(table_id); 102 | readKey.set_partition_id(partition_id); 103 | 104 | readKey.set_key(&key); 105 | readKey.set_value(&value); 106 | 107 | readKey.set_read_request_bit(); 108 | 109 | add_to_read_set(readKey); 110 | } 111 | 112 | template 113 | void update(std::size_t table_id, std::size_t partition_id, 114 | const KeyType &key, const ValueType &value) { 115 | if (execution_phase) { 116 | return; 117 | } 118 | AriaRWKey writeKey; 119 | 120 | writeKey.set_table_id(table_id); 121 | writeKey.set_partition_id(partition_id); 122 | 123 | writeKey.set_key(&key); 124 | // the object pointed by value will not be updated 125 | writeKey.set_value(const_cast(&value)); 126 | 127 | add_to_write_set(writeKey); 128 | } 129 | 130 | std::size_t add_to_read_set(const AriaRWKey &key) { 131 | readSet.push_back(key); 132 | return readSet.size() - 1; 133 | } 134 | 135 | std::size_t add_to_write_set(const AriaRWKey &key) { 136 | writeSet.push_back(key); 137 | return writeSet.size() - 1; 138 | } 139 | 140 | void set_id(std::size_t id) { this->id = id; } 141 | 142 | void set_tid_offset(std::size_t offset) { this->tid_offset = offset; } 143 | 144 | void set_epoch(uint32_t epoch) { this->epoch = epoch; } 145 | 146 | bool process_requests(std::size_t worker_id) { 147 | 148 | // cannot use unsigned type in reverse iteration 149 | for (int i = int(readSet.size()) - 1; i >= 0; i--) { 150 | // early return 151 | if (!readSet[i].get_read_request_bit()) { 152 | break; 153 | } 154 | 155 | AriaRWKey &readKey = readSet[i]; 156 | readRequestHandler(readKey, id, i); 157 | readSet[i].clear_read_request_bit(); 158 | } 159 | 160 | return false; 161 | } 162 | 163 | bool is_read_only() { return writeSet.size() == 0; } 164 | 165 | public: 166 | std::size_t coordinator_id, partition_id, id, tid_offset; 167 | uint32_t epoch; 168 | std::chrono::steady_clock::time_point startTime; 169 | std::size_t pendingResponses; 170 | std::size_t network_size; 171 | 172 | bool abort_lock, abort_no_retry, abort_read_validation; 173 | bool distributed_transaction; 174 | bool execution_phase; 175 | bool waw, war, raw; 176 | 177 | // read_key, id, key_offset 178 | std::function readRequestHandler; 179 | 180 | // processed a request? 181 | std::function remote_request_handler; 182 | 183 | std::function message_flusher; 184 | 185 | Partitioner &partitioner; 186 | Operation operation; // never used 187 | std::vector readSet, writeSet; 188 | }; 189 | } // namespace aria -------------------------------------------------------------------------------- /protocol/AriaFB/AriaFB.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Partitioner.h" 8 | #include "core/Table.h" 9 | #include "protocol/AriaFB/AriaFBHelper.h" 10 | #include "protocol/AriaFB/AriaFBMessage.h" 11 | #include "protocol/AriaFB/AriaFBTransaction.h" 12 | 13 | namespace aria { 14 | 15 | template class AriaFB { 16 | public: 17 | using DatabaseType = Database; 18 | using MetaDataType = std::atomic; 19 | using ContextType = typename DatabaseType::ContextType; 20 | using MessageType = AriaFBMessage; 21 | using TransactionType = AriaFBTransaction; 22 | 23 | using MessageFactoryType = AriaFBMessageFactory; 24 | using MessageHandlerType = AriaFBMessageHandler; 25 | 26 | AriaFB(DatabaseType &db, const ContextType &context, Partitioner &partitioner) 27 | : db(db), context(context), partitioner(partitioner) {} 28 | 29 | void abort(TransactionType &txn, 30 | std::vector> &messages) { 31 | txn.abort_lock = true; 32 | } 33 | 34 | bool commit(TransactionType &txn, 35 | std::vector> &messages) { 36 | 37 | auto &writeSet = txn.writeSet; 38 | for (auto i = 0u; i < writeSet.size(); i++) { 39 | auto &writeKey = writeSet[i]; 40 | auto tableId = writeKey.get_table_id(); 41 | auto partitionId = writeKey.get_partition_id(); 42 | auto table = db.find_table(tableId, partitionId); 43 | 44 | if (partitioner.has_master_partition(partitionId)) { 45 | auto key = writeKey.get_key(); 46 | auto value = writeKey.get_value(); 47 | table->update(key, value); 48 | } else { 49 | auto coordinatorID = partitioner.master_coordinator(partitionId); 50 | txn.network_size += MessageFactoryType::new_write_message( 51 | *messages[coordinatorID], *table, writeKey.get_key(), 52 | writeKey.get_value()); 53 | } 54 | } 55 | 56 | return true; 57 | } 58 | 59 | /* the following functions are for Calvin */ 60 | 61 | void calvin_abort(TransactionType &txn, std::size_t lock_manager_id, 62 | std::size_t n_lock_manager, 63 | std::size_t replica_group_size) { 64 | // release read locks 65 | calvin_release_read_locks(txn, lock_manager_id, n_lock_manager, 66 | replica_group_size); 67 | } 68 | 69 | bool calvin_commit(TransactionType &txn, std::size_t lock_manager_id, 70 | std::size_t n_lock_manager, 71 | std::size_t replica_group_size) { 72 | 73 | // write to db 74 | calvin_write(txn, lock_manager_id, n_lock_manager, replica_group_size); 75 | 76 | // release read/write locks 77 | calvin_release_read_locks(txn, lock_manager_id, n_lock_manager, 78 | replica_group_size); 79 | calvin_release_write_locks(txn, lock_manager_id, n_lock_manager, 80 | replica_group_size); 81 | 82 | return true; 83 | } 84 | 85 | void calvin_write(TransactionType &txn, std::size_t lock_manager_id, 86 | std::size_t n_lock_manager, 87 | std::size_t replica_group_size) { 88 | 89 | auto &writeSet = txn.writeSet; 90 | for (auto i = 0u; i < writeSet.size(); i++) { 91 | auto &writeKey = writeSet[i]; 92 | auto tableId = writeKey.get_table_id(); 93 | auto partitionId = writeKey.get_partition_id(); 94 | auto table = db.find_table(tableId, partitionId); 95 | 96 | if (!partitioner.has_master_partition(partitionId)) { 97 | continue; 98 | } 99 | 100 | if (AriaFBHelper::partition_id_to_lock_manager_id( 101 | writeKey.get_partition_id(), n_lock_manager, 102 | replica_group_size) != lock_manager_id) { 103 | continue; 104 | } 105 | 106 | auto key = writeKey.get_key(); 107 | auto value = writeKey.get_value(); 108 | table->update(key, value); 109 | } 110 | } 111 | 112 | void calvin_release_read_locks(TransactionType &txn, 113 | std::size_t lock_manager_id, 114 | std::size_t n_lock_manager, 115 | std::size_t replica_group_size) { 116 | // release read locks 117 | auto &readSet = txn.readSet; 118 | 119 | for (auto i = 0u; i < readSet.size(); i++) { 120 | auto &readKey = readSet[i]; 121 | auto tableId = readKey.get_table_id(); 122 | auto partitionId = readKey.get_partition_id(); 123 | auto table = db.find_table(tableId, partitionId); 124 | 125 | if (!partitioner.has_master_partition(partitionId)) { 126 | continue; 127 | } 128 | 129 | if (!readKey.get_read_lock_bit()) { 130 | continue; 131 | } 132 | 133 | if (AriaFBHelper::partition_id_to_lock_manager_id( 134 | readKey.get_partition_id(), n_lock_manager, replica_group_size) != 135 | lock_manager_id) { 136 | continue; 137 | } 138 | 139 | auto key = readKey.get_key(); 140 | auto value = readKey.get_value(); 141 | std::atomic &tid = table->search_metadata(key); 142 | AriaFBHelper::read_lock_release(tid); 143 | } 144 | } 145 | 146 | void calvin_release_write_locks(TransactionType &txn, 147 | std::size_t lock_manager_id, 148 | std::size_t n_lock_manager, 149 | std::size_t replica_group_size) { 150 | 151 | // release write lock 152 | auto &writeSet = txn.writeSet; 153 | 154 | for (auto i = 0u; i < writeSet.size(); i++) { 155 | auto &writeKey = writeSet[i]; 156 | auto tableId = writeKey.get_table_id(); 157 | auto partitionId = writeKey.get_partition_id(); 158 | auto table = db.find_table(tableId, partitionId); 159 | 160 | if (!partitioner.has_master_partition(partitionId)) { 161 | continue; 162 | } 163 | 164 | if (AriaFBHelper::partition_id_to_lock_manager_id( 165 | writeKey.get_partition_id(), n_lock_manager, 166 | replica_group_size) != lock_manager_id) { 167 | continue; 168 | } 169 | 170 | auto key = writeKey.get_key(); 171 | auto value = writeKey.get_value(); 172 | std::atomic &tid = table->search_metadata(key); 173 | AriaFBHelper::write_lock_release(tid); 174 | } 175 | } 176 | 177 | private: 178 | DatabaseType &db; 179 | const ContextType &context; 180 | Partitioner &partitioner; 181 | }; 182 | } // namespace aria -------------------------------------------------------------------------------- /protocol/AriaFB/AriaFBRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/7/19. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class AriaFBRWKey { 12 | public: 13 | // local index read bit 14 | 15 | void set_local_index_read_bit() { 16 | clear_local_index_read_bit(); 17 | bitvec |= LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET; 18 | } 19 | 20 | void clear_local_index_read_bit() { 21 | bitvec &= ~(LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET); 22 | } 23 | 24 | uint32_t get_local_index_read_bit() const { 25 | return (bitvec >> LOCAL_INDEX_READ_BIT_OFFSET) & LOCAL_INDEX_READ_BIT_MASK; 26 | } 27 | 28 | // read request bit 29 | 30 | void set_read_request_bit() { 31 | clear_read_request_bit(); 32 | bitvec |= READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET; 33 | } 34 | 35 | void clear_read_request_bit() { 36 | bitvec &= ~(READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET); 37 | } 38 | 39 | uint32_t get_read_request_bit() const { 40 | return (bitvec >> READ_REQUEST_BIT_OFFSET) & READ_REQUEST_BIT_MASK; 41 | } 42 | 43 | // read lock bit 44 | 45 | void set_read_lock_bit() { 46 | clear_read_lock_bit(); 47 | bitvec |= READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET; 48 | } 49 | 50 | void clear_read_lock_bit() { 51 | bitvec &= ~(READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 52 | } 53 | 54 | uint32_t get_read_lock_bit() const { 55 | return (bitvec >> READ_LOCK_BIT_OFFSET) & READ_LOCK_BIT_MASK; 56 | } 57 | 58 | // write lock bit 59 | 60 | void set_write_lock_bit() { 61 | clear_write_lock_bit(); 62 | bitvec |= WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET; 63 | } 64 | 65 | void clear_write_lock_bit() { 66 | bitvec &= ~(WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 67 | } 68 | 69 | uint32_t get_write_lock_bit() const { 70 | return (bitvec >> WRITE_LOCK_BIT_OFFSET) & WRITE_LOCK_BIT_MASK; 71 | } 72 | 73 | // prepare processed bit 74 | 75 | void set_prepare_processed_bit() { 76 | clear_prepare_processed_bit(); 77 | bitvec |= PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET; 78 | } 79 | 80 | void clear_prepare_processed_bit() { 81 | bitvec &= ~(PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET); 82 | } 83 | 84 | uint32_t get_prepare_processed_bit() const { 85 | return (bitvec >> PREPARE_PROCESSED_BIT_OFFSET) & 86 | PREPARE_PROCESSED_BIT_MASK; 87 | } 88 | 89 | // execution processed bit 90 | 91 | void set_execution_processed_bit() { 92 | clear_execution_processed_bit(); 93 | bitvec |= EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET; 94 | } 95 | 96 | void clear_execution_processed_bit() { 97 | bitvec &= ~(EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET); 98 | } 99 | 100 | uint32_t get_execution_processed_bit() const { 101 | return (bitvec >> EXECUTION_PROCESSED_BIT_OFFSET) & 102 | EXECUTION_PROCESSED_BIT_MASK; 103 | } 104 | 105 | // table id 106 | 107 | void set_table_id(uint32_t table_id) { 108 | DCHECK(table_id < (1 << 5)); 109 | clear_table_id(); 110 | bitvec |= table_id << TABLE_ID_OFFSET; 111 | } 112 | 113 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 114 | 115 | uint32_t get_table_id() const { 116 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 117 | } 118 | // partition id 119 | 120 | void set_partition_id(uint32_t partition_id) { 121 | DCHECK(partition_id < (1 << 16)); 122 | clear_partition_id(); 123 | bitvec |= partition_id << PARTITION_ID_OFFSET; 124 | } 125 | 126 | void clear_partition_id() { 127 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 128 | } 129 | 130 | uint32_t get_partition_id() const { 131 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 132 | } 133 | 134 | // key 135 | void set_key(const void *key) { this->key = key; } 136 | 137 | const void *get_key() const { return key; } 138 | 139 | // value 140 | void set_value(void *value) { this->value = value; } 141 | 142 | void *get_value() const { return value; } 143 | 144 | void set_tid(std::atomic *tid) { this->tid = tid; }; 145 | 146 | std::atomic *get_tid() const { return tid; }; 147 | 148 | private: 149 | /* 150 | * A bitvec is a 32-bit word. 151 | * 152 | * [ table id (5) ] | partition id (16) | unused bit (5) | 153 | * prepare processed bit (1) | execute processed bit(1) | 154 | * write lock bit(1) | read lock bit (1) | 155 | * read request bit (1) | local index read (1) ] 156 | * 157 | * local index read is set when the read is from a local read only index. 158 | * read request bit is set when there is a read request. 159 | * write lock bit is set when a write lock is acquired. 160 | * read lock bit is set when a read lock is acquired. 161 | * prepare processed bit is set when process_request has processed this key in 162 | * prepare phase execution processed bit is set when process_request has 163 | * processed this key in execution phase 164 | */ 165 | 166 | uint32_t bitvec = 0; 167 | const void *key = nullptr; 168 | void *value = nullptr; 169 | std::atomic *tid = nullptr; 170 | 171 | public: 172 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 173 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 174 | 175 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 176 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 177 | 178 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_MASK = 0x1; 179 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_OFFSET = 5; 180 | 181 | static constexpr uint32_t PREPARE_PROCESSED_BIT_MASK = 0x1; 182 | static constexpr uint32_t PREPARE_PROCESSED_BIT_OFFSET = 4; 183 | 184 | static constexpr uint32_t WRITE_LOCK_BIT_MASK = 0x1; 185 | static constexpr uint32_t WRITE_LOCK_BIT_OFFSET = 3; 186 | 187 | static constexpr uint32_t READ_LOCK_BIT_MASK = 0x1; 188 | static constexpr uint32_t READ_LOCK_BIT_OFFSET = 2; 189 | 190 | static constexpr uint32_t READ_REQUEST_BIT_MASK = 0x1; 191 | static constexpr uint32_t READ_REQUEST_BIT_OFFSET = 1; 192 | 193 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_MASK = 0x1; 194 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_OFFSET = 0; 195 | }; 196 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Bohm/Bohm.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Table.h" 8 | #include "protocol/Bohm/BohmHelper.h" 9 | #include "protocol/Bohm/BohmMessage.h" 10 | #include "protocol/Bohm/BohmPartitioner.h" 11 | #include "protocol/Bohm/BohmTransaction.h" 12 | 13 | namespace aria { 14 | 15 | template class Bohm { 16 | public: 17 | using DatabaseType = Database; 18 | using MetaDataType = std::atomic; 19 | using ContextType = typename DatabaseType::ContextType; 20 | using MessageType = BohmMessage; 21 | using TransactionType = BohmTransaction; 22 | 23 | using MessageFactoryType = BohmMessageFactory; 24 | using MessageHandlerType = BohmMessageHandler; 25 | 26 | Bohm(DatabaseType &db, BohmPartitioner &partitioner) 27 | : db(db), partitioner(partitioner) {} 28 | 29 | void abort(TransactionType &txn, 30 | std::vector> &messages) { 31 | txn.load_read_count(); 32 | txn.clear_execution_bit(); 33 | txn.abort_read_not_ready = false; 34 | } 35 | 36 | void commit(TransactionType &txn, 37 | std::vector> &messages) { 38 | auto &writeSet = txn.writeSet; 39 | for (auto i = 0u; i < writeSet.size(); i++) { 40 | auto &writeKey = writeSet[i]; 41 | auto tableId = writeKey.get_table_id(); 42 | auto partitionId = writeKey.get_partition_id(); 43 | auto table = db.find_table(tableId, partitionId); 44 | auto key = writeKey.get_key(); 45 | auto value = writeKey.get_value(); 46 | if (partitioner.has_master_partition(partitionId)) { 47 | std::atomic &placeholder = 48 | table->search_metadata(key, txn.id); 49 | CHECK(BohmHelper::is_placeholder_ready(placeholder) == false); 50 | table->update(key, value, txn.id); 51 | BohmHelper::set_placeholder_to_ready(placeholder); 52 | } else { 53 | auto coordinatorID = partitioner.master_coordinator(partitionId); 54 | txn.network_size += MessageFactoryType::new_write_message( 55 | *messages[coordinatorID], *table, txn.id, key, value); 56 | } 57 | } 58 | } 59 | 60 | private: 61 | DatabaseType &db; 62 | BohmPartitioner &partitioner; 63 | }; 64 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Bohm/BohmHelper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | namespace aria { 14 | 15 | class BohmHelper { 16 | public: 17 | using MetaDataType = std::atomic; 18 | 19 | static void read(const std::tuple &row, void *dest, 20 | std::size_t size) { 21 | // placeholder contains the TID of the transaction that will update the 22 | MetaDataType &placeholder = *std::get<0>(row); 23 | DCHECK(is_placeholder_ready(placeholder)) 24 | << "placeholder not ready to read."; 25 | void *src = std::get<1>(row); 26 | std::memcpy(dest, src, size); 27 | } 28 | 29 | // value 0 meaning the value is ready to read 30 | static bool is_placeholder_ready(std::atomic &a) { 31 | return a.load() == 0; 32 | } 33 | 34 | static void set_placeholder_to_ready(std::atomic &a) { a.store(0); } 35 | 36 | // assume the replication group size is 3 and we have partitions 0..8 37 | // the 1st coordinator has partition 0, 3, 6. 38 | // the 2nd coordinator has partition 1, 4, 7. 39 | // the 3rd coordinator has partition 2, 5, 8. 40 | // the function first maps all partition id to 0, 1, 2 and then use % hash to 41 | // assign each partition to a lock manager. 42 | 43 | static std::size_t partition_id_to_worker_id(std::size_t partition_id, 44 | std::size_t n_worker, 45 | std::size_t n_coordinator) { 46 | return partition_id / n_coordinator % n_worker; 47 | } 48 | 49 | public: 50 | static uint64_t get_epoch(uint64_t value) { 51 | return (value >> EPOCH_OFFSET) & EPOCH_MASK; 52 | } 53 | 54 | static uint64_t set_epoch(uint64_t value, uint64_t epoch) { 55 | DCHECK(epoch < (1ull << 32)); 56 | return (value & (~(EPOCH_MASK << EPOCH_OFFSET))) | (epoch << EPOCH_OFFSET); 57 | } 58 | 59 | static uint64_t get_pos(uint64_t value) { 60 | return (value >> POS_OFFSET) & POS_MASK; 61 | } 62 | 63 | static uint64_t set_pos(uint64_t value, uint64_t pos) { 64 | DCHECK(pos < (1ull << 32)); 65 | return (value & (~(POS_MASK << POS_OFFSET))) | (pos << POS_OFFSET); 66 | } 67 | 68 | static uint64_t get_tid(uint64_t epoch, uint64_t pos) { 69 | DCHECK(epoch < (1ull << 32)); 70 | DCHECK(pos < (1ull << 32)); 71 | return (epoch << EPOCH_OFFSET) | pos; 72 | } 73 | 74 | public: 75 | /* 76 | * [ epoch (32) | pos (32) ] 77 | * 78 | */ 79 | 80 | static constexpr int EPOCH_OFFSET = 32; 81 | static constexpr uint64_t EPOCH_MASK = 0xfffffffffffull; 82 | 83 | static constexpr int POS_OFFSET = 0; 84 | static constexpr uint64_t POS_MASK = 0xffffffull; 85 | }; 86 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Bohm/BohmManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Manager.h" 8 | #include "protocol/Bohm/Bohm.h" 9 | #include "protocol/Bohm/BohmExecutor.h" 10 | #include "protocol/Bohm/BohmHelper.h" 11 | #include "protocol/Bohm/BohmPartitioner.h" 12 | #include "protocol/Bohm/BohmTransaction.h" 13 | 14 | #include 15 | #include 16 | 17 | namespace aria { 18 | 19 | template class BohmManager : public aria::Manager { 20 | public: 21 | using base_type = aria::Manager; 22 | 23 | using WorkloadType = Workload; 24 | using DatabaseType = typename WorkloadType::DatabaseType; 25 | using StorageType = typename WorkloadType::StorageType; 26 | 27 | using TransactionType = BohmTransaction; 28 | using ContextType = typename DatabaseType::ContextType; 29 | using RandomType = typename DatabaseType::RandomType; 30 | 31 | BohmManager(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 32 | const ContextType &context, std::atomic &stopFlag) 33 | : base_type(coordinator_id, id, context, stopFlag), db(db), epoch(0), 34 | partitioner(coordinator_id, context.coordinator_num) { 35 | 36 | storages.resize(context.batch_size); 37 | transactions.resize(context.batch_size); 38 | } 39 | 40 | void coordinator_start() override { 41 | 42 | std::size_t n_workers = context.worker_num; 43 | std::size_t n_coordinators = context.coordinator_num; 44 | 45 | while (!stopFlag.load()) { 46 | 47 | // the coordinator on each machine generates 48 | // a batch of transactions using the same random seed. 49 | 50 | epoch.fetch_add(1); 51 | 52 | // LOG(INFO) << "Seed: " << random.get_seed(); 53 | n_started_workers.store(0); 54 | n_completed_workers.store(0); 55 | signal_worker(ExecutorStatus::Bohm_Analysis); 56 | // Allow each worker to analyse the read/write set 57 | // each worker analyse i, i + n, i + 2n transaction 58 | wait_all_workers_start(); 59 | wait_all_workers_finish(); 60 | // wait for all machines until they finish the analysis phase. 61 | wait4_ack(); 62 | 63 | // Allow each worker to insert write sets 64 | n_started_workers.store(0); 65 | n_completed_workers.store(0); 66 | signal_worker(ExecutorStatus::Bohm_Insert); 67 | wait_all_workers_start(); 68 | wait_all_workers_finish(); 69 | // wait for all machines until they finish the execution phase. 70 | wait4_ack(); 71 | 72 | // Allow each worker to run transactions 73 | n_started_workers.store(0); 74 | n_completed_workers.store(0); 75 | signal_worker(ExecutorStatus::Bohm_Execute); 76 | wait_all_workers_start(); 77 | wait_all_workers_finish(); 78 | // wait for all machines until they finish the execution phase. 79 | wait4_ack(); 80 | 81 | // Allow each worker to garbage collect 82 | n_started_workers.store(0); 83 | n_completed_workers.store(0); 84 | signal_worker(ExecutorStatus::Bohm_GC); 85 | wait_all_workers_start(); 86 | wait_all_workers_finish(); 87 | // wait for all machines until they finish the execution phase. 88 | wait4_ack(); 89 | } 90 | 91 | signal_worker(ExecutorStatus::EXIT); 92 | } 93 | 94 | void non_coordinator_start() override { 95 | 96 | std::size_t n_workers = context.worker_num; 97 | std::size_t n_coordinators = context.coordinator_num; 98 | 99 | for (;;) { 100 | // LOG(INFO) << "Seed: " << random.get_seed(); 101 | ExecutorStatus status = wait4_signal(); 102 | if (status == ExecutorStatus::EXIT) { 103 | set_worker_status(ExecutorStatus::EXIT); 104 | break; 105 | } 106 | 107 | DCHECK(status == ExecutorStatus::Bohm_Analysis); 108 | // the coordinator on each machine generates 109 | // a batch of transactions using the same random seed. 110 | // Allow each worker to analyse the read/write set 111 | // each worker analyse i, i + n, i + 2n transaction 112 | 113 | epoch.fetch_add(1); 114 | 115 | n_started_workers.store(0); 116 | n_completed_workers.store(0); 117 | set_worker_status(ExecutorStatus::Bohm_Analysis); 118 | wait_all_workers_start(); 119 | wait_all_workers_finish(); 120 | send_ack(); 121 | 122 | status = wait4_signal(); 123 | DCHECK(status == ExecutorStatus::Bohm_Insert); 124 | // Allow each worker to run transactions 125 | n_started_workers.store(0); 126 | n_completed_workers.store(0); 127 | set_worker_status(ExecutorStatus::Bohm_Insert); 128 | wait_all_workers_start(); 129 | wait_all_workers_finish(); 130 | send_ack(); 131 | 132 | status = wait4_signal(); 133 | DCHECK(status == ExecutorStatus::Bohm_Execute); 134 | // Allow each worker to run transactions 135 | n_started_workers.store(0); 136 | n_completed_workers.store(0); 137 | set_worker_status(ExecutorStatus::Bohm_Execute); 138 | wait_all_workers_start(); 139 | wait_all_workers_finish(); 140 | send_ack(); 141 | 142 | status = wait4_signal(); 143 | DCHECK(status == ExecutorStatus::Bohm_GC); 144 | // Allow each worker to garbage collect 145 | n_started_workers.store(0); 146 | n_completed_workers.store(0); 147 | set_worker_status(ExecutorStatus::Bohm_GC); 148 | wait_all_workers_start(); 149 | wait_all_workers_finish(); 150 | send_ack(); 151 | } 152 | } 153 | 154 | public: 155 | RandomType random; 156 | DatabaseType &db; 157 | std::atomic epoch; 158 | BohmPartitioner partitioner; 159 | std::vector storages; 160 | std::vector> transactions; 161 | }; 162 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Bohm/BohmPartitioner.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Partitioner.h" 8 | 9 | namespace aria { 10 | class BohmPartitioner : public Partitioner { 11 | 12 | public: 13 | BohmPartitioner(std::size_t coordinator_id, std::size_t coordinator_num) 14 | : Partitioner(coordinator_id, coordinator_num) { 15 | CHECK(coordinator_id < coordinator_num); 16 | } 17 | 18 | ~BohmPartitioner() override = default; 19 | 20 | std::size_t replica_num() const override { return 1; } 21 | 22 | bool is_replicated() const override { return false; } 23 | 24 | bool has_master_partition(std::size_t partition_id) const override { 25 | return master_coordinator(partition_id) == coordinator_id; 26 | } 27 | 28 | std::size_t master_coordinator(std::size_t partition_id) const override { 29 | return partition_id % coordinator_num; 30 | } 31 | 32 | bool is_partition_replicated_on(std::size_t partition_id, 33 | std::size_t coordinator_id) const override { 34 | return false; 35 | } 36 | 37 | bool is_backup() const override { return false; } 38 | }; 39 | 40 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Bohm/BohmRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class BohmRWKey { 12 | public: 13 | // local index read bit 14 | 15 | void set_local_index_read_bit() { 16 | clear_local_index_read_bit(); 17 | bitvec |= LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET; 18 | } 19 | 20 | void clear_local_index_read_bit() { 21 | bitvec &= ~(LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET); 22 | } 23 | 24 | uint32_t get_local_index_read_bit() const { 25 | return (bitvec >> LOCAL_INDEX_READ_BIT_OFFSET) & LOCAL_INDEX_READ_BIT_MASK; 26 | } 27 | 28 | // read request bit 29 | 30 | void set_read_request_bit() { 31 | clear_read_request_bit(); 32 | bitvec |= READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET; 33 | } 34 | 35 | void clear_read_request_bit() { 36 | bitvec &= ~(READ_REQUEST_BIT_MASK << READ_REQUEST_BIT_OFFSET); 37 | } 38 | 39 | uint32_t get_read_request_bit() const { 40 | return (bitvec >> READ_REQUEST_BIT_OFFSET) & READ_REQUEST_BIT_MASK; 41 | } 42 | 43 | // prepare processed bit 44 | 45 | void set_prepare_processed_bit() { 46 | clear_prepare_processed_bit(); 47 | bitvec |= PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET; 48 | } 49 | 50 | void clear_prepare_processed_bit() { 51 | bitvec &= ~(PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET); 52 | } 53 | 54 | uint32_t get_prepare_processed_bit() const { 55 | return (bitvec >> PREPARE_PROCESSED_BIT_OFFSET) & 56 | PREPARE_PROCESSED_BIT_MASK; 57 | } 58 | 59 | // execution processed bit 60 | 61 | void set_execution_processed_bit() { 62 | clear_execution_processed_bit(); 63 | bitvec |= EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET; 64 | } 65 | 66 | void clear_execution_processed_bit() { 67 | bitvec &= ~(EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET); 68 | } 69 | 70 | uint32_t get_execution_processed_bit() const { 71 | return (bitvec >> EXECUTION_PROCESSED_BIT_OFFSET) & 72 | EXECUTION_PROCESSED_BIT_MASK; 73 | } 74 | 75 | // table id 76 | 77 | void set_table_id(uint32_t table_id) { 78 | DCHECK(table_id < (1 << 5)); 79 | clear_table_id(); 80 | bitvec |= table_id << TABLE_ID_OFFSET; 81 | } 82 | 83 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 84 | 85 | uint32_t get_table_id() const { 86 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 87 | } 88 | // partition id 89 | 90 | void set_partition_id(uint32_t partition_id) { 91 | DCHECK(partition_id < (1 << 16)); 92 | clear_partition_id(); 93 | bitvec |= partition_id << PARTITION_ID_OFFSET; 94 | } 95 | 96 | void clear_partition_id() { 97 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 98 | } 99 | 100 | uint32_t get_partition_id() const { 101 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 102 | } 103 | 104 | // key 105 | void set_key(const void *key) { this->key = key; } 106 | 107 | const void *get_key() const { return key; } 108 | 109 | // value 110 | void set_value(void *value) { this->value = value; } 111 | 112 | void *get_value() const { return value; } 113 | 114 | void set_tid(std::atomic *tid) { this->tid = tid; }; 115 | 116 | std::atomic *get_tid() const { return tid; }; 117 | 118 | private: 119 | /* 120 | * A bitvec is a 32-bit word. 121 | * 122 | * [ table id (5) ] | partition id (16) | unused bit (7) | 123 | * prepare processed bit (1) | execute processed bit(1) | 124 | * read request bit (1) | local index read (1) ] 125 | * 126 | * local index read is set when the read is from a local read only index. 127 | */ 128 | 129 | uint32_t bitvec = 0; 130 | const void *key = nullptr; 131 | void *value = nullptr; 132 | std::atomic *tid = nullptr; 133 | 134 | public: 135 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 136 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 137 | 138 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 139 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 140 | 141 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_MASK = 0x1; 142 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_OFFSET = 3; 143 | 144 | static constexpr uint32_t PREPARE_PROCESSED_BIT_MASK = 0x1; 145 | static constexpr uint32_t PREPARE_PROCESSED_BIT_OFFSET = 2; 146 | 147 | static constexpr uint32_t READ_REQUEST_BIT_MASK = 0x1; 148 | static constexpr uint32_t READ_REQUEST_BIT_OFFSET = 1; 149 | 150 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_MASK = 0x1; 151 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_OFFSET = 0; 152 | }; 153 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Calvin/Calvin.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Table.h" 8 | #include "protocol/Calvin/CalvinHelper.h" 9 | #include "protocol/Calvin/CalvinMessage.h" 10 | #include "protocol/Calvin/CalvinPartitioner.h" 11 | #include "protocol/Calvin/CalvinTransaction.h" 12 | 13 | namespace aria { 14 | 15 | template class Calvin { 16 | public: 17 | using DatabaseType = Database; 18 | using MetaDataType = std::atomic; 19 | using ContextType = typename DatabaseType::ContextType; 20 | using MessageType = CalvinMessage; 21 | using TransactionType = CalvinTransaction; 22 | 23 | using MessageFactoryType = CalvinMessageFactory; 24 | using MessageHandlerType = CalvinMessageHandler; 25 | 26 | Calvin(DatabaseType &db, CalvinPartitioner &partitioner) 27 | : db(db), partitioner(partitioner) {} 28 | 29 | void abort(TransactionType &txn, std::size_t lock_manager_id, 30 | std::size_t n_lock_manager, std::size_t replica_group_size) { 31 | // release read locks 32 | release_read_locks(txn, lock_manager_id, n_lock_manager, 33 | replica_group_size); 34 | } 35 | 36 | bool commit(TransactionType &txn, std::size_t lock_manager_id, 37 | std::size_t n_lock_manager, std::size_t replica_group_size) { 38 | 39 | // write to db 40 | write(txn, lock_manager_id, n_lock_manager, replica_group_size); 41 | 42 | // release read/write locks 43 | release_read_locks(txn, lock_manager_id, n_lock_manager, 44 | replica_group_size); 45 | release_write_locks(txn, lock_manager_id, n_lock_manager, 46 | replica_group_size); 47 | 48 | return true; 49 | } 50 | 51 | void write(TransactionType &txn, std::size_t lock_manager_id, 52 | std::size_t n_lock_manager, std::size_t replica_group_size) { 53 | 54 | auto &writeSet = txn.writeSet; 55 | for (auto i = 0u; i < writeSet.size(); i++) { 56 | auto &writeKey = writeSet[i]; 57 | auto tableId = writeKey.get_table_id(); 58 | auto partitionId = writeKey.get_partition_id(); 59 | auto table = db.find_table(tableId, partitionId); 60 | 61 | if (!partitioner.has_master_partition(partitionId)) { 62 | continue; 63 | } 64 | 65 | if (CalvinHelper::partition_id_to_lock_manager_id( 66 | writeKey.get_partition_id(), n_lock_manager, 67 | replica_group_size) != lock_manager_id) { 68 | continue; 69 | } 70 | 71 | auto key = writeKey.get_key(); 72 | auto value = writeKey.get_value(); 73 | table->update(key, value); 74 | } 75 | } 76 | 77 | void release_read_locks(TransactionType &txn, std::size_t lock_manager_id, 78 | std::size_t n_lock_manager, 79 | std::size_t replica_group_size) { 80 | // release read locks 81 | auto &readSet = txn.readSet; 82 | 83 | for (auto i = 0u; i < readSet.size(); i++) { 84 | auto &readKey = readSet[i]; 85 | auto tableId = readKey.get_table_id(); 86 | auto partitionId = readKey.get_partition_id(); 87 | auto table = db.find_table(tableId, partitionId); 88 | 89 | if (!partitioner.has_master_partition(partitionId)) { 90 | continue; 91 | } 92 | 93 | if (!readKey.get_read_lock_bit()) { 94 | continue; 95 | } 96 | 97 | if (CalvinHelper::partition_id_to_lock_manager_id( 98 | readKey.get_partition_id(), n_lock_manager, replica_group_size) != 99 | lock_manager_id) { 100 | continue; 101 | } 102 | 103 | auto key = readKey.get_key(); 104 | auto value = readKey.get_value(); 105 | std::atomic &tid = table->search_metadata(key); 106 | CalvinHelper::read_lock_release(tid); 107 | } 108 | } 109 | 110 | void release_write_locks(TransactionType &txn, std::size_t lock_manager_id, 111 | std::size_t n_lock_manager, 112 | std::size_t replica_group_size) { 113 | 114 | // release write lock 115 | auto &writeSet = txn.writeSet; 116 | 117 | for (auto i = 0u; i < writeSet.size(); i++) { 118 | auto &writeKey = writeSet[i]; 119 | auto tableId = writeKey.get_table_id(); 120 | auto partitionId = writeKey.get_partition_id(); 121 | auto table = db.find_table(tableId, partitionId); 122 | 123 | if (!partitioner.has_master_partition(partitionId)) { 124 | continue; 125 | } 126 | 127 | if (CalvinHelper::partition_id_to_lock_manager_id( 128 | writeKey.get_partition_id(), n_lock_manager, 129 | replica_group_size) != lock_manager_id) { 130 | continue; 131 | } 132 | 133 | auto key = writeKey.get_key(); 134 | auto value = writeKey.get_value(); 135 | std::atomic &tid = table->search_metadata(key); 136 | CalvinHelper::write_lock_release(tid); 137 | } 138 | } 139 | 140 | private: 141 | DatabaseType &db; 142 | CalvinPartitioner &partitioner; 143 | }; 144 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Calvin/CalvinHelper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/15/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | namespace aria { 14 | 15 | class CalvinHelper { 16 | 17 | public: 18 | using MetaDataType = std::atomic; 19 | 20 | static std::vector string_to_vint(const std::string &str) { 21 | std::vector vstr; 22 | boost::algorithm::split(vstr, str, boost::is_any_of(",")); 23 | std::vector vint; 24 | for (auto i = 0u; i < vstr.size(); i++) { 25 | vint.push_back(std::atoi(vstr[i].c_str())); 26 | } 27 | return vint; 28 | } 29 | 30 | static std::size_t 31 | n_lock_manager(std::size_t replica_group_id, std::size_t id, 32 | const std::vector &lock_managers) { 33 | CHECK(replica_group_id < lock_managers.size()); 34 | return lock_managers[replica_group_id]; 35 | } 36 | 37 | // assume there are n = 2 lock managers and m = 4 workers 38 | // the following function maps 39 | // (2, 2, 4) => 0 40 | // (3, 2, 4) => 0 41 | // (4, 2, 4) => 1 42 | // (5, 2, 4) => 1 43 | 44 | static std::size_t worker_id_to_lock_manager_id(std::size_t id, 45 | std::size_t n_lock_manager, 46 | std::size_t n_worker) { 47 | if (id < n_lock_manager) { 48 | return id; 49 | } 50 | return (id - n_lock_manager) / (n_worker / n_lock_manager); 51 | } 52 | 53 | // assume the replication group size is 3 and we have partitions 0..8 54 | // the 1st coordinator has partition 0, 3, 6. 55 | // the 2nd coordinator has partition 1, 4, 7. 56 | // the 3rd coordinator has partition 2, 5, 8. 57 | // the function first maps all partition id to 0, 1, 2 and then use % hash to 58 | // assign each partition to a lock manager. 59 | 60 | static std::size_t 61 | partition_id_to_lock_manager_id(std::size_t partition_id, 62 | std::size_t n_lock_manager, 63 | std::size_t replica_group_size) { 64 | return partition_id / replica_group_size % n_lock_manager; 65 | } 66 | 67 | static void read(const std::tuple &row, void *dest, 68 | std::size_t size) { 69 | 70 | MetaDataType &tid = *std::get<0>(row); 71 | void *src = std::get<1>(row); 72 | std::memcpy(dest, src, size); 73 | } 74 | 75 | /** 76 | * 77 | * The following code is adapted from TwoPLHelper.h 78 | * For Calvin, we can use lower 63 bits for read locks. 79 | * However, 511 locks are enough and the code above is well tested. 80 | * 81 | * [write lock bit (1) | read lock bit (9) -- 512 - 1 locks ] 82 | * 83 | */ 84 | 85 | static bool is_read_locked(uint64_t value) { 86 | return value & (READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 87 | } 88 | 89 | static bool is_write_locked(uint64_t value) { 90 | return value & (WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 91 | } 92 | 93 | static uint64_t read_lock_num(uint64_t value) { 94 | return (value >> READ_LOCK_BIT_OFFSET) & READ_LOCK_BIT_MASK; 95 | } 96 | 97 | static uint64_t read_lock_max() { return READ_LOCK_BIT_MASK; } 98 | 99 | static uint64_t read_lock(std::atomic &a) { 100 | uint64_t old_value, new_value; 101 | do { 102 | do { 103 | old_value = a.load(); 104 | } while (is_write_locked(old_value) || 105 | read_lock_num(old_value) == read_lock_max()); 106 | new_value = old_value + (1ull << READ_LOCK_BIT_OFFSET); 107 | } while (!a.compare_exchange_weak(old_value, new_value)); 108 | return remove_lock_bit(old_value); 109 | } 110 | 111 | static uint64_t write_lock(std::atomic &a) { 112 | uint64_t old_value, new_value; 113 | 114 | do { 115 | do { 116 | old_value = a.load(); 117 | } while (is_read_locked(old_value) || is_write_locked(old_value)); 118 | 119 | new_value = old_value + (WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 120 | 121 | } while (!a.compare_exchange_weak(old_value, new_value)); 122 | return remove_lock_bit(old_value); 123 | } 124 | 125 | static void read_lock_release(std::atomic &a) { 126 | uint64_t old_value, new_value; 127 | do { 128 | old_value = a.load(); 129 | DCHECK(is_read_locked(old_value)); 130 | DCHECK(!is_write_locked(old_value)); 131 | new_value = old_value - (1ull << READ_LOCK_BIT_OFFSET); 132 | } while (!a.compare_exchange_weak(old_value, new_value)); 133 | } 134 | 135 | static void write_lock_release(std::atomic &a) { 136 | uint64_t old_value, new_value; 137 | old_value = a.load(); 138 | DCHECK(!is_read_locked(old_value)); 139 | DCHECK(is_write_locked(old_value)); 140 | new_value = old_value - (1ull << WRITE_LOCK_BIT_OFFSET); 141 | bool ok = a.compare_exchange_strong(old_value, new_value); 142 | DCHECK(ok); 143 | } 144 | 145 | static uint64_t remove_lock_bit(uint64_t value) { 146 | return value & ~(LOCK_BIT_MASK << LOCK_BIT_OFFSET); 147 | } 148 | 149 | static uint64_t remove_read_lock_bit(uint64_t value) { 150 | return value & ~(READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 151 | } 152 | 153 | static uint64_t remove_write_lock_bit(uint64_t value) { 154 | return value & ~(WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 155 | } 156 | 157 | public: 158 | static constexpr int LOCK_BIT_OFFSET = 54; 159 | static constexpr uint64_t LOCK_BIT_MASK = 0x3ffull; 160 | 161 | static constexpr int READ_LOCK_BIT_OFFSET = 54; 162 | static constexpr uint64_t READ_LOCK_BIT_MASK = 0x1ffull; 163 | 164 | static constexpr int WRITE_LOCK_BIT_OFFSET = 63; 165 | static constexpr uint64_t WRITE_LOCK_BIT_MASK = 0x1ull; 166 | }; 167 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Calvin/CalvinManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/13/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Manager.h" 8 | #include "protocol/Calvin/Calvin.h" 9 | #include "protocol/Calvin/CalvinExecutor.h" 10 | #include "protocol/Calvin/CalvinHelper.h" 11 | #include "protocol/Calvin/CalvinPartitioner.h" 12 | #include "protocol/Calvin/CalvinTransaction.h" 13 | 14 | #include 15 | #include 16 | 17 | namespace aria { 18 | 19 | template class CalvinManager : public aria::Manager { 20 | public: 21 | using base_type = aria::Manager; 22 | 23 | using WorkloadType = Workload; 24 | using DatabaseType = typename WorkloadType::DatabaseType; 25 | using StorageType = typename WorkloadType::StorageType; 26 | 27 | using TransactionType = CalvinTransaction; 28 | static_assert(std::is_same::value, 30 | "Transaction types do not match."); 31 | using ContextType = typename DatabaseType::ContextType; 32 | using RandomType = typename DatabaseType::RandomType; 33 | 34 | CalvinManager(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 35 | const ContextType &context, std::atomic &stopFlag) 36 | : base_type(coordinator_id, id, context, stopFlag), db(db), 37 | partitioner(coordinator_id, context.coordinator_num, 38 | CalvinHelper::string_to_vint(context.replica_group)) { 39 | 40 | storages.resize(context.batch_size); 41 | transactions.resize(context.batch_size); 42 | } 43 | 44 | void coordinator_start() override { 45 | 46 | std::size_t n_workers = context.worker_num; 47 | std::size_t n_coordinators = context.coordinator_num; 48 | 49 | while (!stopFlag.load()) { 50 | 51 | // the coordinator on each machine generates 52 | // a batch of transactions using the same random seed. 53 | 54 | // LOG(INFO) << "Seed: " << random.get_seed(); 55 | n_started_workers.store(0); 56 | n_completed_workers.store(0); 57 | signal_worker(ExecutorStatus::Analysis); 58 | // Allow each worker to analyse the read/write set 59 | // each worker analyse i, i + n, i + 2n transaction 60 | wait_all_workers_start(); 61 | wait_all_workers_finish(); 62 | 63 | // wait for all machines until they finish the analysis phase. 64 | wait4_ack(); 65 | 66 | // Allow each worker to run transactions 67 | // DB is partitioned by the number of lock managers. 68 | // The first k workers act as lock managers to grant locks to other 69 | // workers The remaining workers run transactions upon assignment via the 70 | // queue. 71 | n_started_workers.store(0); 72 | n_completed_workers.store(0); 73 | clear_lock_manager_status(); 74 | signal_worker(ExecutorStatus::Execute); 75 | wait_all_workers_start(); 76 | wait_all_workers_finish(); 77 | // wait for all machines until they finish the execution phase. 78 | wait4_ack(); 79 | } 80 | 81 | signal_worker(ExecutorStatus::EXIT); 82 | } 83 | 84 | void non_coordinator_start() override { 85 | 86 | std::size_t n_workers = context.worker_num; 87 | std::size_t n_coordinators = context.coordinator_num; 88 | 89 | for (;;) { 90 | // LOG(INFO) << "Seed: " << random.get_seed(); 91 | ExecutorStatus status = wait4_signal(); 92 | if (status == ExecutorStatus::EXIT) { 93 | set_worker_status(ExecutorStatus::EXIT); 94 | break; 95 | } 96 | 97 | DCHECK(status == ExecutorStatus::Analysis); 98 | // the coordinator on each machine generates 99 | // a batch of transactions using the same random seed. 100 | // Allow each worker to analyse the read/write set 101 | // each worker analyse i, i + n, i + 2n transaction 102 | 103 | n_started_workers.store(0); 104 | n_completed_workers.store(0); 105 | set_worker_status(ExecutorStatus::Analysis); 106 | wait_all_workers_start(); 107 | wait_all_workers_finish(); 108 | 109 | send_ack(); 110 | 111 | status = wait4_signal(); 112 | DCHECK(status == ExecutorStatus::Execute); 113 | // Allow each worker to run transactions 114 | // DB is partitioned by the number of lock managers. 115 | // The first k workers act as lock managers to grant locks to other 116 | // workers The remaining workers run transactions upon assignment via the 117 | // queue. 118 | n_started_workers.store(0); 119 | n_completed_workers.store(0); 120 | clear_lock_manager_status(); 121 | set_worker_status(ExecutorStatus::Execute); 122 | wait_all_workers_start(); 123 | wait_all_workers_finish(); 124 | send_ack(); 125 | } 126 | } 127 | 128 | void add_worker(const std::shared_ptr> 129 | 130 | &w) { 131 | workers.push_back(w); 132 | } 133 | 134 | void clear_lock_manager_status() { lock_manager_status.store(0); } 135 | 136 | public: 137 | RandomType random; 138 | DatabaseType &db; 139 | CalvinPartitioner partitioner; 140 | std::atomic lock_manager_status; 141 | std::vector>> workers; 142 | std::vector storages; 143 | std::vector> transactions; 144 | }; 145 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Calvin/CalvinMessage.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/13/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Encoder.h" 8 | #include "common/Message.h" 9 | #include "common/MessagePiece.h" 10 | #include "core/ControlMessage.h" 11 | #include "core/Table.h" 12 | #include "protocol/Calvin/CalvinRWKey.h" 13 | #include "protocol/Calvin/CalvinTransaction.h" 14 | 15 | namespace aria { 16 | 17 | enum class CalvinMessage { 18 | READ_REQUEST = static_cast(ControlMessage::NFIELDS), 19 | NFIELDS 20 | }; 21 | 22 | class CalvinMessageFactory { 23 | 24 | public: 25 | static std::size_t new_read_message(Message &message, ITable &table, 26 | uint32_t tid, uint32_t key_offset, 27 | const void *value) { 28 | 29 | /* 30 | * The structure of a read request: (tid, key offset, value) 31 | */ 32 | 33 | auto value_size = table.value_size(); 34 | 35 | auto message_size = MessagePiece::get_header_size() + sizeof(tid) + 36 | sizeof(key_offset) + value_size; 37 | 38 | auto message_piece_header = MessagePiece::construct_message_piece_header( 39 | static_cast(CalvinMessage::READ_REQUEST), message_size, 40 | table.tableID(), table.partitionID()); 41 | 42 | Encoder encoder(message.data); 43 | encoder << message_piece_header; 44 | encoder << tid << key_offset; 45 | encoder.write_n_bytes(value, value_size); 46 | message.flush(); 47 | return message_size; 48 | } 49 | }; 50 | 51 | class CalvinMessageHandler { 52 | using Transaction = CalvinTransaction; 53 | 54 | public: 55 | static void 56 | read_request_handler(MessagePiece inputPiece, Message &responseMessage, 57 | ITable &table, 58 | std::vector> &txns) { 59 | DCHECK(inputPiece.get_message_type() == 60 | static_cast(CalvinMessage::READ_REQUEST)); 61 | auto table_id = inputPiece.get_table_id(); 62 | auto partition_id = inputPiece.get_partition_id(); 63 | DCHECK(table_id == table.tableID()); 64 | DCHECK(partition_id == table.partitionID()); 65 | auto value_size = table.value_size(); 66 | 67 | /* 68 | * The structure of a read request: (tid, key offset, value) 69 | * The structure of a read response: null 70 | */ 71 | 72 | uint32_t tid; 73 | uint32_t key_offset; 74 | 75 | DCHECK(inputPiece.get_message_length() == 76 | MessagePiece::get_header_size() + sizeof(tid) + sizeof(key_offset) + 77 | value_size); 78 | 79 | StringPiece stringPiece = inputPiece.toStringPiece(); 80 | Decoder dec(stringPiece); 81 | dec >> tid >> key_offset; 82 | DCHECK(tid < txns.size()); 83 | DCHECK(key_offset < txns[tid]->readSet.size()); 84 | CalvinRWKey &readKey = txns[tid]->readSet[key_offset]; 85 | dec.read_n_bytes(readKey.get_value(), value_size); 86 | txns[tid]->remote_read.fetch_add(-1); 87 | } 88 | 89 | static std::vector< 90 | std::function> &)>> 92 | get_message_handlers() { 93 | std::vector< 94 | std::function> &)>> 96 | v; 97 | v.resize(static_cast(ControlMessage::NFIELDS)); 98 | v.push_back(read_request_handler); 99 | return v; 100 | } 101 | }; 102 | 103 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Calvin/CalvinPartitioner.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 2019-09-05. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Partitioner.h" 8 | 9 | namespace aria { 10 | class CalvinPartitioner : public Partitioner { 11 | 12 | public: 13 | CalvinPartitioner(std::size_t coordinator_id, std::size_t coordinator_num, 14 | std::vector replica_group_sizes) 15 | : Partitioner(coordinator_id, coordinator_num) { 16 | 17 | std::size_t size = 0; 18 | for (auto i = 0u; i < replica_group_sizes.size(); i++) { 19 | CHECK(replica_group_sizes[i] > 0); 20 | size += replica_group_sizes[i]; 21 | 22 | if (coordinator_id < size) { 23 | coordinator_start_id = size - replica_group_sizes[i]; 24 | replica_group_id = i; 25 | replica_group_size = replica_group_sizes[i]; 26 | break; 27 | } 28 | } 29 | CHECK(std::accumulate(replica_group_sizes.begin(), 30 | replica_group_sizes.end(), 0u) == coordinator_num); 31 | } 32 | 33 | ~CalvinPartitioner() override = default; 34 | 35 | std::size_t replica_num() const override { return replica_group_size; } 36 | 37 | bool is_replicated() const override { 38 | // replica group in calvin is independent 39 | return false; 40 | } 41 | 42 | bool has_master_partition(std::size_t partition_id) const override { 43 | return master_coordinator(partition_id) == coordinator_id; 44 | } 45 | 46 | std::size_t master_coordinator(std::size_t partition_id) const override { 47 | return partition_id % replica_group_size + coordinator_start_id; 48 | } 49 | 50 | bool is_partition_replicated_on(std::size_t partition_id, 51 | std::size_t coordinator_id) const override { 52 | // replica group in calvin is independent 53 | return false; 54 | } 55 | 56 | bool is_backup() const override { return false; } 57 | 58 | public: 59 | std::size_t replica_group_id; 60 | std::size_t replica_group_size; 61 | 62 | private: 63 | // the first coordinator in this replica group 64 | std::size_t coordinator_start_id; 65 | }; 66 | } // namespace aria 67 | -------------------------------------------------------------------------------- /protocol/Calvin/CalvinRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/14/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class CalvinRWKey { 12 | public: 13 | // local index read bit 14 | 15 | void set_local_index_read_bit() { 16 | clear_local_index_read_bit(); 17 | bitvec |= LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET; 18 | } 19 | 20 | void clear_local_index_read_bit() { 21 | bitvec &= ~(LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET); 22 | } 23 | 24 | uint32_t get_local_index_read_bit() const { 25 | return (bitvec >> LOCAL_INDEX_READ_BIT_OFFSET) & LOCAL_INDEX_READ_BIT_MASK; 26 | } 27 | 28 | // read lock bit 29 | 30 | void set_read_lock_bit() { 31 | clear_read_lock_bit(); 32 | bitvec |= READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET; 33 | } 34 | 35 | void clear_read_lock_bit() { 36 | bitvec &= ~(READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 37 | } 38 | 39 | uint32_t get_read_lock_bit() const { 40 | return (bitvec >> READ_LOCK_BIT_OFFSET) & READ_LOCK_BIT_MASK; 41 | } 42 | 43 | // write lock bit 44 | 45 | void set_write_lock_bit() { 46 | clear_write_lock_bit(); 47 | bitvec |= WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET; 48 | } 49 | 50 | void clear_write_lock_bit() { 51 | bitvec &= ~(WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 52 | } 53 | 54 | uint32_t get_write_lock_bit() const { 55 | return (bitvec >> WRITE_LOCK_BIT_OFFSET) & WRITE_LOCK_BIT_MASK; 56 | } 57 | 58 | // prepare processed bit 59 | 60 | void set_prepare_processed_bit() { 61 | clear_prepare_processed_bit(); 62 | bitvec |= PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET; 63 | } 64 | 65 | void clear_prepare_processed_bit() { 66 | bitvec &= ~(PREPARE_PROCESSED_BIT_MASK << PREPARE_PROCESSED_BIT_OFFSET); 67 | } 68 | 69 | uint32_t get_prepare_processed_bit() const { 70 | return (bitvec >> PREPARE_PROCESSED_BIT_OFFSET) & 71 | PREPARE_PROCESSED_BIT_MASK; 72 | } 73 | 74 | // execution processed bit 75 | 76 | void set_execution_processed_bit() { 77 | clear_execution_processed_bit(); 78 | bitvec |= EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET; 79 | } 80 | 81 | void clear_execution_processed_bit() { 82 | bitvec &= ~(EXECUTION_PROCESSED_BIT_MASK << EXECUTION_PROCESSED_BIT_OFFSET); 83 | } 84 | 85 | uint32_t get_execution_processed_bit() const { 86 | return (bitvec >> EXECUTION_PROCESSED_BIT_OFFSET) & 87 | EXECUTION_PROCESSED_BIT_MASK; 88 | } 89 | 90 | // table id 91 | 92 | void set_table_id(uint32_t table_id) { 93 | DCHECK(table_id < (1 << 5)); 94 | clear_table_id(); 95 | bitvec |= table_id << TABLE_ID_OFFSET; 96 | } 97 | 98 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 99 | 100 | uint32_t get_table_id() const { 101 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 102 | } 103 | // partition id 104 | 105 | void set_partition_id(uint32_t partition_id) { 106 | DCHECK(partition_id < (1 << 16)); 107 | clear_partition_id(); 108 | bitvec |= partition_id << PARTITION_ID_OFFSET; 109 | } 110 | 111 | void clear_partition_id() { 112 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 113 | } 114 | 115 | uint32_t get_partition_id() const { 116 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 117 | } 118 | 119 | // key 120 | void set_key(const void *key) { this->key = key; } 121 | 122 | const void *get_key() const { return key; } 123 | 124 | // value 125 | void set_value(void *value) { this->value = value; } 126 | 127 | void *get_value() const { return value; } 128 | 129 | private: 130 | /* 131 | * A bitvec is a 32-bit word. 132 | * 133 | * [ table id (5) ] | partition id (16) | unused bit (6) | 134 | * prepare processed bit (1) | execute processed bit(1) | 135 | * write lock bit(1) | read lock bit (1) | local index read (1) ] 136 | * 137 | * local index read is set when the read is from a local read only index. 138 | * write lock bit is set when a write lock is acquired. 139 | * read lock bit is set when a read lock is acquired. 140 | * prepare processed bit is set when process_request has processed this key in 141 | * prepare phase exucution processed bit is set when process_request has 142 | * processed this key in execution phase 143 | */ 144 | 145 | uint32_t bitvec = 0; 146 | const void *key = nullptr; 147 | void *value = nullptr; 148 | 149 | public: 150 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 151 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 152 | 153 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 154 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 155 | 156 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_MASK = 0x1; 157 | static constexpr uint32_t EXECUTION_PROCESSED_BIT_OFFSET = 4; 158 | 159 | static constexpr uint32_t PREPARE_PROCESSED_BIT_MASK = 0x1; 160 | static constexpr uint32_t PREPARE_PROCESSED_BIT_OFFSET = 3; 161 | 162 | static constexpr uint32_t WRITE_LOCK_BIT_MASK = 0x1; 163 | static constexpr uint32_t WRITE_LOCK_BIT_OFFSET = 2; 164 | 165 | static constexpr uint32_t READ_LOCK_BIT_MASK = 0x1; 166 | static constexpr uint32_t READ_LOCK_BIT_OFFSET = 1; 167 | 168 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_MASK = 0x1; 169 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_OFFSET = 0; 170 | }; 171 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Pwv/PwvExecutor.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/14/20. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "common/Percentile.h" 8 | #include "core/Delay.h" 9 | #include "core/Worker.h" 10 | #include "glog/logging.h" 11 | 12 | #include "protocol/Pwv/PwvHelper.h" 13 | #include "protocol/Pwv/PwvTransaction.h" 14 | #include "protocol/Pwv/PwvWorkload.h" 15 | 16 | #include 17 | #include 18 | 19 | namespace aria { 20 | 21 | template class PwvExecutor : public Worker { 22 | public: 23 | using DatabaseType = Database; 24 | using WorkloadType = PwvWorkload; 25 | using StorageType = typename DatabaseType::StorageType; 26 | using TransactionType = PwvTransaction; 27 | using ContextType = typename DatabaseType::ContextType; 28 | using RandomType = typename DatabaseType::RandomType; 29 | 30 | PwvExecutor(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 31 | const ContextType &context, 32 | std::vector> &transactions, 33 | std::vector &storages, std::atomic &epoch, 34 | std::atomic &worker_status, 35 | std::atomic &n_complete_workers, 36 | std::atomic &n_started_workers) 37 | : Worker(coordinator_id, id), db(db), context(context), 38 | transactions(transactions), storages(storages), epoch(epoch), 39 | worker_status(worker_status), n_complete_workers(n_complete_workers), 40 | n_started_workers(n_started_workers), workload(db, random), 41 | init_transaction(false), 42 | random(id), // make sure each worker has a different seed. 43 | sleep_random(reinterpret_cast(this)) {} 44 | 45 | ~PwvExecutor() = default; 46 | 47 | void start() override { 48 | 49 | LOG(INFO) << "PwvExecutor" << id << " started. "; 50 | 51 | for (;;) { 52 | 53 | ExecutorStatus status; 54 | do { 55 | status = static_cast(worker_status.load()); 56 | 57 | if (status == ExecutorStatus::EXIT) { 58 | LOG(INFO) << "PwvExecutor" << id << " exits. "; 59 | return; 60 | } 61 | } while (status != ExecutorStatus::Pwv_Analysis); 62 | 63 | n_started_workers.fetch_add(1); 64 | generate_transactions(); 65 | n_complete_workers.fetch_add(1); 66 | // wait to Execute 67 | while (static_cast(worker_status.load()) == 68 | ExecutorStatus::Pwv_Analysis) { 69 | std::this_thread::yield(); 70 | } 71 | 72 | n_started_workers.fetch_add(1); 73 | run_transactions(); 74 | n_complete_workers.fetch_add(1); 75 | // wait to execute 76 | while (static_cast(worker_status.load()) == 77 | ExecutorStatus::Pwv_Execute) { 78 | std::this_thread::yield(); 79 | } 80 | } 81 | } 82 | 83 | void onExit() override { 84 | LOG(INFO) << "Worker " << id << " latency: " << percentile.nth(50) 85 | << " us (50%) " << percentile.nth(75) << " us (75%) " 86 | << percentile.nth(95) << " us (95%) " << percentile.nth(99) 87 | << " us (99%)."; 88 | } 89 | 90 | void push_message(Message *message) override {} 91 | 92 | Message *pop_message() override { return nullptr; } 93 | 94 | void generate_transactions() { 95 | for (auto i = id; i < transactions.size(); i += context.worker_num) { 96 | auto partition_id = random.uniform_dist(0, context.partition_num - 1); 97 | transactions[i] = 98 | workload.next_transaction(context, partition_id, storages[i]); 99 | transactions[i]->build_pieces(); 100 | } 101 | init_transaction = true; 102 | } 103 | 104 | void run_transactions() { 105 | for (auto i = 0u; i < transactions.size(); i++) { 106 | bool commit = transactions[i]->commit(id); 107 | if (transactions[i]->partition_id % context.worker_num == id) { 108 | if (commit) { 109 | n_commit.fetch_add(1); 110 | } else { 111 | n_abort_no_retry.fetch_add(1); 112 | } 113 | } 114 | } 115 | } 116 | 117 | private: 118 | DatabaseType &db; 119 | const ContextType &context; 120 | std::vector> &transactions; 121 | std::vector &storages; 122 | std::atomic &epoch, &worker_status; 123 | std::atomic &n_complete_workers, &n_started_workers; 124 | WorkloadType workload; 125 | bool init_transaction; 126 | RandomType random, sleep_random; 127 | Percentile percentile; 128 | }; 129 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Pwv/PwvHelper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/14/20. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #include "glog/logging.h" 10 | 11 | namespace aria { 12 | 13 | class PwvHelper { 14 | public: 15 | using MetaDataType = std::atomic; 16 | static void read(const std::tuple &row, void *dest, 17 | std::size_t size) { 18 | void *src = std::get<1>(row); 19 | std::memcpy(dest, src, size); 20 | } 21 | }; 22 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Pwv/PwvManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/14/20. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Manager.h" 8 | #include "protocol/Pwv/PwvExecutor.h" 9 | #include "protocol/Pwv/PwvHelper.h" 10 | #include "protocol/Pwv/PwvTransaction.h" 11 | 12 | #include 13 | #include 14 | 15 | namespace aria { 16 | template class PwvManager : public aria::Manager { 17 | public: 18 | using base_type = aria::Manager; 19 | using DatabaseType = Database; 20 | using WorkloadType = PwvWorkload; 21 | using StorageType = typename DatabaseType::StorageType; 22 | using TransactionType = PwvTransaction; 23 | using ContextType = typename DatabaseType::ContextType; 24 | using RandomType = typename DatabaseType::RandomType; 25 | 26 | PwvManager(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 27 | const ContextType &context, std::atomic &stopFlag) 28 | : base_type(coordinator_id, id, context, stopFlag), db(db), epoch(0) { 29 | 30 | storages.resize(context.batch_size); 31 | transactions.resize(context.batch_size); 32 | } 33 | 34 | void coordinator_start() override { 35 | 36 | std::size_t n_workers = context.worker_num; 37 | std::size_t n_coordinators = context.coordinator_num; 38 | 39 | while (!stopFlag.load()) { 40 | 41 | // the coordinator generates a batch of transactions 42 | 43 | epoch.fetch_add(1); 44 | 45 | // LOG(INFO) << "Seed: " << random.get_seed(); 46 | n_started_workers.store(0); 47 | n_completed_workers.store(0); 48 | signal_worker(ExecutorStatus::Pwv_Analysis); 49 | // Allow each worker to analyse the read/write set 50 | // each worker analyse i, i + n, i + 2n transaction 51 | wait_all_workers_start(); 52 | wait_all_workers_finish(); 53 | 54 | // Allow each worker to run transactions 55 | n_started_workers.store(0); 56 | n_completed_workers.store(0); 57 | signal_worker(ExecutorStatus::Pwv_Execute); 58 | wait_all_workers_start(); 59 | wait_all_workers_finish(); 60 | } 61 | 62 | signal_worker(ExecutorStatus::EXIT); 63 | } 64 | 65 | public: 66 | RandomType random; 67 | DatabaseType &db; 68 | std::atomic epoch; 69 | std::vector storages; 70 | std::vector> transactions; 71 | }; 72 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Pwv/PwvRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/14/20. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace aria { 10 | 11 | class PwvRWKey { 12 | public: 13 | // table id 14 | 15 | void set_table_id(uint32_t table_id) { 16 | DCHECK(table_id < (1 << 5)); 17 | clear_table_id(); 18 | bitvec |= table_id << TABLE_ID_OFFSET; 19 | } 20 | 21 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 22 | 23 | uint32_t get_table_id() const { 24 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 25 | } 26 | // partition id 27 | 28 | void set_partition_id(uint32_t partition_id) { 29 | DCHECK(partition_id < (1 << 16)); 30 | clear_partition_id(); 31 | bitvec |= partition_id << PARTITION_ID_OFFSET; 32 | } 33 | 34 | void clear_partition_id() { 35 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 36 | } 37 | 38 | uint32_t get_partition_id() const { 39 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 40 | } 41 | 42 | // key 43 | void set_key(const void *key) { this->key = key; } 44 | 45 | const void *get_key() const { return key; } 46 | 47 | // value 48 | void set_value(void *value) { this->value = value; } 49 | 50 | void *get_value() const { return value; } 51 | 52 | private: 53 | /* 54 | * A bitvec is a 32-bit word. 55 | * 56 | * [ table id (5) ] | partition id (16) | unused bit (11) ] 57 | * 58 | */ 59 | 60 | uint32_t bitvec = 0; 61 | const void *key = nullptr; 62 | void *value = nullptr; 63 | 64 | public: 65 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 66 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 67 | 68 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 69 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 70 | }; 71 | } // namespace aria -------------------------------------------------------------------------------- /protocol/Pwv/PwvWorkload.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 1/16/20. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "benchmark/tpcc/Context.h" 8 | #include "benchmark/tpcc/Database.h" 9 | #include "benchmark/ycsb/Context.h" 10 | #include "benchmark/ycsb/Database.h" 11 | 12 | namespace aria { 13 | template class PwvWorkload { 14 | public: 15 | using DatabaseType = Database; 16 | using StorageType = typename DatabaseType::StorageType; 17 | using TransactionType = PwvTransaction; 18 | using ContextType = typename DatabaseType::ContextType; 19 | using RandomType = typename DatabaseType::RandomType; 20 | 21 | PwvWorkload(DatabaseType &db, RandomType &random) { 22 | CHECK(false) << "this workload is not supported"; 23 | } 24 | std::unique_ptr next_transaction(const ContextType &context, 25 | std::size_t partition_id, 26 | StorageType &storage) { 27 | CHECK(false) << "this workload is not supported"; 28 | return nullptr; 29 | } 30 | }; 31 | 32 | template <> class PwvWorkload { 33 | public: 34 | using DatabaseType = ycsb::Database; 35 | using StorageType = typename DatabaseType::StorageType; 36 | using TransactionType = PwvTransaction; 37 | using ContextType = typename DatabaseType::ContextType; 38 | using RandomType = typename DatabaseType::RandomType; 39 | 40 | PwvWorkload(DatabaseType &db, RandomType &random) : db(db), random(random) {} 41 | 42 | std::unique_ptr next_transaction(const ContextType &context, 43 | std::size_t partition_id, 44 | StorageType &storage) { 45 | 46 | if (context.pwv_ycsb_star) { 47 | auto p = std::make_unique(db, context, random, 48 | storage, partition_id); 49 | return p; 50 | } else { 51 | auto p = std::make_unique(db, context, random, 52 | storage, partition_id); 53 | return p; 54 | } 55 | } 56 | 57 | private: 58 | DatabaseType &db; 59 | RandomType &random; 60 | }; 61 | 62 | template <> class PwvWorkload { 63 | public: 64 | using DatabaseType = tpcc::Database; 65 | using StorageType = typename DatabaseType::StorageType; 66 | using TransactionType = PwvTransaction; 67 | using ContextType = typename DatabaseType::ContextType; 68 | using RandomType = typename DatabaseType::RandomType; 69 | 70 | PwvWorkload(DatabaseType &db, RandomType &random) : db(db), random(random) {} 71 | 72 | std::unique_ptr next_transaction(const ContextType &context, 73 | std::size_t partition_id, 74 | StorageType &storage) { 75 | 76 | int x = random.uniform_dist(1, 100); 77 | std::unique_ptr p; 78 | 79 | if (context.workloadType == tpcc::TPCCWorkloadType::MIXED) { 80 | if (x <= 50) { 81 | p = std::make_unique(db, context, random, 82 | storage, partition_id); 83 | } else { 84 | p = std::make_unique(db, context, random, 85 | storage, partition_id); 86 | } 87 | } else if (context.workloadType == tpcc::TPCCWorkloadType::NEW_ORDER_ONLY) { 88 | p = std::make_unique(db, context, random, storage, 89 | partition_id); 90 | } else { 91 | p = std::make_unique(db, context, random, storage, 92 | partition_id); 93 | } 94 | 95 | return p; 96 | } 97 | 98 | private: 99 | DatabaseType &db; 100 | RandomType &random; 101 | }; 102 | 103 | } // namespace aria 104 | -------------------------------------------------------------------------------- /protocol/TwoPL/TwoPLExecutor.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/11/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include "core/Executor.h" 8 | #include "protocol/TwoPL/TwoPL.h" 9 | 10 | namespace aria { 11 | template 12 | class TwoPLExecutor 13 | : public Executor> 14 | 15 | { 16 | public: 17 | using base_type = Executor>; 18 | 19 | using WorkloadType = Workload; 20 | using ProtocolType = TwoPL; 21 | using DatabaseType = typename WorkloadType::DatabaseType; 22 | using TransactionType = typename WorkloadType::TransactionType; 23 | using ContextType = typename DatabaseType::ContextType; 24 | using RandomType = typename DatabaseType::RandomType; 25 | using MessageType = typename ProtocolType::MessageType; 26 | using MessageFactoryType = typename ProtocolType::MessageFactoryType; 27 | using MessageHandlerType = typename ProtocolType::MessageHandlerType; 28 | 29 | using StorageType = typename WorkloadType::StorageType; 30 | 31 | TwoPLExecutor(std::size_t coordinator_id, std::size_t id, DatabaseType &db, 32 | const ContextType &context, 33 | std::atomic &worker_status, 34 | std::atomic &n_complete_workers, 35 | std::atomic &n_started_workers) 36 | : base_type(coordinator_id, id, db, context, worker_status, 37 | n_complete_workers, n_started_workers) {} 38 | 39 | ~ 40 | 41 | TwoPLExecutor() = default; 42 | 43 | void setupHandlers(TransactionType &txn) 44 | 45 | override { 46 | txn.lock_request_handler = 47 | [this, &txn](std::size_t table_id, std::size_t partition_id, 48 | uint32_t key_offset, const void *key, void *value, 49 | bool local_index_read, bool write_lock, bool &success, 50 | bool &remote) -> uint64_t { 51 | if (local_index_read) { 52 | success = true; 53 | remote = false; 54 | return this->protocol.search(table_id, partition_id, key, value); 55 | } 56 | 57 | ITable *table = this->db.find_table(table_id, partition_id); 58 | 59 | if (this->partitioner->has_master_partition(partition_id)) { 60 | 61 | remote = false; 62 | 63 | std::atomic &tid = table->search_metadata(key); 64 | 65 | if (write_lock) { 66 | TwoPLHelper::write_lock(tid, success); 67 | } else { 68 | TwoPLHelper::read_lock(tid, success); 69 | } 70 | 71 | if (success) { 72 | return this->protocol.search(table_id, partition_id, key, value); 73 | } else { 74 | return 0; 75 | } 76 | 77 | } else { 78 | 79 | remote = true; 80 | 81 | auto coordinatorID = 82 | this->partitioner->master_coordinator(partition_id); 83 | 84 | if (write_lock) { 85 | txn.network_size += MessageFactoryType::new_write_lock_message( 86 | *(this->messages[coordinatorID]), *table, key, key_offset); 87 | } else { 88 | txn.network_size += MessageFactoryType::new_read_lock_message( 89 | *(this->messages[coordinatorID]), *table, key, key_offset); 90 | } 91 | txn.distributed_transaction = true; 92 | return 0; 93 | } 94 | }; 95 | 96 | txn.remote_request_handler = [this]() { return this->process_request(); }; 97 | txn.message_flusher = [this]() { this->flush_messages(); }; 98 | }; 99 | }; 100 | } // namespace aria 101 | -------------------------------------------------------------------------------- /protocol/TwoPL/TwoPLHelper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/11/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace aria { 13 | 14 | class TwoPLHelper { 15 | public: 16 | using MetaDataType = std::atomic; 17 | 18 | static uint64_t read(const std::tuple &row, 19 | void *dest, std::size_t size) { 20 | 21 | MetaDataType &tid = *std::get<0>(row); 22 | void *src = std::get<1>(row); 23 | std::memcpy(dest, src, size); 24 | uint64_t tid_ = tid.load(); 25 | return remove_lock_bit(tid_); 26 | } 27 | 28 | /** 29 | * [write lock bit (1) | read lock bit (9) -- 512 - 1 locks | seq id (54) ] 30 | * 31 | */ 32 | 33 | static bool is_read_locked(uint64_t value) { 34 | return value & (READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 35 | } 36 | 37 | static bool is_write_locked(uint64_t value) { 38 | return value & (WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 39 | } 40 | 41 | static uint64_t read_lock_num(uint64_t value) { 42 | return (value >> READ_LOCK_BIT_OFFSET) & READ_LOCK_BIT_MASK; 43 | } 44 | 45 | static uint64_t read_lock_max() { return READ_LOCK_BIT_MASK; } 46 | 47 | static uint64_t read_lock(std::atomic &a, bool &success) { 48 | uint64_t old_value, new_value; 49 | do { 50 | old_value = a.load(); 51 | if (is_write_locked(old_value) || 52 | read_lock_num(old_value) == read_lock_max()) { 53 | success = false; 54 | return remove_lock_bit(old_value); 55 | } 56 | new_value = old_value + (1ull << READ_LOCK_BIT_OFFSET); 57 | } while (!a.compare_exchange_weak(old_value, new_value)); 58 | success = true; 59 | return remove_lock_bit(old_value); 60 | } 61 | 62 | static uint64_t write_lock(std::atomic &a, bool &success) { 63 | uint64_t old_value = a.load(); 64 | if (is_read_locked(old_value) || is_write_locked(old_value)) { 65 | success = false; 66 | return remove_lock_bit(old_value); 67 | } 68 | uint64_t new_value = 69 | old_value + (WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 70 | success = a.compare_exchange_strong(old_value, new_value); 71 | return remove_lock_bit(old_value); 72 | } 73 | 74 | static uint64_t write_lock(std::atomic &a) { 75 | uint64_t old_value, new_value; 76 | 77 | do { 78 | do { 79 | old_value = a.load(); 80 | } while (is_read_locked(old_value) || is_write_locked(old_value)); 81 | 82 | new_value = old_value + (WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 83 | 84 | } while (!a.compare_exchange_weak(old_value, new_value)); 85 | return remove_lock_bit(old_value); 86 | } 87 | 88 | static void read_lock_release(std::atomic &a) { 89 | uint64_t old_value, new_value; 90 | do { 91 | old_value = a.load(); 92 | DCHECK(is_read_locked(old_value)); 93 | DCHECK(!is_write_locked(old_value)); 94 | new_value = old_value - (1ull << READ_LOCK_BIT_OFFSET); 95 | } while (!a.compare_exchange_weak(old_value, new_value)); 96 | } 97 | 98 | static void write_lock_release(std::atomic &a) { 99 | uint64_t old_value, new_value; 100 | old_value = a.load(); 101 | DCHECK(!is_read_locked(old_value)); 102 | DCHECK(is_write_locked(old_value)); 103 | new_value = old_value - (1ull << WRITE_LOCK_BIT_OFFSET); 104 | bool ok = a.compare_exchange_strong(old_value, new_value); 105 | DCHECK(ok); 106 | } 107 | 108 | static void write_lock_release(std::atomic &a, uint64_t new_value) { 109 | uint64_t old_value; 110 | old_value = a.load(); 111 | DCHECK(!is_read_locked(old_value)); 112 | DCHECK(is_write_locked(old_value)); 113 | DCHECK(!is_read_locked(new_value)); 114 | DCHECK(!is_write_locked(new_value)); 115 | bool ok = a.compare_exchange_weak(old_value, new_value); 116 | DCHECK(ok); 117 | } 118 | 119 | static uint64_t remove_lock_bit(uint64_t value) { 120 | return value & ~(LOCK_BIT_MASK << LOCK_BIT_OFFSET); 121 | } 122 | 123 | static uint64_t remove_read_lock_bit(uint64_t value) { 124 | return value & ~(READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 125 | } 126 | 127 | static uint64_t remove_write_lock_bit(uint64_t value) { 128 | return value & ~(WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 129 | } 130 | 131 | public: 132 | static constexpr int LOCK_BIT_OFFSET = 54; 133 | static constexpr uint64_t LOCK_BIT_MASK = 0x3ffull; 134 | 135 | static constexpr int READ_LOCK_BIT_OFFSET = 54; 136 | static constexpr uint64_t READ_LOCK_BIT_MASK = 0x1ffull; 137 | 138 | static constexpr int WRITE_LOCK_BIT_OFFSET = 63; 139 | static constexpr uint64_t WRITE_LOCK_BIT_MASK = 0x1ull; 140 | }; 141 | } // namespace aria -------------------------------------------------------------------------------- /protocol/TwoPL/TwoPLRWKey.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yi Lu on 9/11/18. 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | namespace aria { 14 | 15 | class TwoPLRWKey { 16 | public: 17 | // local index read bit 18 | 19 | void set_local_index_read_bit() { 20 | clear_local_index_read_bit(); 21 | bitvec |= LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET; 22 | } 23 | 24 | void clear_local_index_read_bit() { 25 | bitvec &= ~(LOCAL_INDEX_READ_BIT_MASK << LOCAL_INDEX_READ_BIT_OFFSET); 26 | } 27 | 28 | uint32_t get_local_index_read_bit() const { 29 | return (bitvec >> LOCAL_INDEX_READ_BIT_OFFSET) & LOCAL_INDEX_READ_BIT_MASK; 30 | } 31 | 32 | // read lock bit 33 | 34 | void set_read_lock_bit() { 35 | clear_read_lock_bit(); 36 | bitvec |= READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET; 37 | } 38 | 39 | void clear_read_lock_bit() { 40 | bitvec &= ~(READ_LOCK_BIT_MASK << READ_LOCK_BIT_OFFSET); 41 | } 42 | 43 | uint32_t get_read_lock_bit() const { 44 | return (bitvec >> READ_LOCK_BIT_OFFSET) & READ_LOCK_BIT_MASK; 45 | } 46 | 47 | // write lock bit 48 | 49 | void set_write_lock_bit() { 50 | clear_write_lock_bit(); 51 | bitvec |= WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET; 52 | } 53 | 54 | void clear_write_lock_bit() { 55 | bitvec &= ~(WRITE_LOCK_BIT_MASK << WRITE_LOCK_BIT_OFFSET); 56 | } 57 | 58 | uint32_t get_write_lock_bit() const { 59 | return (bitvec >> WRITE_LOCK_BIT_OFFSET) & WRITE_LOCK_BIT_MASK; 60 | } 61 | 62 | // read lock request bit 63 | 64 | void set_read_lock_request_bit() { 65 | clear_read_lock_request_bit(); 66 | bitvec |= READ_LOCK_REQUEST_BIT_MASK << READ_LOCK_REQUEST_BIT_OFFSET; 67 | } 68 | 69 | void clear_read_lock_request_bit() { 70 | bitvec &= ~(READ_LOCK_REQUEST_BIT_MASK << READ_LOCK_REQUEST_BIT_OFFSET); 71 | } 72 | 73 | uint32_t get_read_lock_request_bit() const { 74 | return (bitvec >> READ_LOCK_REQUEST_BIT_OFFSET) & 75 | READ_LOCK_REQUEST_BIT_MASK; 76 | } 77 | 78 | // write lock request bit 79 | 80 | void set_write_lock_request_bit() { 81 | clear_write_lock_request_bit(); 82 | bitvec |= WRITE_LOCK_REQUEST_BIT_MASK << WRITE_LOCK_REQUEST_BIT_OFFSET; 83 | } 84 | 85 | void clear_write_lock_request_bit() { 86 | bitvec &= ~(WRITE_LOCK_REQUEST_BIT_MASK << WRITE_LOCK_REQUEST_BIT_OFFSET); 87 | } 88 | 89 | uint32_t get_write_lock_request_bit() const { 90 | return (bitvec >> WRITE_LOCK_REQUEST_BIT_OFFSET) & 91 | WRITE_LOCK_REQUEST_BIT_MASK; 92 | } 93 | 94 | // table id 95 | 96 | void set_table_id(uint32_t table_id) { 97 | DCHECK(table_id < (1 << 5)); 98 | clear_table_id(); 99 | bitvec |= table_id << TABLE_ID_OFFSET; 100 | } 101 | 102 | void clear_table_id() { bitvec &= ~(TABLE_ID_MASK << TABLE_ID_OFFSET); } 103 | 104 | uint32_t get_table_id() const { 105 | return (bitvec >> TABLE_ID_OFFSET) & TABLE_ID_MASK; 106 | } 107 | // partition id 108 | 109 | void set_partition_id(uint32_t partition_id) { 110 | DCHECK(partition_id < (1 << 16)); 111 | clear_partition_id(); 112 | bitvec |= partition_id << PARTITION_ID_OFFSET; 113 | } 114 | 115 | void clear_partition_id() { 116 | bitvec &= ~(PARTITION_ID_MASK << PARTITION_ID_OFFSET); 117 | } 118 | 119 | uint32_t get_partition_id() const { 120 | return (bitvec >> PARTITION_ID_OFFSET) & PARTITION_ID_MASK; 121 | } 122 | 123 | // tid 124 | uint64_t get_tid() const { return tid; } 125 | 126 | void set_tid(uint64_t tid) { this->tid = tid; } 127 | 128 | // key 129 | void set_key(const void *key) { this->key = key; } 130 | 131 | const void *get_key() const { return key; } 132 | 133 | // value 134 | void set_value(void *value) { this->value = value; } 135 | 136 | void *get_value() const { return value; } 137 | 138 | private: 139 | /* 140 | * A bitvec is a 32-bit word. 141 | * 142 | * [ table id (5) ] | partition id (16) | unused bit (6) | 143 | * write lock request bit (1) | read lock request bit (1) 144 | * write lock bit(1) | read lock bit (1) | local index read (1) ] 145 | * 146 | * 147 | * local index read is set when the read is from a local read only index. 148 | * write lock bit is set when a write lock is acquired. 149 | * read lock bit is set when a read lock is acquired. 150 | * write lock request bit is set when a write lock request is needed. 151 | * read lock request bit is set when a read lock request is needed. 152 | * 153 | */ 154 | 155 | uint32_t bitvec = 0; 156 | uint64_t tid = 0; 157 | const void *key = nullptr; 158 | void *value = nullptr; 159 | 160 | public: 161 | static constexpr uint32_t TABLE_ID_MASK = 0x1f; 162 | static constexpr uint32_t TABLE_ID_OFFSET = 27; 163 | 164 | static constexpr uint32_t PARTITION_ID_MASK = 0xffff; 165 | static constexpr uint32_t PARTITION_ID_OFFSET = 11; 166 | 167 | static constexpr uint32_t WRITE_LOCK_REQUEST_BIT_MASK = 0x1; 168 | static constexpr uint32_t WRITE_LOCK_REQUEST_BIT_OFFSET = 4; 169 | 170 | static constexpr uint32_t READ_LOCK_REQUEST_BIT_MASK = 0x1; 171 | static constexpr uint32_t READ_LOCK_REQUEST_BIT_OFFSET = 3; 172 | 173 | static constexpr uint32_t WRITE_LOCK_BIT_MASK = 0x1; 174 | static constexpr uint32_t WRITE_LOCK_BIT_OFFSET = 2; 175 | 176 | static constexpr uint32_t READ_LOCK_BIT_MASK = 0x1; 177 | static constexpr uint32_t READ_LOCK_BIT_OFFSET = 1; 178 | 179 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_MASK = 0x1; 180 | static constexpr uint32_t LOCAL_INDEX_READ_BIT_OFFSET = 0; 181 | }; 182 | } // namespace aria 183 | -------------------------------------------------------------------------------- /scripts/aws_barrier.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 5 - the barrier study 5 | ''' 6 | 7 | import sys 8 | 9 | from utility import load_ips 10 | from utility import get_cmd_string 11 | 12 | # repeat experiments for the following times 13 | REPEAT = 5 14 | 15 | assert len(sys.argv) == 3, "this script expects two parameters" 16 | 17 | machine_id = int(sys.argv[1]) 18 | port = int(sys.argv[2]) 19 | 20 | ips = load_ips('ips.txt') 21 | 22 | n_machines = len(ips) 23 | 24 | assert n_machines == 1, "we are expecting there is 1 machine." 25 | 26 | threads = 12 27 | partition_num = threads 28 | batch_size = 1000 29 | 30 | def print_ycsb(): 31 | 32 | read_write_ratio = 80 33 | zipf = 0.0 34 | keys = 40000 35 | cross_ratio = 100 36 | barrier_delayed_percents = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000] # out of 1K 37 | barrier_artificial_delay_mss = [1000000, 500000, 200000, 100000, 50000, 20000, 10000] # ms 38 | 39 | # kiva 40 | for barrier_delayed_percent in barrier_delayed_percents: 41 | for barrier_artificial_delay_ms in barrier_artificial_delay_mss: 42 | assert barrier_artificial_delay_ms % barrier_delayed_percent == 0 43 | tts = barrier_artificial_delay_ms // barrier_delayed_percent 44 | for i in range(REPEAT): 45 | cmd = get_cmd_string(machine_id, ips, port + i) 46 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --barrier_delayed_percent={barrier_delayed_percent} --barrier_artificial_delay_ms={tts}') 47 | 48 | 49 | def main(): 50 | # ycsb 51 | print_ycsb() 52 | 53 | if __name__ == '__main__': 54 | main() 55 | -------------------------------------------------------------------------------- /scripts/aws_dist_tpcc.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 13(b) -- Performance of each system on YCSB and TPC-C in the distributed setting 5 | Systems: Aria, AriaFB, Calvin and S2PL 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 8, "we are expecting there are 8 machines." 26 | 27 | threads = 12 28 | batch_size = 500 29 | 30 | def print_tpcc(): 31 | 32 | query = 'mixed' 33 | cross_ratios = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] 34 | partition_num = 108 * n_machines 35 | 36 | # aria 37 | for cross_ratio in cross_ratios: 38 | for i in range(REPEAT): 39 | cmd = get_cmd_string(machine_id, ips, port + i) 40 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={cross_ratio} --payment_dist={cross_ratio}') 41 | 42 | # S2PL 43 | for cross_ratio in cross_ratios: 44 | for i in range(REPEAT): 45 | cmd = get_cmd_string(machine_id, ips, port + i) 46 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=TwoPL --partitioner=hash2 --partition_num={partition_num} --threads={threads} --query={query} --neworder_dist={cross_ratio} --payment_dist={cross_ratio}') 47 | 48 | # calvin 49 | n_lock_managers = [1, 2, 3, 4, 6] 50 | for n_lock_manager in n_lock_managers: 51 | for cross_ratio in cross_ratios: 52 | for i in range(REPEAT): 53 | cmd = get_cmd_string(machine_id, ips, port + i) 54 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Calvin --partition_num={partition_num} --threads={threads} --batch_size={batch_size * n_machines} --query={query} --neworder_dist={cross_ratio} --payment_dist={cross_ratio} --lock_manager={n_lock_manager} --replica_group={n_machines} --same_batch=True') 55 | 56 | # ariaFB 57 | n_lock_managers = [1, 2, 3, 4, 6] 58 | for n_lock_manager in n_lock_managers: 59 | for cross_ratio in cross_ratios: 60 | for i in range(REPEAT): 61 | cmd = get_cmd_string(machine_id, ips, port + i) 62 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=AriaFB --partition_num={partition_num} --threads={threads} --batch_size={batch_size * n_machines // 2} --query={query} --neworder_dist={cross_ratio} --payment_dist={cross_ratio} --ariaFB_lock_manager={n_lock_manager} --same_batch=True') 63 | 64 | def main(): 65 | # tpc-c 66 | print_tpcc() 67 | 68 | if __name__ == '__main__': 69 | main() 70 | -------------------------------------------------------------------------------- /scripts/aws_dist_ycsb.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 13(a) -- Performance of each system on YCSB and TPC-C in the distributed setting 5 | Systems: Aria, AriaFB, Calvin, and S2PL 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 8, "we are expecting there are 8 machines." 26 | 27 | threads = 12 28 | partition_num = threads * n_machines 29 | batch_size = 10000 30 | 31 | def print_ycsb(): 32 | 33 | read_write_ratio = 80 34 | zipf = 0.0 35 | keys = 40000 36 | cross_ratios = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] 37 | 38 | # aria 39 | for cross_ratio in cross_ratios: 40 | for i in range(REPEAT): 41 | cmd = get_cmd_string(machine_id, ips, port + i) 42 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --two_partitions=True') 43 | 44 | # S2PL 45 | for cross_ratio in cross_ratios: 46 | for i in range(REPEAT): 47 | cmd = get_cmd_string(machine_id, ips, port + i) 48 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=TwoPL --partitioner=hash2 --partition_num={partition_num} --threads={threads} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --two_partitions=True') 49 | 50 | # calvin 51 | n_lock_managers = [1, 2, 3, 4, 6] 52 | for n_lock_manager in n_lock_managers: 53 | for cross_ratio in cross_ratios: 54 | for i in range(REPEAT): 55 | cmd = get_cmd_string(machine_id, ips, port + i) 56 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Calvin --partition_num={partition_num} --threads={threads} --batch_size={batch_size * n_machines} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --lock_manager={n_lock_manager} --replica_group={n_machines} --same_batch=True --two_partitions=True') 57 | 58 | # ariaFB 59 | n_lock_managers = [1, 2, 3, 4, 6] 60 | for n_lock_manager in n_lock_managers: 61 | for cross_ratio in cross_ratios: 62 | for i in range(REPEAT): 63 | cmd = get_cmd_string(machine_id, ips, port + i) 64 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=AriaFB --partition_num={partition_num} --threads={threads} --batch_size={batch_size * n_machines // 2} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --ariaFB_lock_manager={n_lock_manager} --same_batch=True --two_partitions=True') 65 | 66 | def main(): 67 | # ycsb 68 | print_ycsb() 69 | 70 | if __name__ == '__main__': 71 | main() 72 | -------------------------------------------------------------------------------- /scripts/aws_hstore.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 10(a) - hstore 5 | 6 | Systems: HStore 7 | ''' 8 | 9 | import sys 10 | 11 | from utility import load_ips 12 | from utility import get_cmd_string 13 | 14 | # repeat experiments for the following times 15 | REPEAT = 5 16 | 17 | assert len(sys.argv) == 3, "this script expects two parameters" 18 | 19 | machine_id = int(sys.argv[1]) 20 | port = int(sys.argv[2]) 21 | 22 | ips = load_ips('ips.txt') 23 | 24 | n_machines = len(ips) 25 | 26 | assert n_machines == 1, "we are expecting there are 2 machines." 27 | 28 | threads = 12 29 | partition_num = threads 30 | 31 | def print_ycsb(): 32 | 33 | read_write_ratio = 80 34 | zipf = 0.0 35 | keys = 40000 36 | cross_ratio = 0 37 | 38 | # s2pl 39 | for i in range(REPEAT): 40 | cmd = get_cmd_string(machine_id, ips, port + i) 41 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=TwoPL --partition_num={partition_num} --threads={threads} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf}') 42 | 43 | def main(): 44 | # ycsb 45 | print_ycsb() 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /scripts/aws_reordering.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 9 -- Performance on YCSB-A and YCSB-B, and Figure 11 -- Effectiveness of deterministic reordering. 5 | Systems: Kiva, Bohm and Calvin 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 1, "we are expecting there is 1 machine." 26 | 27 | threads = 12 28 | partition_num = threads 29 | batch_size = 1000 30 | 31 | def print_ycsb(): 32 | 33 | read_write_ratio = 80 34 | zipfs = ["0.0", "0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8", "0.9", "0.999"] 35 | keys = 40000 36 | cross_ratio = 100 37 | 38 | 39 | # global key space 40 | 41 | # aria 42 | for zipf in zipfs: 43 | for i in range(REPEAT): 44 | cmd = get_cmd_string(machine_id, ips, port + i) 45 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --global_key_space=True') 46 | 47 | # aria w/o reordering 48 | for zipf in zipfs: 49 | for i in range(REPEAT): 50 | cmd = get_cmd_string(machine_id, ips, port + i) 51 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --aria_reordering=false --global_key_space=True') 52 | 53 | # bohm 54 | for zipf in zipfs: 55 | for i in range(REPEAT): 56 | cmd = get_cmd_string(machine_id, ips, port + i) 57 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Bohm --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --mvcc=True --bohm_single_spin=True --same_batch=False --global_key_space=True') 58 | 59 | # pwv 60 | for zipf in zipfs: 61 | for i in range(REPEAT): 62 | cmd = get_cmd_string(machine_id, ips, port + i) 63 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Pwv --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --same_batch=False --global_key_space=True') 64 | 65 | # pwv dependent 66 | for zipf in zipfs: 67 | for i in range(REPEAT): 68 | cmd = get_cmd_string(machine_id, ips, port + i) 69 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Pwv --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --pwv_ycsb_star=True --same_batch=False --global_key_space=True') 70 | 71 | 72 | # calvin 73 | n_lock_managers = [1, 2, 3, 4, 6] 74 | for n_lock_manager in n_lock_managers: 75 | for zipf in zipfs: 76 | for i in range(REPEAT): 77 | cmd = get_cmd_string(machine_id, ips, port + i) 78 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Calvin --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --lock_manager={n_lock_manager} --replica_group=1 --same_batch=False --global_key_space=True') 79 | 80 | 81 | # ariaFB 82 | n_lock_managers = [1, 2, 3, 4, 6] 83 | for n_lock_manager in n_lock_managers: 84 | for zipf in zipfs: 85 | for i in range(REPEAT): 86 | cmd = get_cmd_string(machine_id, ips, port + i) 87 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=AriaFB --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --ariaFB_lock_manager={n_lock_manager} --global_key_space=True') 88 | 89 | 90 | def main(): 91 | # ycsb 92 | print_ycsb() 93 | 94 | if __name__ == '__main__': 95 | main() 96 | -------------------------------------------------------------------------------- /scripts/aws_reordering_pb.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 9 -- Performance on YCSB-A and YCSB-B, and Figure 11 -- Effectiveness of deterministic reordering. 5 | Systems: PB 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 2, "we are expecting there are 2 machines." 26 | 27 | threads = 12 28 | partition_num = threads 29 | batch_size = 10000 30 | 31 | def print_ycsb(): 32 | 33 | read_write_ratio = 80 34 | zipfs = ["0.0", "0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8", "0.9", "0.999"] 35 | keys = 40000 36 | cross_ratio = 100 37 | 38 | # TwoPL 39 | for zipf in zipfs: 40 | for i in range(REPEAT): 41 | cmd = get_cmd_string(machine_id, ips, port + i) 42 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=TwoPL --partition_num={partition_num} --threads={threads} --partitioner=pb --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --global_key_space=True') 43 | 44 | 45 | def main(): 46 | # ycsb 47 | print_ycsb() 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /scripts/aws_scalability.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 14 -- Scalability of Kiva on YCSB benchmark 5 | ''' 6 | 7 | import sys 8 | 9 | from utility import load_ips 10 | from utility import get_cmd_string 11 | 12 | # repeat experiments for the following times 13 | REPEAT = 5 14 | 15 | assert len(sys.argv) == 3, "this script expects two parameters" 16 | 17 | machine_id = int(sys.argv[1]) 18 | port = int(sys.argv[2]) 19 | 20 | ips = load_ips('ips.txt') 21 | 22 | n_machines = len(ips) 23 | 24 | assert n_machines == 16, "we are expecting there is 16 machines." 25 | 26 | threads = 12 27 | batch_size = 10000 28 | 29 | def print_ycsb(): 30 | 31 | read_write_ratio = 80 32 | zipf = 0.0 33 | keys = 40000 34 | cross_ratios = [0, 1, 5, 10, 20] 35 | n_nodes = [16, 14, 12, 10, 8, 6, 4, 2] 36 | 37 | for n_node in n_nodes: 38 | if machine_id >= n_node: 39 | break 40 | partition_num = threads * n_node 41 | for cross_ratio in cross_ratios: 42 | for i in range(REPEAT): 43 | cmd = get_cmd_string(machine_id, ips[:n_node], port + i) 44 | print(f'./bench_ycsb --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --read_write_ratio={read_write_ratio} --cross_ratio={cross_ratio} --keys={keys} --zipf={zipf} --two_partitions=True') 45 | 46 | def main(): 47 | # ycsb 48 | print_ycsb() 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /scripts/aws_tpcc.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 13 -- Performance of each system on TPC-C 5 | Systems: Aria, AriaFB, Bohm, Calvin, and PWV 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 1, "we are expecting there is 1 machine." 26 | 27 | 28 | threads = 12 29 | batch_size = 500 30 | 31 | def print_tpcc(): 32 | 33 | query = 'mixed' 34 | neworder_dist = 10 35 | payment_dist = 15 36 | partition_nums = [1, 2, 4, 6, 8, 10, 12, 36, 60, 84, 108, 132, 156, 180] 37 | 38 | # aria 39 | for partition_num in partition_nums: 40 | for i in range(REPEAT): 41 | cmd = get_cmd_string(machine_id, ips, port + i) 42 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Aria --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist}') 43 | 44 | # bohm 45 | for partition_num in partition_nums: 46 | for i in range(REPEAT): 47 | cmd = get_cmd_string(machine_id, ips, port + i) 48 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Bohm --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist} --mvcc=True --bohm_single_spin=True --same_batch=False') 49 | 50 | # pwv 51 | for partition_num in partition_nums: 52 | for i in range(REPEAT): 53 | cmd = get_cmd_string(machine_id, ips, port + i) 54 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Pwv --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist} --same_batch=False') 55 | 56 | # calvin 57 | n_lock_managers = [1, 2, 3, 4, 6] 58 | for n_lock_manager in n_lock_managers: 59 | for partition_num in partition_nums: 60 | for i in range(REPEAT): 61 | cmd = get_cmd_string(machine_id, ips, port + i) 62 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=Calvin --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist} --lock_manager={n_lock_manager} --replica_group=1 --same_batch=False') 63 | 64 | # ariaFB 65 | n_lock_managers = [1, 2, 3, 4, 6] 66 | for n_lock_manager in n_lock_managers: 67 | for partition_num in partition_nums: 68 | for i in range(REPEAT): 69 | cmd = get_cmd_string(machine_id, ips, port + i) 70 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=AriaFB --partition_num={partition_num} --threads={threads} --batch_size={batch_size} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist} --same_batch=False --ariaFB_lock_manager={n_lock_manager}') 71 | 72 | def main(): 73 | # tpc-c 74 | print_tpcc() 75 | 76 | if __name__ == '__main__': 77 | main() 78 | -------------------------------------------------------------------------------- /scripts/aws_tpcc_pb.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | ''' 4 | This script is used to generate commands to plot Figure 12 -- Performance of each system on Scaled TPC-C 5 | Systems: Aria, AriaFB, Bohm, Calvin, and PWV 6 | ''' 7 | 8 | import sys 9 | 10 | from utility import load_ips 11 | from utility import get_cmd_string 12 | 13 | # repeat experiments for the following times 14 | REPEAT = 5 15 | 16 | assert len(sys.argv) == 3, "this script expects two parameters" 17 | 18 | machine_id = int(sys.argv[1]) 19 | port = int(sys.argv[2]) 20 | 21 | ips = load_ips('ips.txt') 22 | 23 | n_machines = len(ips) 24 | 25 | assert n_machines == 2, "we are expecting there is 2 machines." 26 | 27 | 28 | threads = 12 29 | partition_num = threads 30 | 31 | def print_tpcc(): 32 | 33 | query = 'mixed' 34 | neworder_dist = 10 35 | payment_dist = 15 36 | partition_nums = [1, 2, 4, 6, 8, 10, 12, 36, 60, 84, 108, 132, 156, 180] 37 | 38 | # s2pl 39 | for partition_num in partition_nums: 40 | for i in range(REPEAT): 41 | cmd = get_cmd_string(machine_id, ips, port + i) 42 | print(f'./bench_tpcc --logtostderr=1 --id={machine_id} --servers="{cmd}" --protocol=TwoPL --partitioner=pb --partition_num={partition_num} --threads={threads} --query={query} --neworder_dist={neworder_dist} --payment_dist={payment_dist}') 43 | 44 | 45 | 46 | def main(): 47 | # tpc-c 48 | print_tpcc() 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /scripts/pretty_print.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | import sys 4 | 5 | assert len(sys.argv) == 4, "this script expects three parameters" 6 | 7 | n_rows = int(sys.argv[1]) 8 | n_columns = int(sys.argv[2]) 9 | file_path = sys.argv[3] 10 | 11 | def main(): 12 | lines = [line.strip() for line in open(file_path, 'r')] 13 | # remove not finished lines 14 | n_blocks = len(lines) // (n_rows * n_columns) 15 | lines = lines[:n_blocks * n_rows * n_columns] 16 | 17 | 18 | data = [[] for i in range(n_columns)] 19 | 20 | # fill in data 21 | cursor = -1 22 | 23 | for i in range(len(lines)): 24 | if i % n_rows == 0: 25 | cursor += 1 26 | 27 | if cursor == n_columns: 28 | cursor = 0 29 | 30 | data[cursor].append(lines[i]) 31 | 32 | # print 33 | 34 | for i in range(len(data[0])): 35 | for j in range(n_columns): 36 | sys.stdout.write(data[j][i]) 37 | sys.stdout.write( '\n' if j == n_columns - 1 else '\t') 38 | 39 | if __name__ == '__main__': 40 | main() 41 | 42 | 43 | -------------------------------------------------------------------------------- /scripts/scp_script.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | import sys 4 | import os 5 | 6 | from utility import load_ips 7 | 8 | assert len(sys.argv) == 3, "this script expects two parameters" 9 | port = int(sys.argv[1]) 10 | script = sys.argv[2] 11 | 12 | ips = load_ips('ips.txt') 13 | n_machines = len(ips) # 8 14 | #assert n_machines == 8, "we are expecting there are 8 machines." 15 | 16 | for i in range(n_machines): 17 | external_ip = ips[i][1] 18 | os.system("python3 %s %d %d > run.sh" % (script, i, port)) 19 | os.system("chmod u+x run.sh") 20 | os.system("scp run.sh ubuntu@%s:~/scar/run.sh" % external_ip) 21 | -------------------------------------------------------------------------------- /scripts/scp_zip.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | import sys 4 | import os 5 | 6 | from utility import load_ips 7 | 8 | assert len(sys.argv) == 2, "this script expects one parameter" 9 | zip_file = sys.argv[1] 10 | 11 | ips = load_ips('ips.txt') 12 | n_machines = len(ips) # 8 13 | # assert n_machines == 8, "we are expecting there are 8 machines." 14 | 15 | for ip in ips: 16 | external_ip = ip[1] 17 | os.system("scp %s ubuntu@%s:~/" %(zip_file, external_ip)) 18 | -------------------------------------------------------------------------------- /scripts/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | 3 | def load_ips(ip_file_path): 4 | ip_file = open(ip_file_path, 'r') 5 | ips = [line.strip().split('\t') for line in ip_file if line.startswith('#') == False] 6 | return ips 7 | 8 | def get_cmd_string(id, ips, port): 9 | n_nodes = len(ips) 10 | assert id < n_nodes, 'id should be less than length of ips' 11 | cmd = '' 12 | for i in range(n_nodes): 13 | cmd += ';' if i > 0 else '' 14 | (internal_ip, external_ip) = ips[i] 15 | if id == i: 16 | cmd += internal_ip + ':' + str(port) 17 | else: 18 | cmd += external_ip + ':' + str(port) 19 | return cmd --------------------------------------------------------------------------------