├── README.md ├── database ├── Meta.cpp ├── Meta.h ├── README ├── README.md ├── bench │ ├── BenchmarkArguments.h │ ├── BenchmarkInitiator.h │ ├── BenchmarkPopulator.h │ └── BenchmarkSource.h ├── scripts │ ├── compile.sh │ ├── experiment.sh │ ├── get_servers.sh │ ├── kill_servers.sh │ ├── result.sh │ ├── run_test.sh │ └── tail_host_log.sh ├── storage │ ├── ColumnInfo.h │ ├── GAMObject.h │ ├── GAddrArray.h │ ├── HashIndex.h │ ├── HashIndexHelper.h │ ├── Record.h │ ├── RecordSchema.h │ ├── Records.h │ ├── StorageManager.h │ └── Table.h ├── test │ ├── HashIndexTest.cpp │ ├── Makefile │ └── TpccPopulateTest.cpp ├── tpcc │ ├── Makefile │ ├── TpccBenchmarkMain.cpp │ ├── TpccConstants.h │ ├── TpccExecutor.h │ ├── TpccInitiator.h │ ├── TpccKeyGenerator.h │ ├── TpccParams.h │ ├── TpccPopulator.h │ ├── TpccProcedure.h │ ├── TpccRandomGenerator.cpp │ ├── TpccRandomGenerator.h │ ├── TpccRecords.h │ ├── TpccSource.h │ ├── TpccTxnParams.h │ └── config.txt ├── txn │ ├── IORedirector.h │ ├── StoredProcedure.h │ ├── TransactionExecutor.h │ ├── TransactionManager.h │ ├── TransactionManager2PL.cpp │ ├── TransactionManagerST.cpp │ ├── TxnAccess.h │ ├── TxnContext.h │ └── TxnParam.h └── utils │ ├── CharArray.h │ ├── ClusterConfig.h │ ├── ClusterHelper.h │ ├── ClusterSync.h │ ├── FastRandom.h │ ├── PerfStatistics.h │ ├── Profiler.cpp │ ├── Profiler.h │ ├── SpinLock.h │ ├── TimeMeasurer.h │ └── Toolkits.h ├── dht ├── Makefile ├── kv.h ├── kv_benchmark.sh ├── kvbench.cc ├── kvclient.cc └── kvclient.h ├── include ├── MurmurHash.h ├── ae.h ├── anet.h ├── cache.h ├── chars.h ├── client.h ├── config.h ├── directory.h ├── gallocator.h ├── garray.h ├── gfunc.h ├── hashtable.h ├── kernel.h ├── locked_unordered_map.h ├── lockwrapper.h ├── log.h ├── logging.h ├── map.h ├── master.h ├── murmur_hasher.h ├── rdma.h ├── server.h ├── settings.h ├── slabs.h ├── structure.h ├── tcp.h ├── util.h ├── worker.h ├── worker_handle.h ├── workrequest.h └── zmalloc.h ├── lib └── libcuckoo │ ├── .gitignore │ ├── LICENSE │ ├── Makefile.am │ ├── README.md │ ├── cityhash-1.1.1 │ ├── COPYING │ ├── Makefile.am │ ├── NEWS │ ├── README │ ├── configure.ac │ ├── m4 │ │ └── .gitkeep │ └── src │ │ ├── Makefile.am │ │ ├── city-test.cc │ │ ├── city.cc │ │ ├── city.h │ │ └── citycrc.h │ ├── configure.ac │ ├── examples │ ├── Makefile.am │ ├── count_freq.cc │ ├── hellohash.cc │ └── nested_table.cc │ ├── m4 │ ├── ax_cxx_compile_stdcxx_11.m4 │ └── ax_pthread.m4 │ ├── src │ ├── Makefile.am │ ├── city_hasher.hh │ ├── cuckoohash_config.hh │ ├── cuckoohash_map.hh │ ├── cuckoohash_util.hh │ ├── default_hasher.hh │ ├── lazy_array.hh │ └── mainpage.dox │ └── tests │ ├── Makefile.am │ ├── benchmarks │ ├── Makefile.am │ ├── benchmark-driver.sh │ ├── insert_throughput.cc │ ├── read_insert_throughput.cc │ └── read_throughput.cc │ ├── configure.ac │ ├── m4 │ └── .gitkeep │ ├── stress-tests │ ├── Makefile.am │ ├── stress_checked.cc │ └── stress_unchecked.cc │ ├── test_util.hh │ └── unit-tests │ ├── Makefile.am │ ├── catch.hpp │ ├── test_bracket_operator.cc │ ├── test_constructor.cc │ ├── test_hash_properties.cc │ ├── test_iterator.cc │ ├── test_maximum_hashpower.cc │ ├── test_minimum_load_factor.cc │ ├── test_noncopyable_types.cc │ ├── test_resize.cc │ ├── test_runner.cc │ ├── unit_test_util.cc │ └── unit_test_util.hh ├── scripts ├── benchmark-all.sh └── slaves ├── src ├── Makefile ├── MurmurHash.cc ├── ae.cc ├── ae_epoll.cc ├── ae_evport.cc ├── ae_kqueue.cc ├── ae_select.cc ├── anet.cc ├── cache.cc ├── client.cc ├── directory.cc ├── gallocator.cc ├── gfunc.cc ├── local_request.cc ├── local_request_cache.cc ├── local_request_nocache.cc ├── log.cc ├── logging.cc ├── master.cc ├── pending_request.cc ├── rdma.cc ├── remote_request.cc ├── remote_request_cache.cc ├── remote_request_nocache.cc ├── server.cc ├── slabs.cc ├── tcp.cc ├── util.cc ├── worker.cc ├── worker_handle.cc ├── workrequest.cc └── zmalloc.cc └── test ├── Makefile ├── benchmark.cc ├── cs_test.cc ├── example-r.cc ├── example.cc ├── fence_test.cc ├── garray_test.cc ├── gfunc_test.cc ├── hashtable_test.cc ├── hashtable_throw_test.cc ├── lock_test.cc ├── lru_test.cc ├── map_test.cc ├── master_test.cc ├── rw_test.cc ├── slab_test.cc └── worker_test.cc /database/Meta.cpp: -------------------------------------------------------------------------------- 1 | #include "Meta.h" 2 | 3 | namespace Database { 4 | GAlloc** gallocators = NULL; 5 | GAlloc* default_gallocator = NULL; 6 | size_t gThreadCount = 0; 7 | size_t gParamBatchSize = 1000; 8 | } 9 | -------------------------------------------------------------------------------- /database/Meta.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_META_H__ 2 | #define __DATABASE_META_H__ 3 | 4 | #include 5 | #include 6 | 7 | #include "gallocator.h" 8 | 9 | namespace Database { 10 | typedef uint32_t HashcodeType; 11 | typedef uint64_t IndexKey; 12 | 13 | enum LockType 14 | : size_t {NO_LOCK, 15 | READ_LOCK, 16 | WRITE_LOCK, 17 | }; 18 | enum AccessType 19 | : size_t {READ_ONLY, 20 | READ_WRITE, 21 | INSERT_ONLY, 22 | DELETE_ONLY 23 | }; 24 | enum SourceType 25 | : size_t {RANDOM_SOURCE, 26 | PARTITION_SOURCE 27 | }; 28 | 29 | // storage 30 | const size_t kMaxTableNum = 16; 31 | const size_t kMaxColumnNum = 32; 32 | const size_t kMaxSecondaryIndexNum = 5; 33 | const uint64_t kHashIndexBucketHeaderNum = 1000007; 34 | // txn 35 | const size_t kTryLockLimit = 1; 36 | const size_t kMaxAccessLimit = 256; 37 | 38 | extern GAlloc* default_gallocator; 39 | extern GAlloc** gallocators; 40 | extern size_t gThreadCount; 41 | 42 | // source 43 | extern size_t gParamBatchSize; 44 | 45 | } 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /database/README: -------------------------------------------------------------------------------- 1 | A transactional database engine built on top of Global addressable memory. 2 | This engine is benchmarked with TPCC. 3 | 4 | # Feature # 5 | Since GAM maintains the cache coherence, i.e., all records to access would be cached locally before the accesses, two phase commit is not needed and the transactions are executed similar to the case of single machines. 6 | Therefore, we use two phase locking protocol for concurrency control 7 | 8 | # Compile # 9 | It is required that the GAM library is built (in 'code/src'). 10 | 11 | To build the tpcc benchmark, just 12 | ``` 13 | cd tpcc; make clean; make -j 14 | ``` 15 | To build the test for tpcc, simply 16 | ``` 17 | cd test; make clean; make -j 18 | ``` 19 | The file 'scripts/compile.sh' automate this compilation process. 20 | After compilation, the binary files are generated in the folder 'tpcc' and 'test' respectively. 21 | 22 | # Flag # 23 | There are two flags used for compilation to enforce different concurrency control options. 24 | * LOCK: two phase locking 25 | * ST: no concurrency control 26 | 27 | # Run # 28 | 1. Specify the cluster setup in a configuration file, say 'config.txt'. 29 | This file should be written in the format as a number of lines of "HOST_NAME PORT_NUMBER". 30 | 31 | Note that the master node should be written in the first line. 32 | The file 'tpcc/config.txt' provides an example for such a configuration file. 33 | 34 | 2. Start the master node with the specified arguments. 35 | 36 | 3. Start the remaining nodes with the specified arguments. 37 | 38 | 39 | The arguments for 'tpcc' are described as follows. 40 | * -p: the port number for this node (required). 41 | * -c: Number of cores (threads) used (required). 42 | * -sf: the scale factors for populating the tpcc benchmark. 43 | 44 | There are two arguments for '-sf'. 45 | The first argument for '-sf' indicates the number of warehouses, while the second argument for '-sf' is to scale the size of the database. 46 | 47 | * -t: Number of transactions run for each thread 48 | * -d: The distributed ratios. 49 | * -f: The configuration file for the cluster setup 50 | * -r: The read ratio, used to modify the workload 51 | * -l: The time locality, used to modify the workload 52 | 53 | You can also simply run 'tpcc' without any arguments to show the description for each argument. 54 | 55 | The folder 'scripts' automate the runnning of tpcc benchmark. 56 | 57 | Under this folder, 'experiment.sh' can run the benchmark under varying distributed ratios, read ratios, and time locality; 58 | 'run_test.sh' can automate the running of our tests. 59 | 60 | Notes: To have a large global memory space, you may need to modify size in 'include/struture.h'. 61 | 62 | 63 | # Acknowledgement # 64 | This implementation is adapted from an open-source transactional database prototype on single machines: 65 | https://github.com/Cavalia/Cavalia 66 | -------------------------------------------------------------------------------- /database/README.md: -------------------------------------------------------------------------------- 1 | A transactional database engine built on top of Global addressable memory. 2 | This engine is benchmarked with TPCC. 3 | 4 | # Feature # 5 | Since GAM maintains the cache coherence, i.e., all records to access would be cached locally before the accesses, two phase commit is not needed and the transactions are executed similar to the case of single machines. 6 | Therefore, we use two phase locking protocol for concurrency control 7 | 8 | # Compile # 9 | It is required that the GAM library is built (in 'code/src'). 10 | 11 | To build the tpcc benchmark, just 12 | ``` 13 | cd tpcc; make clean; make -j 14 | ``` 15 | To build the test for tpcc, simply 16 | ``` 17 | cd test; make clean; make -j 18 | ``` 19 | The file 'scripts/compile.sh' automate this compilation process. 20 | After compilation, the binary files are generated in the folder 'tpcc' and 'test' respectively. 21 | 22 | # Flag # 23 | There are two flags used for compilation to enforce different concurrency control options. 24 | * LOCK: two phase locking 25 | * ST: no concurrency control 26 | 27 | # Run # 28 | 1. Specify the cluster setup in a configuration file, say 'config.txt'. 29 | This file should be written in the format as a number of lines of "HOST_NAME PORT_NUMBER". 30 | 31 | Note that the master node should be written in the first line. 32 | The file 'tpcc/config.txt' provides an example for such a configuration file. 33 | 34 | 2. Start the master node with the specified arguments. 35 | 36 | 3. Start the remaining nodes with the specified arguments. 37 | 38 | 39 | The arguments for 'tpcc' are described as follows. 40 | * -p: the port number for this node (required). 41 | * -c: Number of cores (threads) used (required). 42 | * -sf: the scale factors for populating the tpcc benchmark. 43 | 44 | There are two arguments for '-sf'. 45 | The first argument for '-sf' indicates the number of warehouses, while the second argument for '-sf' is to scale the size of the database. 46 | 47 | * -t: Number of transactions run for each thread 48 | * -d: The distributed ratios. 49 | * -f: The configuration file for the cluster setup 50 | * -r: The read ratio, used to modify the workload 51 | * -l: The time locality, used to modify the workload 52 | 53 | You can also simply run 'tpcc' without any arguments to show the description for each argument. 54 | 55 | The folder 'scripts' automate the runnning of tpcc benchmark. 56 | 57 | Under this folder, 'experiment.sh' can run the benchmark under varying distributed ratios, read ratios, and time locality; 58 | 'run_test.sh' can automate the running of our tests. 59 | 60 | Notes: To have a large global memory space, you may need to modify size in 'include/struture.h'. 61 | 62 | 63 | # Acknowledgement # 64 | This implementation is adapted from an open-source transactional database prototype on single machines: 65 | https://github.com/Cavalia/Cavalia 66 | 67 | The license of Cavalia is attached bellows. 68 | 69 | Copyright (C) 2015-16, School of Computing, National University of Singapore 70 | 71 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 72 | 73 | http://www.apache.org/licenses/LICENSE-2.0 74 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 75 | -------------------------------------------------------------------------------- /database/bench/BenchmarkInitiator.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_BENCHMARK_INITIATOR_H__ 2 | #define __DATABASE_BENCHMARK_INITIATOR_H__ 3 | 4 | #include "gallocator.h" 5 | #include "ClusterHelper.h" 6 | #include "ClusterConfig.h" 7 | #include "StorageManager.h" 8 | #include "Profiler.h" 9 | #include "PerfStatistics.h" 10 | 11 | namespace Database { 12 | class BenchmarkInitiator { 13 | public: 14 | BenchmarkInitiator(const size_t& thread_count, 15 | ClusterConfig* config) 16 | : thread_count_(thread_count), 17 | config_(config) { 18 | } 19 | 20 | void InitGAllocator() { 21 | ServerInfo master = config_->GetMasterHostInfo(); 22 | ServerInfo myhost = config_->GetMyHostInfo(); 23 | 24 | Conf* conf = new Conf(); 25 | conf->loglevel = LOG_WARNING; 26 | conf->size = 1024 * 1024L * 512 * 2 * 16; 27 | conf->is_master = config_->IsMaster(); 28 | conf->master_ip = ClusterHelper::GetIpByHostName(master.addr_); 29 | conf->worker_ip = ClusterHelper::GetIpByHostName(myhost.addr_); 30 | int partition_id = config_->GetMyPartitionId(); 31 | // to avoid port conflicts on the same node 32 | conf->worker_port += partition_id; 33 | 34 | std::cout << "GAllocator config info: is_master=" << conf->is_master 35 | << ",master_ip=" << conf->master_ip << ",master_port=" 36 | << conf->master_port << ",worker_ip=" << conf->worker_ip 37 | << ",worker_port=" << conf->worker_port << std::endl; 38 | 39 | default_gallocator = GAllocFactory::CreateAllocator(conf); 40 | std::cout << "create default gallocator" << std::endl; 41 | gallocators = new GAlloc*[thread_count_]; 42 | for (size_t i = 0; i < thread_count_; ++i) { 43 | gallocators[i] = GAllocFactory::CreateAllocator(conf); 44 | } 45 | } 46 | 47 | GAddr InitStorage() { 48 | GAddr storage_addr = Gnullptr; 49 | int my_partition_id = config_->GetMyPartitionId(); 50 | int partition_num = config_->GetPartitionNum(); 51 | if (config_->IsMaster()) { 52 | // RecordSchema 53 | std::vector schemas; 54 | this->RegisterSchemas(schemas); 55 | // StorageManager 56 | storage_addr = default_gallocator->AlignedMalloc( 57 | StorageManager::GetSerializeSize()); 58 | this->RegisterTables(storage_addr, schemas); 59 | } 60 | return storage_addr; 61 | } 62 | 63 | protected: 64 | virtual void RegisterTables(const GAddr& storage_addr, 65 | const std::vector& schemas) {} 66 | 67 | virtual void RegisterSchemas(std::vector& schemas) {} 68 | 69 | const size_t thread_count_; 70 | ClusterConfig* config_; 71 | }; 72 | } 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /database/bench/BenchmarkPopulator.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_BENCHMARK_POPULATOR_H__ 3 | #define __DATABASE_BENCHMARK_POPULATOR_H__ 4 | 5 | #include 6 | 7 | #include "StorageManager.h" 8 | #include "Meta.h" 9 | #include "TimeMeasurer.h" 10 | 11 | namespace Database { 12 | class BenchmarkPopulator { 13 | public: 14 | BenchmarkPopulator(StorageManager *storage_manager) 15 | : storage_manager_(storage_manager) { 16 | } 17 | 18 | virtual ~BenchmarkPopulator() { 19 | } 20 | 21 | void Start() { 22 | std::cout << "start population" << std::endl; 23 | TimeMeasurer timer; 24 | timer.StartTimer(); 25 | StartPopulate(); 26 | timer.EndTimer(); 27 | std::cout << "populate elapsed time=" << timer.GetElapsedMilliSeconds() 28 | << "ms" << std::endl; 29 | } 30 | 31 | virtual void StartPopulate() = 0; 32 | 33 | private: 34 | BenchmarkPopulator(const BenchmarkPopulator &); 35 | BenchmarkPopulator& operator=(const BenchmarkPopulator &); 36 | 37 | protected: 38 | StorageManager *storage_manager_; 39 | }; 40 | } 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /database/bench/BenchmarkSource.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_BENCHMARK_SOURCE_H__ 3 | #define __DATABASE_BENCHMARK_SOURCE_H__ 4 | 5 | #include 6 | 7 | #include "Meta.h" 8 | #include "IORedirector.h" 9 | #include "TimeMeasurer.h" 10 | 11 | namespace Database { 12 | class BenchmarkSource { 13 | public: 14 | BenchmarkSource(IORedirector* redirector, size_t num_txn, size_t source_type, 15 | size_t thread_count, size_t dist_ratio) 16 | : redirector_ptr_(redirector), 17 | num_txn_(num_txn), 18 | source_type_(source_type), 19 | dist_ratio_(dist_ratio), 20 | thread_count_(thread_count) { 21 | } 22 | ~BenchmarkSource() { 23 | } 24 | 25 | void Start() { 26 | TimeMeasurer timer; 27 | timer.StartTimer(); 28 | StartGeneration(); 29 | timer.EndTimer(); 30 | std::cout << "source elapsed time=" << timer.GetElapsedMilliSeconds() 31 | << "ms" << std::endl; 32 | } 33 | 34 | virtual void StartGeneration() = 0; 35 | 36 | protected: 37 | IORedirector* redirector_ptr_; 38 | const size_t num_txn_; 39 | const size_t dist_ratio_; 40 | const size_t source_type_; 41 | const size_t thread_count_; 42 | }; 43 | } 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /database/scripts/compile.sh: -------------------------------------------------------------------------------- 1 | PROJ_DIR=~/programs/gam/code 2 | GAM_CORE=${PROJ_DIR}/src 3 | TPCC_DIR=${PROJ_DIR}/database/tpcc 4 | TEST_DIR=${PROJ_DIR}/database/test 5 | CUR_DIR=`pwd` 6 | cd ${GAM_CORE} && make clean && make -j && cd ${TPCC_DIR} && make clean && make -j && cd ${TEST_DIR} && make clean && make -j && cd ${CUR_DIR} 7 | #cd ${GAM_CORE} && make clean && make -j && cd ${TPCC_DIR} && make clean && make -j && cd ${CUR_DIR} 8 | -------------------------------------------------------------------------------- /database/scripts/experiment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | # With the specified arguments for benchmark setting, 5 | # this script runs tpcc for varied distributed ratios 6 | 7 | # specify your hosts_file here 8 | # hosts_file specify a list of host names and port numbers, with the host names in the first column 9 | hosts_file="../tpcc/config.txt" 10 | # specify your directory for log files 11 | output_dir="/data/wentian" 12 | 13 | # working environment 14 | proj_dir="~/programs/gam/code" 15 | bin_dir="${proj_dir}/database/tpcc" 16 | script_dir="{proj_dir}/database/scripts" 17 | ssh_opts="-o StrictHostKeyChecking=no" 18 | 19 | hosts_list=`./get_servers.sh ${hosts_file} | tr "\\n" " "` 20 | hosts=(`echo ${hosts_list}`) 21 | master_host=${hosts[0]} 22 | 23 | USER_ARGS="$@" 24 | echo "input Arguments: ${USER_ARGS}" 25 | echo "launch..." 26 | 27 | launch () { 28 | dist_ratio=$1 29 | output_file="${output_dir}/${dist_ratio}_tpcc.log" 30 | script="cd ${bin_dir} && ./tpcc ${USER_ARGS} -d${dist_ratio} > ${output_file} 2>&1" 31 | 32 | echo "start master: ssh ${ssh_opts} ${master_host} "$script" &" 33 | ssh ${ssh_opts} ${master_host} "$script" & 34 | sleep 3 35 | for ((i=1;i<${#hosts[@]};i++)); do 36 | host=${hosts[$i]} 37 | echo "start worker: ssh ${ssh_opts} ${host} "$script" &" 38 | ssh ${ssh_opts} ${host} "$script" & 39 | sleep 1 40 | done 41 | wait 42 | echo "done for ${dist_ratio}" 43 | } 44 | 45 | run_tpcc () { 46 | dist_ratios=(0 10 20 30 40 50 60 70 80 90 100) 47 | for dist_ratio in ${dist_ratios[@]}; do 48 | launch ${dist_ratio} 49 | done 50 | } 51 | 52 | vary_read_ratios () { 53 | #read_ratios=(0 30 50 70 90 100) 54 | read_ratios=(0) 55 | for read_ratio in ${read_ratios[@]}; do 56 | old_user_args=${USER_ARGS} 57 | USER_ARGS="${USER_ARGS} -r${read_ratio}" 58 | run_tpcc 59 | USER_ARGS=${old_user_args} 60 | done 61 | } 62 | 63 | vary_temp_locality () { 64 | #localities=(0 30 50 70 90 100) 65 | localities=(0 50 100) 66 | for locality in ${localities[@]}; do 67 | old_user_args=${USER_ARGS} 68 | USER_ARGS="${USER_ARGS -l${locality}}" 69 | run_tpcc 70 | USER_ARGS=${old_user_args} 71 | done 72 | } 73 | 74 | auto_fill_params () { 75 | # so that users don't need to specify parameters for themselves 76 | USER_ARGS="-p11111 -sf32 -sf10 -c4 -t200000" 77 | } 78 | 79 | auto_fill_params 80 | # run standard tpcc 81 | run_tpcc 82 | 83 | # vary_read_ratios 84 | #vary_temp_locality 85 | -------------------------------------------------------------------------------- /database/scripts/get_servers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Print out the first column of a file 3 | 4 | if [ $# -ne 1 ]; then 5 | echo "USAGE: ./get_servers.h INPUT_FILE" 6 | fi 7 | 8 | input_file=$1 9 | awk '{print $1}' ${input_file} 10 | -------------------------------------------------------------------------------- /database/scripts/kill_servers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | # kill all processes located in HOSTS_FILE 5 | 6 | # specify your hosts_file here 7 | hosts_file="../tpcc/config.txt" 8 | 9 | hosts_list=`./get_servers.sh ${hosts_file} | tr "\\n" " "` 10 | HOSTS=(`echo ${hosts_list}`) 11 | 12 | APP_NAME="tpcc" 13 | #APP_NAME="hash_index_test" 14 | if [ $# -eq 1 ]; then 15 | APP_NAME="$1" 16 | fi 17 | SCRIPT="pkill -f ${APP_NAME}" 18 | SSH_OPTS="-o StrictHostKeyChecking=no" 19 | 20 | #echo "${HOSTS[@]}" 21 | #echo ${APP_NAME}"" 22 | printf "kill " 23 | for HOST_NAME in ${HOSTS[@]} ; do 24 | echo "ssh ${SSH_OPTS} ${HOST_NAME} ${SCRIPT}" 25 | ssh ${SSH_OPTS} ${HOST_NAME} "${SCRIPT}" 26 | printf "." 27 | done 28 | echo "done" 29 | 30 | -------------------------------------------------------------------------------- /database/scripts/result.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script extract the running results of tpcc benchmark and write to OUTFILE 3 | OUTFILE=result.out 4 | INPUTDIR=/data/wentian 5 | INFILE=tpcc.log 6 | 7 | echo "" > $OUTFILE 8 | DIST_RATIO_ARRAY=(0 10 20 30 40 50 60 70 80 90 100) 9 | for DIST_RATIO in ${DIST_RATIO_ARRAY[@]}; do 10 | INPUT=$INPUTDIR/${DIST_RATIO}_$INFILE 11 | echo "INPUT=$INPUT" 12 | awk 'BEGIN {throughput=0; abort_rate=0;} 13 | /total_throughput/{throughput=$2; } 14 | /abort_rate/{abort_rate=$2; } 15 | END {print throughput,"\t", abort_rate; }' $INPUT >> $OUTFILE 16 | done 17 | echo "done" 18 | -------------------------------------------------------------------------------- /database/scripts/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | # specify your hosts_file here 5 | # hosts_file specify a list of host names and port numbers, with the host names in the first column 6 | hosts_file="../tpcc/config.txt" 7 | # specify your directory for log files 8 | output_dir="/data/wentian" 9 | 10 | # working environment 11 | proj_dir="~/programs/gam/code" 12 | bin_dir="${proj_dir}/database/test" 13 | script_dir="{proj_dir}/database/scripts" 14 | ssh_opts="-o StrictHostKeyChecking=no" 15 | bin_file=hash_index_test 16 | 17 | hosts_list=`./get_servers.sh ${hosts_file} | tr "\\n" " "` 18 | hosts=(`echo ${hosts_list}`) 19 | master_host=${hosts[0]} 20 | 21 | USER_ARGS="$@" 22 | echo "input Arguments: ${USER_ARGS}" 23 | echo "launch..." 24 | 25 | run_test () { 26 | output_file="${output_dir}/${bin_file}.log" 27 | script="cd ${bin_dir} && ./${bin_file} ${USER_ARGS} > ${output_file} 2>&1" 28 | 29 | echo "start master: ssh ${ssh_opts} ${master_host} "$script" &" 30 | ssh ${ssh_opts} ${master_host} "$script" & 31 | sleep 3 32 | for ((i=1;i<${#hosts[@]};i++)); do 33 | host=${hosts[$i]} 34 | echo "start worker: ssh ${ssh_opts} ${host} "$script" &" 35 | ssh ${ssh_opts} ${host} "$script" & 36 | sleep 1 37 | done 38 | wait 39 | } 40 | 41 | 42 | auto_fill_params () { 43 | # so that users don't need to specify parameters for themselves 44 | USER_ARGS="-p11111 -sf32 -sf10 -c4 -t200000 -f../tpcc/config.txt" 45 | } 46 | 47 | auto_fill_params 48 | bin_file=hash_index_test 49 | run_test 50 | bin_file=tpcc_populate_test 51 | run_test 52 | -------------------------------------------------------------------------------- /database/scripts/tail_host_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # helper file that can print out the logs of the remote processes 3 | 4 | if [ "$#" -ne 3 ]; then 5 | echo "$0 host_name file_name line_num" 6 | exit -1 7 | fi 8 | 9 | host_name=$1 10 | file_path=$2 11 | line_num=$3 12 | 13 | SCRIPT="tail -${line_num} ${file_path}" 14 | SSH_OPTS="-o StrictHostKeyChecking=no" 15 | 16 | echo "command: ssh ${SSH_OPTS} ${host_name} "${SCRIPT}"" 17 | ssh ${SSH_OPTS} ${host_name} "${SCRIPT}" 18 | -------------------------------------------------------------------------------- /database/storage/ColumnInfo.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_STORAGE_COLUMN_INFO_H__ 3 | #define __DATABASE_STORAGE_COLUMN_INFO_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "Meta.h" 9 | 10 | namespace Database { 11 | enum ValueType 12 | : size_t {INT, 13 | INT8, 14 | INT16, 15 | INT32, 16 | INT64, 17 | DOUBLE, 18 | FLOAT, 19 | VARCHAR, 20 | META 21 | }; 22 | 23 | struct MetaColumn { 24 | bool is_visible_; 25 | // uint64_t timestamp_; 26 | }; 27 | 28 | const size_t kIntSize = sizeof(int); 29 | const size_t kInt8Size = sizeof(int8_t); 30 | const size_t kInt16Size = sizeof(int16_t); 31 | const size_t kInt32Size = sizeof(int32_t); 32 | const size_t kInt64Size = sizeof(int64_t); 33 | const size_t kFloatSize = sizeof(float); 34 | const size_t kDoubleSize = sizeof(double); 35 | const size_t kMetaSize = sizeof(MetaColumn); 36 | 37 | struct ColumnInfo { 38 | ColumnInfo() { 39 | column_size_ = 0; 40 | column_offset_ = 0; 41 | } 42 | ColumnInfo(const char* column_name, const ValueType& column_type) { 43 | Init(column_name, column_type); 44 | } 45 | ColumnInfo(const char* column_name, const ValueType& column_type, 46 | const size_t& column_size) { 47 | Init(column_name, column_type, column_size); 48 | } 49 | ColumnInfo(const ColumnInfo& column_info) { 50 | Copy(column_info); 51 | } 52 | 53 | void Copy(const ColumnInfo& column_info) { 54 | memcpy(column_name_, column_info.column_name_, 16); 55 | column_type_ = column_info.column_type_; 56 | column_size_ = column_info.column_size_; 57 | column_offset_ = column_info.column_offset_; 58 | } 59 | 60 | void Init(const char* column_name, const ValueType &column_type, 61 | const size_t& column_size = 0) { 62 | assert(strlen(column_name) < 16); 63 | size_t len = strlen(column_name); 64 | memcpy(column_name_, column_name, len); 65 | column_name_[len] = '\0'; 66 | column_type_ = column_type; 67 | if (column_type != ValueType::VARCHAR) { 68 | switch (column_type) { 69 | case INT: 70 | column_size_ = kIntSize; 71 | break; 72 | case INT8: 73 | column_size_ = kInt8Size; 74 | break; 75 | case INT16: 76 | column_size_ = kInt16Size; 77 | break; 78 | case INT32: 79 | column_size_ = kInt32Size; 80 | break; 81 | case INT64: 82 | column_size_ = kInt64Size; 83 | break; 84 | case DOUBLE: 85 | column_size_ = kDoubleSize; 86 | break; 87 | case FLOAT: 88 | column_size_ = kFloatSize; 89 | break; 90 | case META: 91 | column_size_ = kMetaSize; 92 | default: 93 | break; 94 | } 95 | } else { 96 | column_size_ = column_size; 97 | assert(column_size_ > 0); 98 | } 99 | 100 | column_offset_ = 0; 101 | } 102 | 103 | char column_name_[16]; 104 | ValueType column_type_; 105 | size_t column_size_; 106 | // used in RecordSchema 107 | size_t column_offset_; 108 | }; 109 | } 110 | 111 | #endif 112 | -------------------------------------------------------------------------------- /database/storage/GAMObject.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_STORAGE_GAM_OBJECT_H__ 2 | #define __DATABASE_STORAGE_GAM_OBJECT_H__ 3 | 4 | #include "gallocator.h" 5 | 6 | namespace Database{ 7 | class GAMObject { 8 | public: 9 | // Write the content to the global memory addr 10 | virtual void Serialize(const GAddr& addr, GAlloc *gallocator) = 0; 11 | // Read the content from the global memory addr 12 | virtual void Deserialize(const GAddr& addr, GAlloc *gallocator) = 0; 13 | }; 14 | } 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /database/storage/Record.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_STORAGE_RECORD_H__ 3 | #define __DATABASE_STORAGE_RECORD_H__ 4 | 5 | #include "GAMObject.h" 6 | #include "CharArray.h" 7 | #include "Meta.h" 8 | #include "RecordSchema.h" 9 | 10 | namespace Database { 11 | class Record : public GAMObject{ 12 | public: 13 | Record(RecordSchema *schema_ptr) : schema_ptr_(schema_ptr) { 14 | data_size_ = schema_ptr_->GetSchemaSize(); 15 | data_ptr_ = new char[data_size_]; 16 | } 17 | ~Record() { 18 | Release(); 19 | } 20 | 21 | void SetColumn(size_t col_id, void *data) { 22 | size_t offset = schema_ptr_->GetColumnOffset(col_id); 23 | size_t col_size = schema_ptr_->GetColumnSize(col_id); 24 | memcpy(data_ptr_ + offset, data, col_size); 25 | } 26 | void SetColumn(size_t col_id, void *data, size_t data_size) { 27 | size_t offset = schema_ptr_->GetColumnOffset(col_id); 28 | memcpy(data_ptr_ + offset, data, data_size); 29 | } 30 | void GetColumn(size_t col_id, void *data) const { 31 | size_t offset = schema_ptr_->GetColumnOffset(col_id); 32 | size_t col_size = schema_ptr_->GetColumnSize(col_id); 33 | memcpy(data, data_ptr_ + offset, col_size); 34 | } 35 | 36 | bool GetVisible() const { 37 | size_t meta_col_id = schema_ptr_->GetMetaColumnId(); 38 | MetaColumn meta_col; 39 | GetColumn(meta_col_id, &meta_col); 40 | return meta_col.is_visible_; 41 | } 42 | void SetVisible(bool val) { 43 | size_t meta_col_id = schema_ptr_->GetMetaColumnId(); 44 | MetaColumn meta_col; 45 | this->GetColumn(meta_col_id, &meta_col); 46 | meta_col.is_visible_ = val; 47 | this->SetColumn(meta_col_id, &meta_col); 48 | } 49 | RecordSchema* GetSchema() { 50 | return schema_ptr_; 51 | } 52 | size_t GetSchemaSize() const { 53 | return schema_ptr_->GetSchemaSize(); 54 | } 55 | 56 | 57 | virtual void Serialize(const GAddr& addr, GAlloc *gallocator) { 58 | gallocator->Write(addr, data_ptr_, data_size_); 59 | } 60 | void Serialize(const GAddr& addr, GAlloc *gallocator, size_t col_id) { 61 | size_t offset = schema_ptr_->GetColumnOffset(col_id); 62 | size_t col_size = schema_ptr_->GetColumnSize(col_id); 63 | gallocator->Write(addr, offset, data_ptr_ + offset, col_size); 64 | } 65 | 66 | virtual void Deserialize(const GAddr& addr, GAlloc *gallocator) { 67 | data_size_ = schema_ptr_->GetSchemaSize(); 68 | if (!data_ptr_) { 69 | data_ptr_ = new char[data_size_]; 70 | } 71 | gallocator->Read(addr, data_ptr_, data_size_); 72 | } 73 | 74 | size_t GetSerializeSize() const { 75 | return schema_ptr_->GetSchemaSize(); 76 | } 77 | 78 | private: 79 | Record(const Record&); 80 | Record& operator=(const Record&); 81 | 82 | void Release() { 83 | if (data_ptr_) { 84 | delete[] data_ptr_; 85 | data_ptr_ = nullptr; 86 | } 87 | data_size_ = 0; 88 | } 89 | 90 | private: 91 | RecordSchema *schema_ptr_; 92 | char *data_ptr_; 93 | size_t data_size_; 94 | }; 95 | } 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /database/storage/Records.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_STORAGE_RECORDS_H__ 3 | #define __DATABASE_STORAGE_RECORDS_H__ 4 | 5 | #include "Record.h" 6 | 7 | namespace Database { 8 | class Records { 9 | public: 10 | Records(size_t max_size) 11 | : max_size_(max_size) { 12 | record_count_ = 0; 13 | records_ = new Record*[max_size_]; 14 | } 15 | ~Records() { 16 | delete[] records_; 17 | records_ = nullptr; 18 | } 19 | void InsertRecord(Record* record) { 20 | assert(record_count_ < max_size_); 21 | records_[record_count_++] = record; 22 | } 23 | void Clear() { 24 | record_count_ = 0; 25 | } 26 | public: 27 | size_t max_size_; 28 | size_t record_count_; 29 | Record **records_; 30 | }; 31 | } 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /database/storage/StorageManager.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_STORAGE_STORAGE_MANAGER_H__ 2 | #define __DATABASE_STORAGE_STORAGE_MANAGER_H__ 3 | 4 | #include 5 | #include 6 | 7 | #include "Table.h" 8 | 9 | namespace Database { 10 | class StorageManager : public GAMObject{ 11 | public: 12 | StorageManager() { 13 | tables_ = nullptr; 14 | table_count_ = 0; 15 | } 16 | ~StorageManager() { 17 | if (tables_) { 18 | assert(table_count_ > 0); 19 | for (size_t i = 0; i < table_count_; ++i) { 20 | delete tables_[i]; 21 | tables_[i] = nullptr; 22 | } 23 | delete[] tables_; 24 | tables_ = nullptr; 25 | } 26 | } 27 | 28 | void RegisterTables(const std::vector& schemas, 29 | GAlloc* gallocator) { 30 | table_count_ = schemas.size(); 31 | assert(table_count_ < kMaxTableNum); 32 | tables_ = new Table*[table_count_]; 33 | for (size_t i = 0; i < table_count_; ++i) { 34 | Table* table = new Table(); 35 | table->Init(i, schemas[i], gallocator); 36 | tables_[i] = table; 37 | } 38 | } 39 | 40 | size_t GetTableCount() const { 41 | return table_count_; 42 | } 43 | 44 | virtual void Serialize(const GAddr& addr, GAlloc *gallocator) { 45 | gallocator->Write(addr, &table_count_, sizeof(size_t)); 46 | GAddr cur_addr = GADD(addr, sizeof(size_t)); 47 | for (size_t i = 0; i < table_count_; ++i) { 48 | tables_[i]->Serialize(cur_addr, gallocator); 49 | cur_addr = GADD(cur_addr, Table::GetSerializeSize()); 50 | } 51 | } 52 | 53 | virtual void Deserialize(const GAddr& addr, GAlloc *gallocator) { 54 | gallocator->Read(addr, &table_count_, sizeof(size_t)); 55 | GAddr cur_addr = GADD(addr, sizeof(size_t)); 56 | tables_ = new Table*[table_count_]; 57 | for (size_t i = 0; i < table_count_; ++i) { 58 | Table* table = new Table(); 59 | table->Deserialize(cur_addr, gallocator); 60 | tables_[i] = table; 61 | cur_addr = GADD(cur_addr, Table::GetSerializeSize()); 62 | } 63 | } 64 | 65 | static size_t GetSerializeSize() { 66 | return sizeof(size_t) + kMaxTableNum * Table::GetSerializeSize(); 67 | } 68 | 69 | public: 70 | Table **tables_; 71 | 72 | private: 73 | size_t table_count_; 74 | }; 75 | } 76 | #endif 77 | -------------------------------------------------------------------------------- /database/storage/Table.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_STORAGE_TABLE_H__ 2 | #define __DATABASE_STORAGE_TABLE_H__ 3 | 4 | #include 5 | 6 | #include "Record.h" 7 | #include "RecordSchema.h" 8 | #include "Meta.h" 9 | #include "HashIndex.h" 10 | #include "Profiler.h" 11 | 12 | namespace Database { 13 | class Table : public GAMObject{ 14 | public: 15 | Table() { 16 | schema_ptr_ = nullptr; 17 | primary_index_ = nullptr; 18 | secondary_indexes_ = nullptr; 19 | } 20 | ~Table() { 21 | if (primary_index_) { 22 | delete primary_index_; 23 | primary_index_ = nullptr; 24 | } 25 | } 26 | 27 | void Init(size_t table_id, RecordSchema* schema_ptr, GAlloc* gallocator) { 28 | table_id_ = table_id; 29 | schema_ptr_ = schema_ptr; 30 | secondary_count_ = 0; 31 | secondary_indexes_ = nullptr; 32 | 33 | primary_index_ = new HashIndex(); 34 | primary_index_->Init(kHashIndexBucketHeaderNum, gallocator); 35 | } 36 | 37 | // return false if the key exists in primary index already 38 | bool InsertRecord(const IndexKey* keys, size_t key_num, 39 | const GAddr& data_addr, GAlloc* gallocator, size_t thread_id) { 40 | assert(key_num == secondary_count_ + 1); 41 | return primary_index_->InsertRecord(keys[0], data_addr, 42 | gallocator, thread_id); 43 | } 44 | 45 | GAddr SearchRecord(const IndexKey& key, GAlloc* gallocator, 46 | size_t thread_id) { 47 | return primary_index_->SearchRecord(key, 48 | gallocator, thread_id); 49 | } 50 | 51 | void ReportTableSize() const { 52 | uint64_t size = primary_index_->GetRecordCount() 53 | * schema_ptr_->GetSchemaSize(); 54 | std::cout << "table_id=" << table_id_ << ", size=" 55 | << size * 1.0 / 1000 / 1000 << "MB" << std::endl; 56 | } 57 | 58 | size_t GetTableId() const { 59 | return table_id_; 60 | } 61 | size_t GetSecondaryCount() const { 62 | return secondary_count_; 63 | } 64 | RecordSchema* GetSchema() { 65 | return schema_ptr_; 66 | } 67 | size_t GetSchemaSize() const { 68 | return schema_ptr_->GetSchemaSize(); 69 | } 70 | HashIndex* GetPrimaryIndex() { 71 | return primary_index_; 72 | } 73 | 74 | virtual void Serialize(const GAddr& addr, GAlloc *gallocator) { 75 | size_t off = 0; 76 | gallocator->Write(addr, off, &table_id_, sizeof(size_t)); 77 | off += sizeof(size_t); 78 | gallocator->Write(addr, off, &secondary_count_, sizeof(size_t)); 79 | off += sizeof(size_t); 80 | GAddr cur_addr = GADD(addr, off); 81 | schema_ptr_->Serialize(cur_addr, gallocator); 82 | cur_addr = GADD(cur_addr, schema_ptr_->GetSerializeSize()) ; 83 | primary_index_->Serialize(cur_addr, gallocator); 84 | } 85 | 86 | virtual void Deserialize(const GAddr& addr, GAlloc *gallocator) { 87 | size_t off = 0; 88 | gallocator->Read(addr, off, &table_id_, sizeof(size_t)); 89 | off += sizeof(size_t); 90 | gallocator->Read(addr, off, &secondary_count_, sizeof(size_t)); 91 | off += sizeof(size_t); 92 | GAddr cur_addr = GADD(addr, off); 93 | schema_ptr_ = new RecordSchema(table_id_); 94 | schema_ptr_->Deserialize(cur_addr, gallocator); 95 | cur_addr = GADD(cur_addr, schema_ptr_->GetSerializeSize()); 96 | primary_index_ = new HashIndex(); 97 | primary_index_->Deserialize(cur_addr, gallocator); 98 | } 99 | 100 | static size_t GetSerializeSize() { 101 | size_t ret = sizeof(size_t) * 2; 102 | ret += RecordSchema::GetSerializeSize(); 103 | ret += HashIndex::GetSerializeSize(); 104 | return ret; 105 | } 106 | 107 | private: 108 | size_t table_id_; 109 | size_t secondary_count_; 110 | 111 | RecordSchema *schema_ptr_; 112 | HashIndex *primary_index_; 113 | HashIndex **secondary_indexes_; // Currently disabled 114 | }; 115 | } 116 | #endif 117 | -------------------------------------------------------------------------------- /database/test/HashIndexTest.cpp: -------------------------------------------------------------------------------- 1 | #include "HashIndex.h" 2 | #include "ClusterHelper.h" 3 | #include "ClusterSync.h" 4 | #include "BenchmarkArguments.h" 5 | #include "BenchmarkInitiator.h" 6 | #include 7 | 8 | using namespace Database; 9 | 10 | // thread sync 11 | volatile bool is_begin = false; 12 | 13 | static uint64_t GetKey(size_t partition_id, size_t thread_id, size_t op_id) { 14 | uint64_t ret = op_id; 15 | ret |= (partition_id << 55) | (thread_id << 45); 16 | return ret; 17 | } 18 | void ThreadExecute(size_t partition_id, size_t thread_id, HashIndex *hash_index) { 19 | while (!is_begin) ; 20 | GAlloc *gallocator = gallocators[thread_id]; 21 | for (int i = 0; i < num_txn; ++i) { 22 | uint64_t key = GetKey(partition_id, thread_id, i); 23 | uint64_t value = key; 24 | GAddr data_addr = gallocator->Malloc(sizeof(uint64_t)); 25 | gallocator->Write(data_addr, &value, sizeof(uint64_t)); 26 | bool ret = hash_index->InsertRecord(key, data_addr, gallocator, thread_id); 27 | assert(ret); 28 | } 29 | } 30 | void ThreadCheck(size_t partition_id, size_t thread_id, HashIndex *hash_index) { 31 | while (!is_begin) ; 32 | GAlloc *gallocator = gallocators[thread_id]; 33 | for (int i = 0; i < num_txn; ++i) { 34 | uint64_t key = GetKey(partition_id, thread_id, i); 35 | uint64_t exp_value = key; 36 | GAddr data_addr = hash_index->SearchRecord( 37 | key, gallocator, thread_id); 38 | uint64_t act_value= 0; 39 | gallocator->Read(data_addr, &act_value, sizeof(uint64_t)); 40 | assert(act_value == exp_value); 41 | } 42 | } 43 | // example command: ./hash_index_test -p11111 -sf1 -sf1 -c4 -t1000000 44 | int main(int argc, char *argv[]) { 45 | ArgumentsParser(argc, argv); 46 | std::string my_host_name = ClusterHelper::GetLocalHostName(); 47 | ClusterConfig config(my_host_name, port, config_filename); 48 | ClusterSync synchronizer(&config); 49 | 50 | BenchmarkInitiator initiator(gThreadCount, &config); 51 | // gam storage 52 | initiator.InitGAllocator(); 53 | synchronizer.Fence(); 54 | // Initialize hash index 55 | GAddr storage_addr = Gnullptr; 56 | if (config.IsMaster()) { 57 | storage_addr = default_gallocator->AlignedMalloc( 58 | HashIndex::GetSerializeSize()); 59 | HashIndex hash_index; 60 | hash_index.Init(kHashIndexBucketHeaderNum, default_gallocator); 61 | hash_index.Serialize(storage_addr, default_gallocator); 62 | } 63 | synchronizer.MasterBroadcast(&storage_addr); 64 | std::cout << "storage_addr=" << storage_addr << std::endl; 65 | HashIndex hash_index; 66 | hash_index.Deserialize(storage_addr, default_gallocator); 67 | synchronizer.Fence(); 68 | std::cout << "start to insert to hash index" << std::endl; 69 | 70 | { 71 | boost::thread_group thread_group; 72 | for (size_t i = 0; i < gThreadCount; ++i) { 73 | thread_group.create_thread( 74 | boost::bind(&ThreadExecute, config.GetMyPartitionId(), i, &hash_index) 75 | ); 76 | } 77 | is_begin = true; 78 | thread_group.join_all(); 79 | } 80 | std::cout << "finish insert to hash index" << std::endl; 81 | synchronizer.Fence(); 82 | 83 | // check 84 | is_begin = false; 85 | synchronizer.Fence(); 86 | std::cout << "start check..." << std::endl; 87 | { 88 | boost::thread_group thread_group; 89 | for (size_t i = 0; i < gThreadCount; ++i) { 90 | thread_group.create_thread( 91 | boost::bind(&ThreadCheck, config.GetMyPartitionId(), i, &hash_index) 92 | ); 93 | } 94 | is_begin = true; 95 | thread_group.join_all(); 96 | } 97 | std::cout << "finish check" << std::endl; 98 | synchronizer.Fence(); 99 | 100 | std::cout << "exit..." << std::endl; 101 | return 0; 102 | } 103 | -------------------------------------------------------------------------------- /database/test/Makefile: -------------------------------------------------------------------------------- 1 | CPP = g++ -std=c++11 2 | GAM_INCLUDE = -I../../include -I../ -I../../lib/libcuckoo/src -I../../lib/libcuckoo/cityhash-1.1.1/src 3 | DB_INCLUDE = -I.. -I../tpcc -I../bench -I../storage -I../txn -I../utils 4 | INCLUDE = $(GAM_INCLUDE) $(DB_INCLUDE) 5 | LIBS = ../../src/libgalloc.a -libverbs -lboost_filesystem -lboost_system -lboost_date_time -lboost_thread ../../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a 6 | CFLAGS += -O3 -g -DLOCK -DBACKOFF #-DPROFILE 7 | 8 | all: tpcc_populate_test hash_index_test 9 | build: tpcc_populate_test hash_index_test 10 | 11 | SM_OBJ = ../tpcc/TpccRandomGenerator.o ../Meta.o ../txn/TransactionManager2PL.o ../txn/TransactionManagerST.o ../utils/Profiler.o 12 | OBJ = $(SM_OBJ) TpccPopulateTest.o HashIndexTest.o 13 | TPCC_POPULATE_OBJ = TpccPopulateTest.o $(SM_OBJ) 14 | HASH_INDEX_OBJ = HashIndexTest.o $(SM_OBJ) 15 | 16 | $(OBJ): %.o : %.cpp 17 | $(CPP) $< $(CFLAGS) $(INCLUDE) -g -c -o $@ 18 | 19 | tpcc_populate_test: $(TPCC_POPULATE_OBJ) 20 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $(TPCC_POPULATE_OBJ) $(LIBS) 21 | 22 | hash_index_test: $(HASH_INDEX_OBJ) 23 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $(HASH_INDEX_OBJ) $(LIBS) 24 | 25 | clean: 26 | rm -rf tpcc_populate_test hash_index_test *.o ../*.o ../bench/*.o ../storage/*.o ../txn/*.o ../utils/*.o 27 | -------------------------------------------------------------------------------- /database/tpcc/Makefile: -------------------------------------------------------------------------------- 1 | CPP = g++ -std=c++11 2 | GAM_INCLUDE = -I../../include -I../ -I../../lib/libcuckoo/src -I../../lib/libcuckoo/cityhash-1.1.1/src 3 | DB_INCLUDE = -I.. -I../bench -I../storage -I../txn -I../utils 4 | INCLUDE = $(GAM_INCLUDE) $(DB_INCLUDE) 5 | LIBS = ../../src/libgalloc.a -libverbs -lboost_filesystem -lboost_system -lboost_date_time -lboost_thread ../../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a 6 | CFLAGS += -O3 -DLOCK -DBACKOFF #-DPROFILE 7 | 8 | all: tpcc 9 | build: tpcc 10 | 11 | SRC = TpccBenchmarkMain.cpp TpccRandomGenerator.cpp ../Meta.cpp ../txn/TransactionManager2PL.cpp ../txn/TransactionManagerST.cpp ../utils/Profiler.cpp 12 | OBJ = TpccBenchmarkMain.o TpccRandomGenerator.o ../Meta.o ../txn/TransactionManager2PL.o ../txn/TransactionManagerST.o ../utils/Profiler.o 13 | 14 | $(OBJ): %.o : %.cpp 15 | $(CPP) $< $(CFLAGS) $(INCLUDE) -g -c -o $@ 16 | 17 | tpcc: $(OBJ) 18 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 19 | 20 | clean: 21 | rm -rf tpcc *.o ../*.o ../bench/*.o ../storage/*.o ../txn/*.o ../utils/*.o 22 | -------------------------------------------------------------------------------- /database/tpcc/TpccBenchmarkMain.cpp: -------------------------------------------------------------------------------- 1 | #include "TpccExecutor.h" 2 | #include "TpccPopulator.h" 3 | #include "TpccSource.h" 4 | #include "TpccInitiator.h" 5 | #include "TpccConstants.h" 6 | #include "Meta.h" 7 | #include "TpccParams.h" 8 | #include "BenchmarkArguments.h" 9 | #include "ClusterHelper.h" 10 | #include "ClusterSync.h" 11 | #include 12 | 13 | using namespace Database::TpccBenchmark; 14 | using namespace Database; 15 | 16 | void ExchPerfStatistics(ClusterConfig* config, 17 | ClusterSync* synchronizer, PerfStatistics* s); 18 | 19 | int main(int argc, char* argv[]) { 20 | ArgumentsParser(argc, argv); 21 | 22 | std::string my_host_name = ClusterHelper::GetLocalHostName(); 23 | ClusterConfig config(my_host_name, port, config_filename); 24 | ClusterSync synchronizer(&config); 25 | FillScaleParams(config); 26 | PrintScaleParams(); 27 | 28 | TpccInitiator initiator(gThreadCount, &config); 29 | // initialize GAM storage layer 30 | initiator.InitGAllocator(); 31 | synchronizer.Fence(); 32 | // initialize benchmark data 33 | GAddr storage_addr = initiator.InitStorage(); 34 | synchronizer.MasterBroadcast(&storage_addr); 35 | std::cout << "storage_addr=" << storage_addr << std::endl; 36 | StorageManager storage_manager; 37 | storage_manager.Deserialize(storage_addr, default_gallocator); 38 | 39 | // populate database 40 | INIT_PROFILE_TIME(gThreadCount); 41 | TpccPopulator populator(&storage_manager, &tpcc_scale_params); 42 | populator.Start(); 43 | REPORT_PROFILE_TIME 44 | (gThreadCount); 45 | synchronizer.Fence(); 46 | 47 | // generate workload 48 | IORedirector redirector(gThreadCount); 49 | size_t access_pattern = 0; 50 | TpccSource sourcer(&tpcc_scale_params, &redirector, num_txn, 51 | SourceType::PARTITION_SOURCE, gThreadCount, dist_ratio, 52 | config.GetMyPartitionId()); 53 | //TpccSource sourcer(&tpcc_scale_params, &redirector, num_txn, SourceType::RANDOM_SOURCE, gThreadCount, dist_ratio); 54 | sourcer.Start(); 55 | synchronizer.Fence(); 56 | 57 | { 58 | // warm up 59 | INIT_PROFILE_TIME(gThreadCount); 60 | TpccExecutor executor(&redirector, &storage_manager, gThreadCount); 61 | executor.Start(); 62 | REPORT_PROFILE_TIME(gThreadCount); 63 | } 64 | synchronizer.Fence(); 65 | 66 | { 67 | // run workload 68 | INIT_PROFILE_TIME(gThreadCount); 69 | TpccExecutor executor(&redirector, &storage_manager, gThreadCount); 70 | executor.Start(); 71 | REPORT_PROFILE_TIME 72 | (gThreadCount); 73 | ExchPerfStatistics(&config, &synchronizer, &executor.GetPerfStatistics()); 74 | } 75 | 76 | std::cout << "prepare to exit..." << std::endl; 77 | synchronizer.Fence(); 78 | std::cout << "over.." << std::endl; 79 | return 0; 80 | } 81 | 82 | void ExchPerfStatistics(ClusterConfig* config, 83 | ClusterSync* synchronizer, PerfStatistics* s) { 84 | PerfStatistics *stats = new PerfStatistics[config->GetPartitionNum()]; 85 | synchronizer->MasterCollect( 86 | s, stats); 87 | synchronizer->MasterBroadcast(stats); 88 | for (size_t i = 0; i < config->GetPartitionNum(); ++i) { 89 | stats[i].Print(); 90 | stats[0].Aggregate(stats[i]); 91 | } 92 | stats[0].PrintAgg(); 93 | delete[] stats; 94 | stats = nullptr; 95 | } 96 | 97 | 98 | -------------------------------------------------------------------------------- /database/tpcc/TpccExecutor.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TPCC_EXECUTOR_H__ 3 | #define __DATABASE_TPCC_EXECUTOR_H__ 4 | 5 | #include "TransactionExecutor.h" 6 | #include "TpccProcedure.h" 7 | 8 | namespace Database { 9 | namespace TpccBenchmark { 10 | class TpccExecutor : public TransactionExecutor { 11 | public: 12 | TpccExecutor(IORedirector* const redirector, 13 | StorageManager *storage_manager, size_t thread_count_) 14 | : TransactionExecutor(redirector, storage_manager, thread_count_) { 15 | } 16 | ~TpccExecutor() { 17 | } 18 | 19 | virtual void PrepareProcedures() { 20 | registers_[TupleType::DELIVERY] = []() { 21 | DeliveryProcedure* procedure = new DeliveryProcedure(); 22 | return procedure; 23 | }; 24 | registers_[TupleType::NEW_ORDER] = []() { 25 | NewOrderProcedure* procedure = new NewOrderProcedure(); 26 | return procedure; 27 | }; 28 | registers_[TupleType::PAYMENT] = []() { 29 | PaymentProcedure* procedure = new PaymentProcedure(); 30 | return procedure; 31 | }; 32 | registers_[TupleType::ORDER_STATUS] = []() { 33 | OrderStatusProcedure* procedure = new OrderStatusProcedure(); 34 | return procedure; 35 | }; 36 | registers_[TupleType::STOCK_LEVEL] = []() { 37 | StockLevelProcedure* procedure = new StockLevelProcedure(); 38 | return procedure; 39 | }; 40 | 41 | deregisters_[TupleType::DELIVERY] = [](StoredProcedure* procedure) { 42 | delete procedure; 43 | procedure = NULL; 44 | }; 45 | deregisters_[TupleType::NEW_ORDER] = [](StoredProcedure* procedure) { 46 | delete procedure; 47 | procedure = NULL; 48 | }; 49 | deregisters_[TupleType::PAYMENT] = [](StoredProcedure* procedure) { 50 | delete procedure; 51 | procedure = NULL; 52 | }; 53 | deregisters_[TupleType::ORDER_STATUS] = [](StoredProcedure* procedure) { 54 | delete procedure; 55 | procedure = NULL; 56 | }; 57 | deregisters_[TupleType::STOCK_LEVEL] = [](StoredProcedure* procedure) { 58 | delete procedure; 59 | procedure = NULL; 60 | }; 61 | } 62 | 63 | }; 64 | } 65 | } 66 | #endif 67 | -------------------------------------------------------------------------------- /database/tpcc/TpccKeyGenerator.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_TPCC_KEY_GENERATOR_H__ 2 | #define __DATABASE_TPCC_KEY_GENERATOR_H__ 3 | 4 | #include "gallocator.h" 5 | #include "Meta.h" 6 | #include "TpccConstants.h" 7 | #include "TpccRecords.h" 8 | #include "TpccParams.h" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace Database { 15 | namespace TpccBenchmark { 16 | /******************** get primary key **********************/ 17 | static IndexKey GetItemPrimaryKey(int i_id, int w_id) { 18 | assert(i_id >= 1 && i_id <= tpcc_scale_params.num_items_); 19 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 20 | | (((IndexKey) ITEM_TABLE_ID) << kTableIdLowBits); 21 | k = k | i_id; 22 | return k; 23 | } 24 | static IndexKey GetWarehousePrimaryKey(int w_id) { 25 | assert(w_id >= 1 && w_id <= tpcc_scale_params.num_warehouses_); 26 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 27 | | (((IndexKey) WAREHOUSE_TABLE_ID) << kTableIdLowBits); 28 | return k; 29 | } 30 | static IndexKey GetDistrictPrimaryKey(int d_id, int d_w_id) { 31 | assert( 32 | d_id >= 1 && d_id <= tpcc_scale_params.num_districts_per_warehouse_ 33 | && d_w_id >= 1 && d_w_id <= tpcc_scale_params.num_warehouses_); 34 | IndexKey k = (((IndexKey) d_w_id) << kWarehouseBits) 35 | | (((IndexKey) DISTRICT_TABLE_ID) << kTableIdLowBits); 36 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 37 | return k; 38 | } 39 | static IndexKey GetCustomerPrimaryKey(int c_id, int c_d_id, int c_w_id) { 40 | assert(c_id >= 1 && c_id <= tpcc_scale_params.num_customers_per_district_); 41 | IndexKey k = (((IndexKey) c_w_id) << kWarehouseBits) 42 | | (((IndexKey) CUSTOMER_TABLE_ID) << kTableIdLowBits); 43 | k = k | (((IndexKey) c_d_id) << kDistrictLowBits); 44 | k = k | c_id; 45 | return k; 46 | } 47 | //TODO: a more feasible encoder for order_key 48 | static IndexKey GetOrderPrimaryKey(int o_id, int o_d_id, int o_w_id) { 49 | IndexKey k = (((IndexKey) o_w_id) << kWarehouseBits) 50 | | (((IndexKey) ORDER_TABLE_ID) << kTableIdLowBits); 51 | k = k | (((IndexKey) o_d_id) << kDistrictLowBits); 52 | k = k | (((IndexKey) o_id) << kOrderIdLowBits); 53 | return k; 54 | } 55 | static IndexKey GetDistrictNewOrderPrimaryKey(int d_id, int w_id) { 56 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 57 | | (((IndexKey) DISTRICT_NEW_ORDER_TABLE_ID) << kTableIdLowBits); 58 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 59 | return k; 60 | } 61 | static IndexKey GetNewOrderPrimaryKey(int o_id, int d_id, int w_id) { 62 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 63 | | (((IndexKey) NEW_ORDER_TABLE_ID) << kTableIdLowBits); 64 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 65 | k = k | (((IndexKey) o_id) << kOrderIdLowBits); 66 | return k; 67 | } 68 | static IndexKey GetOrderLinePrimaryKey(int o_id, int d_id, int w_id, 69 | int ol_no) { 70 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 71 | | (((IndexKey) ORDER_LINE_TABLE_ID) << kTableIdLowBits); 72 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 73 | k = k | (((IndexKey) o_id) << kOrderIdLowBits); 74 | k = k | ol_no; 75 | return k; 76 | } 77 | static IndexKey GetHistoryPrimaryKey(int c_id, int d_id, int w_id) { 78 | // as new HISTORY record inserted, history record may not be unique 79 | // anymore 80 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 81 | | (((IndexKey) HISTORY_TABLE_ID) << kTableIdLowBits); 82 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 83 | k = k | c_id; 84 | return k; 85 | } 86 | static IndexKey GetStockPrimaryKey(int i_id, int w_id) { 87 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 88 | | (((IndexKey) STOCK_TABLE_ID) << kTableIdLowBits); 89 | k = k | i_id; 90 | return k; 91 | } 92 | static IndexKey GetOrderLineSecondaryKey(int o_id, int d_id, int w_id) { 93 | IndexKey k = (((IndexKey) w_id) << kWarehouseBits) 94 | | (((IndexKey) ORDER_LINE_TABLE_ID) << kTableIdLowBits); 95 | k = k | (((IndexKey) d_id) << kDistrictLowBits); 96 | k = k | (((IndexKey) o_id) << kOrderIdLowBits); 97 | return k; 98 | } 99 | 100 | } 101 | } 102 | 103 | #endif 104 | -------------------------------------------------------------------------------- /database/tpcc/TpccParams.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_TPCC_PARAMS_H__ 2 | #define __DATABASE_TPCC_PARAMS_H__ 3 | 4 | #include "Meta.h" 5 | #include "TpccConstants.h" 6 | #include "ClusterConfig.h" 7 | #include "BenchmarkArguments.h" 8 | 9 | #include 10 | 11 | namespace Database { 12 | namespace TpccBenchmark { 13 | 14 | struct TpccScaleParams { 15 | int num_warehouses_; 16 | int starting_warehouse_; 17 | int ending_warehouse_; 18 | int partition_id_; 19 | double scale_factor_; 20 | int num_items_; 21 | int num_districts_per_warehouse_; 22 | int num_customers_per_district_; 23 | int num_new_orders_per_district_; 24 | }; 25 | // Global 26 | TpccScaleParams tpcc_scale_params; 27 | 28 | static void FillScaleParams(ClusterConfig& config) { 29 | assert(factor_count == 2); 30 | tpcc_scale_params.num_warehouses_ = (int) scale_factors[0]; 31 | tpcc_scale_params.partition_id_ = config.GetMyPartitionId(); 32 | int partition_num = config.GetPartitionNum(); 33 | int num_wh_per_par = (tpcc_scale_params.num_warehouses_ + partition_num - 1) 34 | / partition_num; 35 | tpcc_scale_params.starting_warehouse_ = 1 36 | + num_wh_per_par * tpcc_scale_params.partition_id_; 37 | tpcc_scale_params.ending_warehouse_ = tpcc_scale_params.starting_warehouse_ 38 | + num_wh_per_par - 1; 39 | if (tpcc_scale_params.ending_warehouse_ > tpcc_scale_params.num_warehouses_) { 40 | tpcc_scale_params.ending_warehouse_ = tpcc_scale_params.num_warehouses_; 41 | } 42 | assert( 43 | tpcc_scale_params.starting_warehouse_ 44 | <= tpcc_scale_params.ending_warehouse_); 45 | tpcc_scale_params.scale_factor_ = scale_factors[1]; 46 | tpcc_scale_params.num_items_ = static_cast(NUM_ITEMS 47 | / tpcc_scale_params.scale_factor_); 48 | tpcc_scale_params.num_districts_per_warehouse_ = DISTRICTS_PER_WAREHOUSE; 49 | tpcc_scale_params.num_customers_per_district_ = 50 | static_cast(CUSTOMERS_PER_DISTRICT / tpcc_scale_params.scale_factor_); 51 | tpcc_scale_params.num_new_orders_per_district_ = 52 | static_cast(INITIAL_NEW_ORDERS_PER_DISTRICT 53 | / tpcc_scale_params.scale_factor_); 54 | } 55 | 56 | static void PrintScaleParams() { 57 | std::cout << "============= tpcc_scale_params ===========" << std::endl; 58 | std::cout << "num_warehouses=" << tpcc_scale_params.num_warehouses_ 59 | << ",starting_warehouse_=" << tpcc_scale_params.starting_warehouse_ 60 | << ",ending_warehouse_=" << tpcc_scale_params.ending_warehouse_ 61 | << std::endl; 62 | std::cout << "partition_id_" << tpcc_scale_params.partition_id_ 63 | << "\nscale_factor_=" << tpcc_scale_params.scale_factor_ 64 | << "\nnum_items_=" << tpcc_scale_params.num_items_ 65 | << "\nnum_districts_per_warehouse_=" 66 | << tpcc_scale_params.num_districts_per_warehouse_ 67 | << "\nnum_customers_per_district_=" 68 | << tpcc_scale_params.num_customers_per_district_ 69 | << "\nnum_new_orders_per_district_=" 70 | << tpcc_scale_params.num_new_orders_per_district_ << std::endl; 71 | std::cout << "dist_ratio=" << dist_ratio << ", gStandard=" << gStandard 72 | << ", gForceRandomAccess=" << gForceRandomAccess 73 | << ", gTimeLocality=" << gTimeLocality << ", gReadRatio=" 74 | << gReadRatio << std::endl; 75 | std::cout << "============= end ===========" << std::endl; 76 | } 77 | 78 | } 79 | } 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /database/tpcc/TpccRandomGenerator.cpp: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #include "TpccRandomGenerator.h" 3 | 4 | namespace Database { 5 | namespace TpccBenchmark { 6 | const std::string TpccRandomGenerator::syllables_[10] = { "BAR", "OUGHT", 7 | "ABLE", "PRI", "PRES", "ESE", "ANTI", "CALLY", "ATION", "EING" }; 8 | const int TpccRandomGenerator::cLast_ = TpccRandomGenerator::GenerateInteger( 9 | 0, 255); 10 | const int TpccRandomGenerator::cId_ = TpccRandomGenerator::GenerateInteger( 11 | 0, 1023); 12 | const int TpccRandomGenerator::orderlineItemId_ = 13 | TpccRandomGenerator::GenerateInteger(0, 8191); 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /database/tpcc/TpccTxnParams.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TPCC_TXN_PARAMS_H__ 3 | #define __DATABASE_TPCC_TXN_PARAMS_H__ 4 | 5 | #include "CharArray.h" 6 | #include "TpccConstants.h" 7 | #include 8 | 9 | namespace Database { 10 | namespace TpccBenchmark { 11 | 12 | class DeliveryParam : public TxnParam { 13 | public: 14 | DeliveryParam() { 15 | type_ = DELIVERY; 16 | } 17 | virtual ~DeliveryParam() { 18 | } 19 | 20 | public: 21 | int w_id_; 22 | int o_carrier_id_; 23 | int64_t ol_delivery_d_; 24 | //#if defined(SLICE) 25 | // additional parameters 26 | int no_o_ids_[DISTRICTS_PER_WAREHOUSE]; 27 | double sums_[DISTRICTS_PER_WAREHOUSE]; 28 | int c_ids_[DISTRICTS_PER_WAREHOUSE]; 29 | //#endif 30 | }; 31 | 32 | class NewOrderParam : public TxnParam { 33 | public: 34 | NewOrderParam() { 35 | type_ = NEW_ORDER; 36 | } 37 | virtual ~NewOrderParam() { 38 | } 39 | 40 | public: 41 | int w_id_; 42 | int d_id_; 43 | int c_id_; 44 | int64_t o_entry_d_; 45 | size_t ol_cnt_; 46 | int i_ids_[15]; 47 | int i_w_ids_[15]; 48 | int i_qtys_[15]; 49 | 50 | // to change read ratio 51 | size_t item_access_type_[15]; 52 | size_t stock_access_type_[15]; 53 | size_t warehouse_access_type_; 54 | size_t district_access_type_; 55 | size_t customer_access_type_; 56 | 57 | // change inserts to read or write 58 | size_t new_order_access_type_; 59 | size_t order_access_type_; 60 | size_t order_line_access_type_[15]; 61 | 62 | //#if defined(SLICE) 63 | // additional parameters 64 | int next_o_id_; 65 | std::string s_dists_[15]; 66 | double ol_amounts_[15]; 67 | //#endif 68 | }; 69 | 70 | class PaymentParam : public TxnParam { 71 | public: 72 | PaymentParam() { 73 | type_ = PAYMENT; 74 | } 75 | virtual ~PaymentParam() { 76 | } 77 | 78 | public: 79 | int w_id_; 80 | int d_id_; 81 | double h_amount_; 82 | int c_w_id_; 83 | int c_d_id_; 84 | int c_id_; 85 | std::string c_last_; 86 | int64_t h_date_; 87 | 88 | // to modify read ratio 89 | size_t warehouse_access_type_; 90 | size_t district_access_type_; 91 | size_t customer_access_type_; 92 | // change insert to read or write 93 | size_t history_access_type_; 94 | 95 | }; 96 | 97 | class OrderStatusParam : public TxnParam { 98 | public: 99 | OrderStatusParam() { 100 | type_ = ORDER_STATUS; 101 | } 102 | virtual ~OrderStatusParam() { 103 | } 104 | 105 | public: 106 | int w_id_; 107 | int d_id_; 108 | std::string c_last_; 109 | int c_id_; 110 | }; 111 | 112 | class StockLevelParam : public TxnParam { 113 | public: 114 | StockLevelParam() { 115 | type_ = STOCK_LEVEL; 116 | } 117 | virtual ~StockLevelParam() { 118 | } 119 | 120 | public: 121 | int w_id_; 122 | int d_id_; 123 | }; 124 | } 125 | } 126 | 127 | #endif 128 | -------------------------------------------------------------------------------- /database/tpcc/config.txt: -------------------------------------------------------------------------------- 1 | ciidaa-a01 11111 2 | ciidaa-a02 11111 3 | ciidaa-a03 11111 4 | ciidaa-a04 11111 5 | ciidaa-a05 11111 6 | ciidaa-a06 11111 7 | ciidaa-a07 11111 8 | ciidaa-a08 11111 9 | -------------------------------------------------------------------------------- /database/txn/IORedirector.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN_IO_REDIRECTOR_H__ 3 | #define __DATABASE_TXN_IO_REDIRECTOR_H__ 4 | 5 | #include 6 | #include "TxnParam.h" 7 | 8 | namespace Database { 9 | class IORedirector { 10 | public: 11 | IORedirector(const size_t &thread_count) 12 | : thread_count_(thread_count), 13 | curr_thread_id_(0) { 14 | input_batches_ = new std::vector[thread_count]; 15 | } 16 | ~IORedirector() { 17 | delete[] input_batches_; 18 | input_batches_ = NULL; 19 | } 20 | 21 | std::vector *GetParameterBatches() { 22 | return input_batches_; 23 | } 24 | std::vector *GetParameterBatches(const size_t &thread_id) { 25 | return &(input_batches_[thread_id]); 26 | } 27 | void PushParameterBatch(ParamBatch *tuples) { 28 | input_batches_[curr_thread_id_].push_back(tuples); 29 | curr_thread_id_ = (curr_thread_id_ + 1) % thread_count_; 30 | } 31 | 32 | private: 33 | IORedirector(const IORedirector &); 34 | IORedirector& operator=(const IORedirector &); 35 | 36 | protected: 37 | std::vector *input_batches_; 38 | size_t thread_count_; 39 | size_t curr_thread_id_; 40 | }; 41 | } 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /database/txn/StoredProcedure.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN__STORED_PROCEDURE_H__ 3 | #define __DATABASE_TXN__STORED_PROCEDURE_H__ 4 | 5 | #include "StorageManager.h" 6 | #include "TransactionManager.h" 7 | #include "TxnContext.h" 8 | #include "Meta.h" 9 | 10 | namespace Database { 11 | #define DB_QUERY(statement) \ 12 | if (transaction_manager_->statement == false) return false; 13 | 14 | #define DB_QUERY_CALLBACK(statement, callback) \ 15 | if (transaction_manager_->statement == false) { callback; return false; } 16 | 17 | class StoredProcedure { 18 | public: 19 | StoredProcedure() { 20 | context_.txn_type_ = 0; 21 | thread_id_ = 0; 22 | } 23 | StoredProcedure(const size_t &txn_type) { 24 | context_.txn_type_ = txn_type; 25 | thread_id_ = 0; 26 | } 27 | virtual ~StoredProcedure() { 28 | } 29 | 30 | void SetTransactionManager(TransactionManager *transaction_manager) { 31 | transaction_manager_ = transaction_manager; 32 | thread_id_ = transaction_manager_->GetThreadId(); 33 | } 34 | 35 | virtual bool Execute(TxnParam *param, CharArray &ret) { 36 | return true; 37 | } 38 | 39 | private: 40 | StoredProcedure(const StoredProcedure&); 41 | StoredProcedure& operator=(const StoredProcedure&); 42 | 43 | protected: 44 | TxnContext context_; 45 | TransactionManager *transaction_manager_; 46 | size_t thread_id_; 47 | }; 48 | } 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /database/txn/TransactionManager.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN_TRANSACTION_MANAGER_H__ 3 | #define __DATABASE_TXN_TRANSACTION_MANAGER_H__ 4 | 5 | #include 6 | #include 7 | 8 | #include "Meta.h" 9 | #include "StorageManager.h" 10 | #include "Record.h" 11 | #include "Records.h" 12 | #include "TxnParam.h" 13 | #include "CharArray.h" 14 | #include "TxnContext.h" 15 | #include "TxnAccess.h" 16 | #include "Profiler.h" 17 | #include "log.h" 18 | 19 | namespace Database { 20 | class TransactionManager { 21 | public: 22 | TransactionManager(StorageManager *storage_manager, size_t thread_count, 23 | size_t thread_id) 24 | : storage_manager_(storage_manager), 25 | thread_count_(thread_count), 26 | thread_id_(thread_id) { 27 | } 28 | ~TransactionManager() { 29 | } 30 | 31 | bool InsertRecord(TxnContext* context, size_t table_id, const IndexKey* keys, 32 | size_t key_num, Record *record, const GAddr& data_addr); 33 | 34 | bool SearchRecord(TxnContext* context, size_t table_id, 35 | const IndexKey& primary_key, Record *&record, 36 | AccessType access_type) { 37 | PROFILE_TIME_START(thread_id_, INDEX_READ); 38 | GAddr data_addr = storage_manager_->tables_[table_id]->SearchRecord( 39 | primary_key, gallocators[thread_id_], thread_id_); 40 | PROFILE_TIME_END(thread_id_, INDEX_READ); 41 | if (data_addr != Gnullptr) { 42 | bool ret = SelectRecordCC(context, table_id, record, data_addr, 43 | access_type); 44 | return ret; 45 | } else { 46 | epicLog(LOG_WARNING, "table_id=%d cannot find the record with key=%lx", 47 | table_id, primary_key); 48 | return false; 49 | } 50 | } 51 | 52 | bool SearchRecords(TxnContext* context, size_t table_id, size_t index_id, 53 | const IndexKey& secondary_key, Records *records, 54 | AccessType access_type) { 55 | epicLog(LOG_FATAL, "not supported for now"); 56 | return true; 57 | } 58 | 59 | bool CommitTransaction(TxnContext* context, TxnParam* param, 60 | CharArray& ret_str); 61 | 62 | void AbortTransaction(); 63 | 64 | size_t GetThreadId() const { 65 | return thread_id_; 66 | } 67 | 68 | private: 69 | bool SelectRecordCC(TxnContext* context, size_t table_id, 70 | Record *&record, const GAddr &data_addr, 71 | AccessType access_type); 72 | 73 | bool TryWLockRecord(const GAddr& data_addr, size_t schema_size) { 74 | epicLog(LOG_DEBUG, "this=%p, data_addr=%lx, schema_size=%d", 75 | this, data_addr, schema_size); 76 | bool success = true; 77 | size_t try_count = 0; 78 | while (gallocators[thread_id_]->Try_WLock(data_addr, schema_size) != 0) { 79 | if (++try_count >= kTryLockLimit) { 80 | success = false; 81 | break; 82 | } 83 | } 84 | return success; 85 | } 86 | 87 | bool TryRLockRecord(const GAddr& data_addr, size_t schema_size) { 88 | epicLog(LOG_DEBUG, "this=%p, data_addr=%lx, schema_size=%d", 89 | this, data_addr, schema_size); 90 | bool success = true; 91 | size_t try_count = 0; 92 | while (gallocators[thread_id_]->Try_RLock(data_addr, schema_size) != 0) { 93 | if (++try_count >= kTryLockLimit) { 94 | success = false; 95 | break; 96 | } 97 | } 98 | return success; 99 | } 100 | 101 | void UnLockRecord(const GAddr &data_addr, size_t schema_size) { 102 | epicLog(LOG_DEBUG, "this=%p, data_addr=%lx, schema_size=%d", 103 | this, data_addr, schema_size); 104 | gallocators[thread_id_]->UnLock(data_addr, schema_size); 105 | } 106 | 107 | public: 108 | StorageManager* storage_manager_; 109 | protected: 110 | size_t thread_id_; 111 | size_t thread_count_; 112 | 113 | AccessList access_list_; 114 | }; 115 | } 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /database/txn/TransactionManagerST.cpp: -------------------------------------------------------------------------------- 1 | #if defined(ST) 2 | #include "TransactionManager.h" 3 | 4 | namespace Database { 5 | bool TransactionManager::InsertRecord(TxnContext* context, size_t table_id, const IndexKey* keys, 6 | size_t key_num, Record *record, const GAddr& record_addr) { 7 | Access *access = access_list_.NewAccess(); 8 | access->access_type_ = INSERT_ONLY; 9 | access->access_record_ = record; 10 | access->access_addr_ = record_addr; 11 | return true; 12 | } 13 | 14 | bool TransactionManager::SelectRecordCC(TxnContext* context, size_t table_id, 15 | Record *&record, const GAddr& record_addr, AccessType access_type) { 16 | RecordSchema *schema_ptr = storage_manager_->tables_[table_id]->GetSchema(); 17 | record = new Record(schema_ptr); 18 | record->Deserialize(record_addr, gallocators[thread_id_]); 19 | Access *access = access_list_.NewAccess(); 20 | access->access_type_ = access_type; 21 | access->access_record_ = record; 22 | access->access_addr_ = record_addr; 23 | return true; 24 | } 25 | 26 | bool TransactionManager::CommitTransaction(TxnContext* context, TxnParam* param, CharArray& ret_str) { 27 | for (size_t i = 0; i < access_list_.access_count_; ++i) { 28 | Access *access = access_list_.GetAccess(i); 29 | if (access->access_type_ == INSERT_ONLY) { 30 | gallocators[thread_id_]->Free(access->access_addr_); 31 | } 32 | delete access->access_record_; 33 | access->access_record_ = nullptr; 34 | access->access_addr_ = Gnullptr; 35 | } 36 | access_list_.Clear(); 37 | return true; 38 | } 39 | 40 | void TransactionManager::AbortTransaction() { 41 | assert(false); 42 | } 43 | } 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /database/txn/TxnAccess.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN_TXN_ACCESS_H__ 3 | #define __DATABASE_TXN_TXN_ACCESS_H__ 4 | 5 | #include "Record.h" 6 | #include "gallocator.h" 7 | 8 | namespace Database { 9 | struct Access { 10 | Access() 11 | : access_record_(nullptr), access_addr_(Gnullptr) { 12 | } 13 | AccessType access_type_; 14 | Record *access_record_; 15 | GAddr access_addr_; 16 | }; 17 | 18 | template 19 | class AccessList { 20 | public: 21 | AccessList() 22 | : access_count_(0) { 23 | } 24 | 25 | Access *NewAccess() { 26 | assert(access_count_ < N); 27 | Access *ret = &(accesses_[access_count_]); 28 | ++access_count_; 29 | return ret; 30 | } 31 | 32 | Access *GetAccess(const size_t &index) { 33 | return &(accesses_[index]); 34 | } 35 | 36 | void Clear() { 37 | access_count_ = 0; 38 | } 39 | 40 | public: 41 | size_t access_count_; 42 | private: 43 | Access accesses_[N]; 44 | }; 45 | } 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /database/txn/TxnContext.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN_TXN_CONTEXT_H__ 3 | #define __DATABASE_TXN_TXN_CONTEXT_H__ 4 | 5 | namespace Database { 6 | struct TxnContext { 7 | TxnContext() 8 | : txn_type_(0), 9 | is_ready_only_(false), 10 | is_dependent_(false) { 11 | } 12 | size_t txn_type_; 13 | bool is_ready_only_; 14 | bool is_dependent_; 15 | }; 16 | } 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /database/txn/TxnParam.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_TXN_TXN_PARAM_H__ 3 | #define __DATABASE_TXN_TXN_PARAM_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "CharArray.h" 9 | #include "Meta.h" 10 | 11 | namespace Database { 12 | class TxnParam { 13 | public: 14 | TxnParam() { 15 | } 16 | virtual ~TxnParam() { 17 | } 18 | 19 | public: 20 | size_t type_; 21 | }; 22 | 23 | class ParamBatch { 24 | public: 25 | ParamBatch() { 26 | params_ = new TxnParam*[gParamBatchSize]; 27 | param_count_ = 0; 28 | batch_size_ = gParamBatchSize; 29 | } 30 | ParamBatch(const size_t &batch_size) { 31 | params_ = new TxnParam*[batch_size]; 32 | param_count_ = 0; 33 | batch_size_ = batch_size; 34 | } 35 | ~ParamBatch() { 36 | delete[] params_; 37 | params_ = NULL; 38 | } 39 | 40 | void push_back(TxnParam *tuple) { 41 | assert(param_count_ < batch_size_); 42 | params_[param_count_] = tuple; 43 | ++param_count_; 44 | } 45 | 46 | size_t size() const { 47 | return param_count_; 48 | } 49 | 50 | TxnParam* get(const size_t idx) const { 51 | return params_[idx]; 52 | } 53 | 54 | private: 55 | TxnParam **params_; 56 | size_t param_count_; 57 | size_t batch_size_; 58 | }; 59 | 60 | struct ParamPtrWrapper { 61 | size_t part_id_; 62 | TxnParam *param_; 63 | }; 64 | 65 | class ParamBatchWrapper { 66 | public: 67 | ParamBatchWrapper() { 68 | params_ = new ParamPtrWrapper[gParamBatchSize]; 69 | param_count_ = 0; 70 | batch_size_ = gParamBatchSize; 71 | } 72 | ParamBatchWrapper(const size_t &batch_size) { 73 | params_ = new ParamPtrWrapper[batch_size]; 74 | param_count_ = 0; 75 | batch_size_ = batch_size; 76 | } 77 | ~ParamBatchWrapper() { 78 | delete[] params_; 79 | params_ = NULL; 80 | } 81 | 82 | void push_back(TxnParam *tuple, const size_t &part_id) { 83 | assert(param_count_ < batch_size_); 84 | params_[param_count_].param_ = tuple; 85 | params_[param_count_].part_id_ = part_id; 86 | ++param_count_; 87 | } 88 | 89 | size_t size() const { 90 | return param_count_; 91 | } 92 | 93 | ParamPtrWrapper* get(const size_t idx) const { 94 | return &(params_[idx]); 95 | } 96 | 97 | private: 98 | ParamPtrWrapper *params_; 99 | size_t param_count_; 100 | size_t batch_size_; 101 | }; 102 | } 103 | #endif 104 | -------------------------------------------------------------------------------- /database/utils/CharArray.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_CHAR_ARRAY_H__ 3 | #define __DATABASE_UTILS_CHAR_ARRAY_H__ 4 | 5 | #include 6 | #include 7 | 8 | struct CharArray { 9 | CharArray() 10 | : size_(0), 11 | char_ptr_(NULL) { 12 | } 13 | size_t size_; 14 | char *char_ptr_; 15 | 16 | void HardCopy(const CharArray &char_array) { 17 | size_ = char_array.size_; 18 | char_ptr_ = new char[size_]; 19 | memcpy(char_ptr_, char_array.char_ptr_, size_); 20 | } 21 | 22 | void SoftCopy(const CharArray &char_array) { 23 | size_ = char_array.size_; 24 | char_ptr_ = char_array.char_ptr_; 25 | } 26 | 27 | void Memcpy(const size_t &offset, const CharArray &char_array) const { 28 | memcpy(char_ptr_ + offset, char_array.char_ptr_, char_array.size_); 29 | } 30 | 31 | void Memcpy(const size_t &offset, const char *char_ptr, 32 | const size_t &size) const { 33 | memcpy(char_ptr_ + offset, char_ptr, size); 34 | } 35 | 36 | void Memset(const size_t &offset, const int &value, 37 | const size_t &size) const { 38 | memset(char_ptr_ + offset, value, size); 39 | } 40 | 41 | void Allocate(const size_t &size) { 42 | size_ = size; 43 | char_ptr_ = new char[size_]; 44 | memset(char_ptr_, 0, size_); 45 | } 46 | 47 | void Release() { 48 | size_ = 0; 49 | if (char_ptr_ != NULL) { 50 | delete[] char_ptr_; 51 | char_ptr_ = NULL; 52 | } 53 | } 54 | 55 | void Clear() { 56 | size_ = 0; 57 | memset(char_ptr_, 0, size_); 58 | } 59 | }; 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /database/utils/ClusterConfig.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_UTILS_CLUSTER_CONFIG_H__ 2 | #define __DATABASE_UTILS_CLUSTER_CONFIG_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace Database { 9 | struct ServerInfo { 10 | ServerInfo(const std::string& addr, const int port) 11 | : addr_(addr), 12 | port_no_(port) { 13 | } 14 | ServerInfo() { 15 | } 16 | 17 | std::string addr_; 18 | int port_no_; 19 | }; 20 | 21 | class ClusterConfig { 22 | public: 23 | ClusterConfig(const std::string& my_host_name, const int port, 24 | const std::string& config_filename) 25 | : my_info_(my_host_name, port), config_filename_(config_filename) { 26 | this->ReadConfigFile(); 27 | } 28 | ~ClusterConfig() { 29 | } 30 | 31 | ServerInfo GetMyHostInfo() const { 32 | return my_info_; 33 | } 34 | 35 | ServerInfo GetMasterHostInfo() const { 36 | return server_info_.at(0); 37 | } 38 | 39 | size_t GetPartitionNum() const { 40 | return server_info_.size(); 41 | } 42 | 43 | size_t GetMyPartitionId() const { 44 | for (size_t i = 0; i < server_info_.size(); ++i) { 45 | ServerInfo host = server_info_.at(i); 46 | if (host.addr_ == my_info_.addr_ && host.port_no_ == my_info_.port_no_) { 47 | return i; 48 | } 49 | } 50 | return server_info_.size() + 1; 51 | } 52 | 53 | bool IsMaster() const { 54 | ServerInfo my = GetMyHostInfo(); 55 | ServerInfo master = GetMasterHostInfo(); 56 | return my.addr_ == master.addr_ && my.port_no_ == master.port_no_; 57 | } 58 | 59 | private: 60 | void ReadConfigFile() { 61 | std::string name; 62 | int port; 63 | 64 | std::ifstream readfile(config_filename_); 65 | assert(readfile.is_open() == true); 66 | while (!readfile.eof()) { 67 | name = ""; 68 | port = -1; 69 | readfile >> name >> port; 70 | if (name == "" && port < 0) 71 | continue; 72 | server_info_.push_back(ServerInfo(name, port)); 73 | } 74 | readfile.close(); 75 | 76 | for (auto& entry : server_info_) { 77 | std::cout << "server name=" << entry.addr_ << ",port_no=" 78 | << entry.port_no_ << std::endl; 79 | } 80 | } 81 | private: 82 | std::vector server_info_; 83 | ServerInfo my_info_; 84 | std::string config_filename_; 85 | }; 86 | } 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /database/utils/ClusterHelper.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_UTILS_CLUSTER_HELPER_H__ 2 | #define __DATABASE_UTILS_CLUSTER_HELPER_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | class ClusterHelper { 12 | public: 13 | static std::string GetLocalHostName() { 14 | char hostname[256]; 15 | hostname[255] = '\0'; 16 | gethostname(hostname, 255); 17 | return std::string(hostname, strlen(hostname)); 18 | } 19 | 20 | static std::string GetIpByHostName(const std::string& hostname) { 21 | hostent* record = gethostbyname(hostname.c_str()); 22 | in_addr* addr = (in_addr*) record->h_addr; 23 | std::string ret = inet_ntoa(*addr); 24 | return ret; 25 | } 26 | }; 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /database/utils/ClusterSync.h: -------------------------------------------------------------------------------- 1 | #ifndef __DATABASE_UTILS_CLUSTER_SYNCHRONIZER_H__ 2 | #define __DATABASE_UTILS_CLUSTER_SYNCHRONIZER_H__ 3 | 4 | #include "gallocator.h" 5 | #include "ClusterConfig.h" 6 | 7 | namespace Database { 8 | class ClusterSync{ 9 | public: 10 | ClusterSync(ClusterConfig *config) : config_(config) { 11 | sync_key_ = 0; 12 | } 13 | 14 | void Fence() { 15 | size_t partition_id = config_->GetMyPartitionId(); 16 | size_t partition_num = config_->GetPartitionNum(); 17 | bool *flags = new bool[partition_num]; 18 | memset(flags, 0, sizeof(bool)*partition_num); 19 | this->MasterCollect(flags + partition_id, flags); 20 | this->MasterBroadcast(flags + partition_id); 21 | delete[] flags; 22 | flags = nullptr; 23 | } 24 | 25 | template 26 | void MasterCollect(T *send, T *receive) { 27 | T data; 28 | size_t partition_id = config_->GetMyPartitionId(); 29 | size_t partition_num = config_->GetPartitionNum(); 30 | if (config_->IsMaster()) { 31 | for (size_t i = 0; i < partition_num; ++i) { 32 | if (i != partition_id) { 33 | default_gallocator->Get( 34 | (uint64_t)(sync_key_ + i), &data); 35 | memcpy(receive + i, &data, sizeof(T)); 36 | } 37 | else { 38 | memcpy(receive + i, send, sizeof(T)); 39 | } 40 | } 41 | } 42 | else { 43 | default_gallocator->Put((uint64_t) 44 | (sync_key_ + partition_id), send, sizeof(T)); 45 | } 46 | sync_key_ += partition_num; 47 | } 48 | 49 | template 50 | void MasterBroadcast(T *send) { 51 | size_t partition_id = config_->GetMyPartitionId(); 52 | size_t partition_num = config_->GetPartitionNum(); 53 | if (config_->IsMaster()) { 54 | default_gallocator->Put( 55 | (uint64_t)(sync_key_ + partition_id), send, sizeof(T)); 56 | } 57 | else { 58 | const size_t master_partition_id = 0; 59 | default_gallocator->Get((uint64_t) 60 | (sync_key_ + master_partition_id), send); 61 | } 62 | sync_key_ += partition_num; 63 | } 64 | 65 | private: 66 | ClusterConfig *config_; 67 | uint64_t sync_key_; 68 | }; 69 | } 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /database/utils/FastRandom.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_FAST_RANDOM_H__ 3 | #define __DATABASE_UTILS_FAST_RANDOM_H__ 4 | 5 | #include 6 | #include 7 | class fast_random { 8 | public: 9 | fast_random(unsigned long seed) 10 | : seed(0) { 11 | set_seed0(seed); 12 | } 13 | 14 | inline unsigned long next() { 15 | return ((unsigned long) next(32) << 32) + next(32); 16 | } 17 | 18 | inline uint32_t next_u32() { 19 | return next(32); 20 | } 21 | 22 | inline uint16_t next_u16() { 23 | return (uint16_t) next(16); 24 | } 25 | 26 | /** [0.0, 1.0) */ 27 | inline double next_uniform() { 28 | return (((unsigned long) next(26) << 27) + next(27)) / (double) (1L << 53); 29 | } 30 | 31 | inline char next_char() { 32 | return next(8) % 256; 33 | } 34 | 35 | inline char next_readable_char() { 36 | static const char readables[] = 37 | "0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; 38 | return readables[next(6)]; 39 | } 40 | 41 | inline std::string next_string(size_t len) { 42 | std::string s(len, 0); 43 | for (size_t i = 0; i < len; i++) 44 | s[i] = next_char(); 45 | return s; 46 | } 47 | 48 | inline std::string next_readable_string(size_t len) { 49 | std::string s(len, 0); 50 | for (size_t i = 0; i < len; i++) 51 | s[i] = next_readable_char(); 52 | return s; 53 | } 54 | 55 | inline unsigned long get_seed() { 56 | return seed; 57 | } 58 | 59 | inline void set_seed(unsigned long seed) { 60 | this->seed = seed; 61 | } 62 | 63 | private: 64 | inline void set_seed0(unsigned long seed) { 65 | this->seed = (seed ^ 0x5DEECE66DL) & ((1L << 48) - 1); 66 | } 67 | 68 | inline unsigned long next(unsigned int bits) { 69 | seed = (seed * 0x5DEECE66DL + 0xBL) & ((1L << 48) - 1); 70 | return (unsigned long) (seed >> (48 - bits)); 71 | } 72 | 73 | unsigned long seed; 74 | }; 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /database/utils/PerfStatistics.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_PERFORMANCE_STATISTICS_H__ 3 | #define __DATABASE_UTILS_PERFORMANCE_STATISTICS_H__ 4 | 5 | #include 6 | #include 7 | 8 | namespace Database { 9 | struct PerfStatistics { 10 | PerfStatistics() { 11 | total_count_ = 0; 12 | total_abort_count_ = 0; 13 | thread_count_ = 0; 14 | elapsed_time_ = 0; 15 | throughput_ = 0.0; 16 | 17 | agg_total_count_ = 0; 18 | agg_thread_count_ = 0; 19 | agg_total_abort_count_ = 0; 20 | agg_elapsed_time_ = 0; 21 | agg_node_num_ = 0; 22 | longest_elapsed_time_ = 0; 23 | agg_throughput_ = 0.0; 24 | } 25 | void PrintAgg() { 26 | std::cout 27 | << "==================== perf statistics summary ====================" 28 | << std::endl; 29 | double abort_rate = agg_total_abort_count_ * 1.0 / (agg_total_count_ + 1); 30 | printf( 31 | "agg_total_count\t%lld\nagg_total_abort_count\t%lld\nabort_rate\t%lf\n", 32 | agg_total_count_, agg_total_abort_count_, abort_rate); 33 | printf( 34 | "per_node_elapsed_time\t%lf\ntotal_throughput\t%lf\nper_node_throughput\t%lf\nper_core_throughput\t%lf\n", 35 | agg_elapsed_time_ * 1.0 / agg_node_num_, agg_throughput_, 36 | agg_throughput_ / agg_node_num_, agg_throughput_ / agg_thread_count_); 37 | 38 | /*std::cout << "agg_total_count=" << agg_total_count_ <<", agg_total_abort_count=" << agg_total_abort_count_ <<", abort_rate=" << abort_rate << std::endl; 39 | std::cout << "per node elapsed time=" << agg_elapsed_time_ * 1.0 / agg_node_num_ << "ms." << std::endl; 40 | std::cout << "total throughput=" << agg_throughput_ << "K tps,per node throughput=" 41 | << agg_throughput_ / agg_node_num_ << "K tps." << ",per core throughput=" << agg_throughput_ / agg_thread_count_ << std::endl;*/ 42 | std::cout << "==================== end ====================" << std::endl; 43 | } 44 | void Print() { 45 | std::cout << "total_count=" << total_count_ << ",total_abort_count=" 46 | << total_abort_count_ << ",throughput=" << throughput_ 47 | << ",thread_count_=" << thread_count_ << ",elapsed_time=" 48 | << elapsed_time_ << std::endl; 49 | } 50 | void Aggregate(const PerfStatistics& obj) { 51 | agg_total_count_ += obj.total_count_; 52 | agg_total_abort_count_ += obj.total_abort_count_; 53 | agg_throughput_ += obj.throughput_; 54 | agg_thread_count_ += obj.thread_count_; 55 | agg_elapsed_time_ += obj.elapsed_time_; 56 | agg_node_num_++; 57 | } 58 | 59 | long long total_count_; 60 | long long total_abort_count_; 61 | long long thread_count_; 62 | long long elapsed_time_; // in milli seconds 63 | double throughput_; 64 | 65 | long long agg_total_count_; 66 | long long agg_total_abort_count_; 67 | double agg_throughput_; 68 | long long agg_thread_count_; 69 | long long agg_elapsed_time_; 70 | long long agg_node_num_; 71 | long long longest_elapsed_time_; 72 | }; 73 | } 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /database/utils/Profiler.cpp: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #include "Profiler.h" 3 | 4 | namespace Database { 5 | TimeMeasurer** profiler_timers = NULL; 6 | long long** profile_elapsed_time = NULL; 7 | } 8 | -------------------------------------------------------------------------------- /database/utils/Profiler.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_PROFILER_H__ 3 | #define __DATABASE_UTILS_PROFILER_H__ 4 | 5 | #include "TimeMeasurer.h" 6 | #include 7 | #include 8 | 9 | namespace Database { 10 | enum PhaseType 11 | : size_t { 12 | INDEX_READ, 13 | INDEX_INSERT, 14 | INDEX_INSERT_LOCK, 15 | INDEX_INSERT_MUTATE, 16 | INDEX_INSERT_GALLOCATE, 17 | CC_SELECT, 18 | CC_INSERT, 19 | CC_COMMIT, 20 | CC_ABORT, 21 | TXN_ABORT, 22 | TXN_EXECUTE, 23 | LOCK_READ_TEST, 24 | LOCK_WRITE_TEST, 25 | POPULATE_DISK, 26 | POPULATE_GALLOCATE, 27 | POPULATE_INSERT, 28 | kPhaseCount 29 | }; 30 | 31 | const static std::string phase_type_string[kPhaseCount] = { "INDEX_READ", 32 | "INDEX_INSERT", "INDEX_INSERT_LOCK", "INDEX_INSERT_MUTATE", 33 | "INDEX_INSERT_GALLOCATE", "CC_SELECT", "CC_INSERT", "CC_COMMIT", "CC_ABORT", 34 | "TXN_ABORT", "TXN_EXECUTE", "LOCK_READ", "LOCK_WRITE", "POPULATE_DISK", 35 | "POPULATE_GALLOCATE", "POPULATE_INSERT" }; 36 | 37 | extern TimeMeasurer** profiler_timers; 38 | extern long long** profile_elapsed_time; 39 | } 40 | 41 | #if defined(PROFILE) 42 | #define INIT_PROFILE_TIME(thread_count) \ 43 | profiler_timers = new TimeMeasurer*[thread_count]; \ 44 | profile_elapsed_time = new long long*[thread_count]; \ 45 | for (int i = 0; i < thread_count; ++i){ \ 46 | profiler_timers[i] = new TimeMeasurer[kPhaseCount]; \ 47 | profile_elapsed_time[i] = new long long[kPhaseCount]; \ 48 | memset(profile_elapsed_time[i], 0, sizeof(long long) * kPhaseCount); \ 49 | } 50 | 51 | #define PROFILE_TIME_START(thread_id, phase_id) \ 52 | profiler_timers[thread_id][phase_id].StartTimer(); 53 | 54 | #define PROFILE_TIME_END(thread_id, phase_id) \ 55 | profiler_timers[thread_id][phase_id].EndTimer(); \ 56 | profile_elapsed_time[thread_id][phase_id] += profiler_timers[thread_id][phase_id].GetElapsedNanoSeconds(); 57 | 58 | #define REPORT_PROFILE_TIME(thread_count)\ 59 | std::cout << "************** profile time *****************" << std::endl; \ 60 | for (int i = 0; i < thread_count; ++i){ \ 61 | std::cout << "thread_id=" << i << ": "; \ 62 | for (int j = 0; j < kPhaseCount; ++j){ \ 63 | std::cout << phase_type_string[j] << "=" << profile_elapsed_time[i][j] * 1.0 / 1000.0 / 1000.0 << "ms,";\ 64 | } \ 65 | std::cout << std::endl; \ 66 | } \ 67 | std::cout << "************** end profile time *****************" << std::endl; 68 | #else 69 | #define INIT_PROFILE_TIME(thread_count) ; 70 | #define PROFILE_TIME_START(thread_id, phase_id) ; 71 | #define PROFILE_TIME_END(thread_id, phase_id) ; 72 | #define REPORT_PROFILE_TIME ; 73 | #endif 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /database/utils/SpinLock.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_SPIN_LOCK_H__ 3 | #define __DATABASE_UTILS_SPIN_LOCK_H__ 4 | 5 | #if defined(PTHREAD_LOCK) || defined(BUILTIN_LOCK) 6 | #include 7 | #else 8 | #include 9 | #include 10 | #include 11 | #endif 12 | 13 | #if defined(PTHREAD_LOCK) 14 | struct SpinLock { 15 | SpinLock() {} 16 | 17 | void Init() { 18 | pthread_spin_init(&spinlock_, PTHREAD_PROCESS_PRIVATE); 19 | } 20 | 21 | inline void Lock() { 22 | pthread_spin_lock(&spinlock_); 23 | } 24 | 25 | inline void Unlock() { 26 | pthread_spin_unlock(&spinlock_); 27 | } 28 | 29 | private: 30 | pthread_spinlock_t spinlock_; 31 | }; 32 | 33 | #elif defined(BUILTIN_LOCK) 34 | struct SpinLock { 35 | SpinLock() {} 36 | 37 | void Init() { 38 | spinlock_ = 0; 39 | } 40 | 41 | inline void Lock() { 42 | while(__sync_lock_test_and_set(&spinlock_, 1)) { 43 | while(spinlock_); 44 | } 45 | } 46 | 47 | inline void Unlock() { 48 | __asm__ __volatile__("" ::: "memory"); 49 | spinlock_ = 0; 50 | } 51 | 52 | private: 53 | volatile bool spinlock_; 54 | }; 55 | 56 | #else 57 | struct SpinLock { 58 | SpinLock() { 59 | } 60 | 61 | void Init() { 62 | memset(&spinlock_, 0, sizeof(spinlock_)); 63 | } 64 | 65 | inline void Lock() { 66 | spinlock_.lock(); 67 | } 68 | 69 | inline void Unlock() { 70 | spinlock_.unlock(); 71 | } 72 | 73 | inline bool IsLocked() const { 74 | return spinlock_.v_ == 1; 75 | } 76 | 77 | private: 78 | boost::detail::spinlock spinlock_; 79 | }; 80 | 81 | #endif 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /database/utils/TimeMeasurer.h: -------------------------------------------------------------------------------- 1 | // NOTICE: this file is adapted from Cavalia 2 | #ifndef __DATABASE_UTILS_TIME_MEASURER_H__ 3 | #define __DATABASE_UTILS_TIME_MEASURER_H__ 4 | 5 | #include 6 | using std::chrono::high_resolution_clock; 7 | using std::chrono::system_clock; 8 | using std::chrono::milliseconds; 9 | using std::chrono::microseconds; 10 | using std::chrono::nanoseconds; 11 | 12 | class TimeMeasurer { 13 | public: 14 | TimeMeasurer() { 15 | } 16 | ~TimeMeasurer() { 17 | } 18 | 19 | void StartTimer() { 20 | start_time_ = high_resolution_clock::now(); 21 | } 22 | 23 | void EndTimer() { 24 | end_time_ = high_resolution_clock::now(); 25 | } 26 | 27 | long long GetElapsedMilliSeconds() { 28 | return std::chrono::duration_cast(end_time_ - start_time_) 29 | .count(); 30 | } 31 | 32 | long long GetElapsedMicroSeconds() { 33 | return std::chrono::duration_cast(end_time_ - start_time_) 34 | .count(); 35 | } 36 | 37 | long long GetElapsedNanoSeconds() { 38 | return std::chrono::duration_cast(end_time_ - start_time_) 39 | .count(); 40 | } 41 | 42 | static system_clock::time_point GetTimePoint() { 43 | return high_resolution_clock::now(); 44 | } 45 | 46 | static long long CalcMilliSecondDiff(system_clock::time_point &start, 47 | system_clock::time_point &end) { 48 | return std::chrono::duration_cast(end - start).count(); 49 | } 50 | 51 | private: 52 | TimeMeasurer(const TimeMeasurer&); 53 | TimeMeasurer& operator=(const TimeMeasurer&); 54 | 55 | private: 56 | system_clock::time_point start_time_; 57 | system_clock::time_point end_time_; 58 | }; 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /dht/Makefile: -------------------------------------------------------------------------------- 1 | CPP = g++ 2 | INCLUDE = -I../include -I../lib/libcuckoo/src -I../lib/libcuckoo/cityhash-1.1.1/src 3 | LIBS = ../src/libgalloc.a -libverbs -lpthread ../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a -lboost_thread -lboost_system 4 | #LIBS = ../src/libgalloc.a -libverbs -lpthread ../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a /users/zhongle/local/lib/libboost_thread.so /users/zhongle/local/lib/libboost_system.so 5 | CFLAGS += -rdynamic -std=c++11 -O3 -DDHT 6 | 7 | %.o: %.cc 8 | $(CPP) $(CFLAGS) $(INCLUDE) -c -o $@ $^ 9 | 10 | benchmark: kvbench.o kvclient.o 11 | cd ../src; make clean; make dht=1 -j; cd ../dht 12 | $(CPP) -o $@ $^ $(LIBS) 13 | 14 | clean: 15 | rm *.o benchmark 16 | 17 | -------------------------------------------------------------------------------- /dht/kv.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef KV_H_ 4 | #define KV_H_ 5 | 6 | #include "../include/structure.h" 7 | #include "chars.h" 8 | #include "../include/log.h" 9 | 10 | namespace dht{ 11 | 12 | using hash_t = uint64_t; 13 | using hkey_t = char*; 14 | 15 | #define HASH_TABLE_NOT_EXIST 1 16 | 17 | #define BKT_SIZE 128 18 | #define ENTRY_SIZE 12 // 12bit tag + 20bit size + 64bit value 19 | #define BKT_SLOT (BKT_SIZE / ENTRY_SIZE) 20 | 21 | #define TAG_BITS 11 22 | #define TAG_MASK ((1UL << TAG_BITS) - 1) 23 | #define TAG(hash) (hash & TAG_MASK) 24 | 25 | #define BKT_BITS 24 26 | #define NBKT (1UL << BKT_BITS) 27 | #define BKT_MASK ((1UL << BKT_BITS) - 1) 28 | #define BUCKET(hash) ((hash >> TAG_BITS) & BKT_MASK) 29 | 30 | #define TBL_BITS (64 - TAG_BITS - BKT_BITS) 31 | #define TBL(hash) ((hash >> (64 - TBL_BITS))) 32 | 33 | #define SZ_BITS 20 34 | #define SZ_MASK ((1 << SZ_BITS) - 1) 35 | 36 | #define MAX_KEY_SIZE 1024 37 | #define MAX_VAL_SIZE 16384 38 | #define MAX_KV_SIZE (8 + MAX_KEY_SIZE + MAX_VAL_SIZE) 39 | 40 | static void parseEntry(char* entry, uint32_t& tag, uint32_t& size, GAddr& addr) { 41 | readInteger(entry, tag, addr); 42 | tag &= ((1UL << 31) - 1); 43 | size = (tag & SZ_MASK); 44 | tag = (tag >> SZ_BITS); 45 | } 46 | 47 | static void updateEntry(char* entry, uint32_t tag, uint32_t size, GAddr addr) { 48 | uint32_t etag = (1UL << 31) | (tag << SZ_BITS) | (size & SZ_MASK); 49 | appendInteger(entry, etag, addr); 50 | } 51 | 52 | //static int getTag(char* entry, uint32_t& tag) { 53 | // readInteger(entry, tag); 54 | // tag &= ((1UL << 31 ) - 1); 55 | // return (tag & (1UL<<31)); 56 | //} 57 | 58 | static bool matchTag(char* entry, uint32_t tag) { 59 | uint32_t etag; 60 | readInteger(entry, etag); 61 | if (etag == 0) return false; 62 | etag &= ((1UL<<31)-1); 63 | etag = (etag >> SZ_BITS); 64 | return tag == etag; 65 | } 66 | 67 | static bool containTag(char* entry) { 68 | uint32_t tag; 69 | readInteger(entry, tag); 70 | return tag != 0; 71 | } 72 | 73 | }; 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /dht/kv_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | bench='/user/caiqc/gam-release/dht/benchmark' 3 | master="ciidaa-a05" 4 | log_dir="/user/caiqc/log" 5 | 6 | mk_dat_dir() { 7 | for ((id = 5; id <= 20; id++)); do 8 | if ((id < 10)); then 9 | node="ciidaa-a0"$id 10 | else 11 | node="ciidaa-a"$id 12 | fi 13 | ssh $node "if [ ! -d $log_dir ]; then mkdir -p $log_dir; fi" 14 | done 15 | } 16 | 17 | kill_all() { 18 | for ((id = 13; id <= 20; id++)); do 19 | if ((id < 10)); then 20 | node="ciidaa-a0"$id 21 | else 22 | node="ciidaa-a"$id 23 | fi 24 | 25 | ssh $node "sudo killall benchmark" 26 | done 27 | sleep 1 28 | } 29 | 30 | run_client() { 31 | local nc=$1 32 | local nt=$2 33 | local ratio=$3 34 | local cid=0 35 | is_master=1 36 | for ((id = 21 - $nc; id <= 20; id++)); do 37 | node="ciidaa-a"$id 38 | log_file="$log_dir/$node"_"$nc"_"$nt"_"$ratio"_"$cid".dat 39 | if [ "$is_master" -eq 1 ]; then 40 | master=$node 41 | fi 42 | echo "run client at $node with master $master" 43 | if [ "$cid" -lt "$(($nc - 1))" ]; then 44 | cmd="ssh $node \"$bench --is_master $is_master --ip_master $master --ip_worker $node --no_client $nc --get_ratio $ratio --no_thread $nt --client_id $cid 1>$log_file 2>/dev/null &\"" 45 | eval $cmd 46 | is_master=0 47 | else 48 | cmd="ssh $node \"$bench --is_master $is_master --ip_master $master --ip_worker $node --no_client $nc --get_ratio $ratio --no_thread $nt --client_id $cid | tee $log_file \"" 49 | eval $cmd 50 | fi 51 | ((cid++)) 52 | sleep 1 53 | done 54 | } 55 | 56 | 57 | clients=8 58 | ratios=(100 99 90 50 0) 59 | #mk_dat_dir 60 | for ((thread = 1; thread<=1; thread++)); do 61 | for ratio in "${ratios[@]}"; do 62 | kill_all 63 | echo "run benchmark with $clients clients $thread threads and $ratio get_ratio " 64 | run_client $clients $thread $ratio 65 | done 66 | done 67 | -------------------------------------------------------------------------------- /dht/kvclient.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef KV_CLIENT_H_ 4 | #define KV_CLIENT_H_ 5 | 6 | #include "kv.h" 7 | #include 8 | 9 | class GAlloc; 10 | 11 | namespace dht{ 12 | 13 | struct kv{ 14 | uint32_t klen; 15 | uint32_t vlen; 16 | char* key; 17 | char* value; 18 | hash_t hkey; 19 | 20 | char* base(){ 21 | return key - sizeof(klen) - sizeof(vlen); 22 | } 23 | 24 | uint32_t size() { 25 | return klen + vlen + sizeof(klen) + sizeof(vlen); 26 | } 27 | 28 | uint64_t hash() { 29 | return hkey; 30 | } 31 | 32 | kv() = delete; 33 | 34 | kv(uint32_t klen, uint32_t vlen, char* key, char* value, char* kv){ 35 | this->klen = klen; 36 | this->vlen = vlen; 37 | char* p = kv; 38 | p += appendInteger(p, klen, vlen); 39 | this->key = p; 40 | this->value = p + klen; 41 | strncpy(this->key, key, klen); 42 | strncpy(this->value, value, vlen); 43 | hkey = std::stoull(std::string(key)); 44 | } 45 | 46 | kv(char* kv) { 47 | kv += readInteger(kv, klen, vlen); 48 | epicAssert(klen >= 0 && vlen >= 0); 49 | key = kv; 50 | value = kv + klen; 51 | } 52 | }; 53 | 54 | class kvClient{ 55 | private: 56 | std::vector htables_; 57 | GAlloc* allocator_; 58 | int klen_; 59 | int vlen_; 60 | char bkt_[BKT_SIZE]; 61 | char kv_[MAX_KV_SIZE]; 62 | 63 | GAddr getBktAddr(hash_t); 64 | public: 65 | kvClient(GAlloc* alloc); 66 | int put(char*, char*); 67 | int get(hash_t, kv**); 68 | int del(hash_t); 69 | }; 70 | }; 71 | #endif 72 | 73 | -------------------------------------------------------------------------------- /include/MurmurHash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * MurmurHash.h 3 | * MYUtilities 4 | * 5 | * This file created by Jens Alfke on 3/17/08. 6 | * Algorithm & source code by Austin Appleby, released to public domain. 7 | * 8 | * 9 | */ 10 | 11 | #include 12 | #include 13 | 14 | /** An extremely efficient general-purpose hash function. 15 | Murmurhash is claimed to be more than twice as fast as the nearest competitor, 16 | and to offer better-distributed output with fewer collisions. 17 | It is, however not suitable for cryptographic use. 18 | Hash values will differ between bit- and little-endian CPUs, so they shouldn't 19 | be stored persistently or transmitted over the network. 20 | 21 | Written by Austin Appleby: */ 22 | 23 | uint32_t MurmurHash2(const void * key, size_t len, uint32_t seed); 24 | 25 | -------------------------------------------------------------------------------- /include/anet.h: -------------------------------------------------------------------------------- 1 | /* anet.c -- Basic TCP socket stuff made a bit less boring 2 | * 3 | * Copyright (c) 2006-2012, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef ANET_H 32 | #define ANET_H 33 | 34 | #define ANET_OK 0 35 | #define ANET_ERR -1 36 | #define ANET_ERR_LEN 256 37 | 38 | /* Flags used with certain functions. */ 39 | #define ANET_NONE 0 40 | #define ANET_IP_ONLY (1<<0) 41 | 42 | #if defined(__sun) || defined(_AIX) 43 | #define AF_LOCAL AF_UNIX 44 | #endif 45 | 46 | #ifdef _AIX 47 | #undef ip_len 48 | #endif 49 | 50 | #include 51 | 52 | int anetTcpConnect(char *err, char *addr, int port); 53 | int anetTcpNonBlockConnect(char *err, char *addr, int port); 54 | int anetTcpNonBlockBindConnect(char *err, char *addr, int port, 55 | char *source_addr); 56 | int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, 57 | char *source_addr); 58 | int anetUnixConnect(char *err, char *path); 59 | int anetUnixNonBlockConnect(char *err, char *path); 60 | int anetRead(int fd, char *buf, int count); 61 | int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); 62 | int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); 63 | int anetTcpServer(char *err, int port, char *bindaddr, int backlog); 64 | int anetTcp6Server(char *err, int port, char *bindaddr, int backlog); 65 | int anetUnixServer(char *err, char *path, mode_t perm, int backlog); 66 | int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, 67 | int *port); 68 | int anetUnixAccept(char *err, int serversock); 69 | int anetWrite(int fd, char *buf, int count); 70 | int anetNonBlock(char *err, int fd); 71 | int anetBlock(char *err, int fd); 72 | int anetEnableTcpNoDelay(char *err, int fd); 73 | int anetDisableTcpNoDelay(char *err, int fd); 74 | int anetTcpKeepAlive(char *err, int fd); 75 | int anetSendTimeout(char *err, int fd, long long ms); 76 | int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); 77 | int anetKeepAlive(char *err, int fd, int interval); 78 | int anetSockName(int fd, char *ip, size_t ip_len, int *port); 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /include/chars.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | 5 | #ifndef CHARS_H 6 | #define CHARS_H 7 | 8 | static inline int appendInteger(char* buf) {return 0;} 9 | static inline int readInteger(char* buf) {return 0;} 10 | 11 | template::value || std::is_pointer::value>::type> 13 | static int appendInteger(char* buf, Type1 value, Types ... values) { 14 | *(Type1*) buf = value; 15 | return sizeof(Type1) + appendInteger(buf + sizeof(Type1), values...); 16 | } 17 | 18 | template::value || std::is_pointer::value>::type> 20 | static int readInteger(char* buf, Type1& value, Types&... values) { 21 | value = *(Type1*) buf; 22 | return sizeof(Type1) + readInteger(buf + sizeof(Type1), values...); 23 | } 24 | 25 | #endif 26 | 27 | -------------------------------------------------------------------------------- /include/client.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef CLIENT_H 4 | #define CLIENT_H 5 | 6 | #include 7 | 8 | #include "lockwrapper.h" 9 | #include "rdma.h" 10 | #include "structure.h" 11 | 12 | class Server; 13 | 14 | //TODO: consider to replace Client by RdmaContext 15 | class Client { 16 | private: 17 | RdmaContext *ctx; /* remote client */ 18 | 19 | RdmaResource* resource; 20 | char* connstr = nullptr; 21 | 22 | //remote worker info 23 | /* 24 | * when connected to master, wid is its own wid 25 | * otherwise, it's the id of the remote pair 26 | */ 27 | int wid = 0; 28 | Size size = 0; 29 | Size free = 0; 30 | 31 | LockWrapper lock_; 32 | 33 | public: 34 | Client(RdmaResource* res, bool isForMaster, 35 | const char *rdmaConnStr = nullptr); 36 | 37 | //used only among workers 38 | int ExchConnParam(const char* ip, int port, Server* server); 39 | const char* GetConnString(int workerid = 0); 40 | int SetRemoteConnParam(const char *conn); 41 | 42 | inline bool IsForMaster() { 43 | return ctx->IsMaster(); 44 | } 45 | inline int GetWorkerId() { 46 | return wid; 47 | } 48 | inline void SetMemStat(int size, int free) { 49 | this->size = size; 50 | this->free = free; 51 | } 52 | inline Size GetFreeMem() { 53 | return this->free; 54 | } 55 | inline Size GetTotalMem() { 56 | return this->size; 57 | } 58 | inline void* ToLocal(GAddr addr) { 59 | return TO_LOCAL(addr, ctx->GetBase()); 60 | } 61 | inline GAddr ToGlobal(void* ptr) { 62 | if (ptr) { 63 | return TO_GLOB(ptr, ctx->GetBase(), wid); 64 | } else { 65 | return EMPTY_GLOB(wid); 66 | } 67 | } 68 | 69 | inline uint32_t GetQP() { 70 | return ctx->GetQP(); 71 | } 72 | 73 | inline ssize_t Send(const void* buf, size_t len, unsigned int id = 0, 74 | bool signaled = false) { 75 | return ctx->Send(buf, len, id, signaled); 76 | } 77 | inline ssize_t Write(raddr dest, raddr src, size_t len, unsigned int id = 0, 78 | bool signaled = false) { 79 | return ctx->Write(dest, src, len, id, signaled); 80 | } 81 | inline ssize_t WriteWithImm(raddr dest, raddr src, size_t len, uint32_t imm, 82 | unsigned int id = 0, bool signaled = false) { 83 | return ctx->WriteWithImm(dest, src, len, imm, id, signaled); 84 | } 85 | 86 | inline int PostRecv(int n) { 87 | return ctx->PostRecv(n); 88 | } 89 | 90 | inline char* GetFreeSlot() { 91 | return ctx->GetFreeSlot(); 92 | } 93 | inline char* RecvComp(ibv_wc& wc) { 94 | return ctx->RecvComp(wc); 95 | } 96 | inline unsigned int SendComp(ibv_wc& wc) { 97 | return ctx->SendComp(wc); 98 | } 99 | inline unsigned int WriteComp(ibv_wc& wc) { 100 | return ctx->WriteComp(wc); 101 | } 102 | 103 | //below are used for multithread 104 | inline void lock() { 105 | epicLog(LOG_INFO, "trying to lock client %d", GetWorkerId()); 106 | lock_.lock(); 107 | epicLog(LOG_INFO, "locked client %d", GetWorkerId()); 108 | } 109 | inline void unlock() { 110 | lock_.unlock(); 111 | epicLog(LOG_INFO, "unlock client %d", GetWorkerId()); 112 | } 113 | 114 | ~Client(); 115 | }; 116 | #endif 117 | -------------------------------------------------------------------------------- /include/gfunc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_GFUNC_H_ 5 | #define INCLUDE_GFUNC_H_ 6 | 7 | #include 8 | #include "settings.h" 9 | #ifdef GFUNC_SUPPORT 10 | 11 | typedef void GFunc(void* addr, uint64_t arg); 12 | void Incr(void* ptr, uint64_t); 13 | void IncrDouble(void* ptr, uint64_t); 14 | 15 | // register graph engine GFunc, which will be applied in homenode 16 | void GatherPagerank(void *ptr, uint64_t); 17 | void ApplyPagerank(void *ptr, uint64_t); 18 | void ScatterPagerank(void *ptr, uint64_t); 19 | 20 | int GetGFuncID(GFunc* gfunc); 21 | GFunc* GetGFunc(int id); 22 | 23 | #endif 24 | 25 | #endif /* INCLUDE_GFUNC_H_ */ 26 | -------------------------------------------------------------------------------- /include/kernel.h: -------------------------------------------------------------------------------- 1 | // adapted from linux kernel 2 | 3 | #ifndef LINUX_KERNEL_H_ 4 | #define LINUX_KERNEL_H_ 5 | 6 | #define likely(x) __builtin_expect(!!(x), 1) 7 | #define unlikely(x) __builtin_expect(!!(x), 0) 8 | 9 | #define __ALIGN_MASK(x,mask) (((x)+(mask))&~(mask)) 10 | #define ALIGN(x,a) __ALIGN_MASK(x,(a)-1) 11 | 12 | #ifndef roundup 13 | # define roundup(x, y) ((((x) + ((y) - 1)) / (y)) * (y)) 14 | #endif /* !defined(roundup) */ 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /include/locked_unordered_map.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef INCLUDE_LOCKEDUNORDEREDMAP_H_ 4 | #define INCLUDE_LOCKEDUNORDEREDMAP_H_ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "../lib/libcuckoo/src/cuckoohash_map.hh" 12 | #include "lockwrapper.h" 13 | 14 | using std::mutex; 15 | using std::array; 16 | using std::string; 17 | using std::exception; 18 | 19 | template 20 | class UnorderedMap : public std::unordered_map { 21 | public: 22 | UnorderedMap(string name = "DEFAULT_HASHTABLE_NAME") { 23 | this->name = name; 24 | } 25 | void lock(Key key); 26 | void unlock(Key key); 27 | 28 | private: 29 | LockWrapper lock_; 30 | string name; 31 | }; 32 | 33 | template 34 | inline void UnorderedMap::lock(Key key) { 35 | // epicLog(LOG_DEBUG, "trying to lock %s", name.c_str()); 36 | // lock_.lock(); 37 | // epicLog(LOG_DEBUG, "locked %s", name.c_str()); 38 | } 39 | 40 | template 41 | inline void UnorderedMap::unlock(Key key) { 42 | // epicLog(LOG_DEBUG, "trying to lock %s", name.c_str()); 43 | // lock_.unlock(); 44 | // epicLog(LOG_DEBUG, "locked %s", name.c_str()); 45 | } 46 | 47 | #endif 48 | 49 | -------------------------------------------------------------------------------- /include/lockwrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_LOCKWRAPPER_H_ 5 | #define INCLUDE_LOCKWRAPPER_H_ 6 | 7 | #include 8 | #include "settings.h" 9 | 10 | class LockWrapper { 11 | #ifdef USE_ATOMIC 12 | bool lock_ = 0; 13 | #else 14 | std::mutex lock_; 15 | #endif 16 | 17 | public: 18 | inline void lock() { 19 | #ifdef USE_ATOMIC 20 | while (__atomic_test_and_set(&lock_, __ATOMIC_RELAXED)) 21 | ; 22 | #else 23 | lock_.lock(); 24 | #endif 25 | } 26 | 27 | inline void unlock() { 28 | #ifdef USE_ATOMIC 29 | __atomic_clear(&lock_, __ATOMIC_RELAXED); 30 | #else 31 | lock_.unlock(); 32 | #endif 33 | } 34 | 35 | inline bool try_lock() { 36 | #ifdef USE_ATOMIC 37 | return !__atomic_test_and_set(&lock_, __ATOMIC_RELAXED); 38 | #else 39 | return lock_.try_lock(); 40 | #endif 41 | } 42 | 43 | }; 44 | 45 | #endif /* INCLUDE_LOCKWRAPPER_H_ */ 46 | -------------------------------------------------------------------------------- /include/log.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_LOG_H_ 5 | #define INCLUDE_LOG_H_ 6 | 7 | #include 8 | 9 | #define LOG_FATAL 0 10 | #define LOG_WARNING 1 11 | #define LOG_INFO 2 12 | #define LOG_DEBUG 3 13 | 14 | #define MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ 15 | 16 | #if defined __cplusplus && __GNUC_PREREQ (2,95) 17 | # define __ASSERT_VOID_CAST static_cast 18 | #else 19 | # define __ASSERT_VOID_CAST (void) 20 | #endif 21 | 22 | void _epicLog(char* file, char *func, int lineno, int level, const char *fmt, ...); 23 | void PrintStackTrace(); 24 | 25 | //#ifdef NDEBUG 26 | //#define epicLog(level, fmt, ...) (__ASSERT_VOID_CAST (0)) 27 | //#else 28 | #define epicLog(level, fmt, ...) _epicLog ((char*)__FILE__, (char*)__func__, __LINE__, level, fmt, ## __VA_ARGS__) 29 | //#endif 30 | 31 | #ifdef NDEBUG 32 | #define epicAssert(_e) (__ASSERT_VOID_CAST (0)) 33 | #else 34 | #define epicAssert(_e) ((_e)?(void)0 : (epicLog(LOG_WARNING, #_e" Assert Failed"),PrintStackTrace(),assert(false))) 35 | #endif 36 | #define epicPanic(fmt, ...) epicLog(LOG_FATAL, fmt, ##__VA_ARGS__),exit(1) 37 | 38 | #endif /* INCLUDE_LOG_H_ */ 39 | -------------------------------------------------------------------------------- /include/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef LOGGING_H 4 | #define LOGGING_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "structure.h" 13 | //#include "gallocator.h" 14 | 15 | class Log { 16 | public: 17 | friend class Worker; 18 | Log(const Log&) = delete; 19 | Log& operator=(const Log&) = delete; 20 | void logWrite(GAddr, Size, const void*); 21 | void logOwner(int, GAddr); 22 | 23 | private: 24 | void* base_; 25 | std::atomic_uint_fast32_t spos_, epos_; 26 | std::atomic_bool running_; 27 | char* buf_; 28 | int fd_; 29 | std::future async_; 30 | std::thread::id thread_; 31 | 32 | void* toLocal(GAddr addr) { 33 | return TO_LOCAL(addr, base_); 34 | } 35 | 36 | void write(); 37 | int writeToBuf(void*, Size, int); 38 | int reserve(Size); 39 | Log(void*); 40 | ~Log(); 41 | }; 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /include/master.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_MASTER_H_ 5 | #define INCLUDE_MASTER_H_ 6 | 7 | #include 8 | #include 9 | #include "ae.h" 10 | #include "settings.h" 11 | #include "log.h" 12 | #include "server.h" 13 | 14 | class Master : public Server { 15 | //the service thread 16 | thread* st; 17 | 18 | //worker list: ip:port,ip:port 19 | string worker_ips; 20 | 21 | //worker counter 22 | int workers; 23 | 24 | queue unsynced_workers; 25 | unordered_map> kvs; 26 | 27 | unordered_map>> to_serve_kv_request; 28 | 29 | public: 30 | Master(const Conf& conf); 31 | inline void Join() {st->join();} 32 | 33 | inline bool IsMaster() {return true;} 34 | inline int GetWorkerId() {return 0;} 35 | 36 | void Broadcast(const char* buf, size_t len); 37 | 38 | void ProcessRequest(Client* client, WorkRequest* wr); 39 | //some post process after accepting a TCP connection (e.g., send the worker list) 40 | int PostAcceptWorker(int, void*); 41 | //inline int PostConnectMaster(int fd, void* data) {return 0;} //not used 42 | 43 | ~Master(); 44 | }; 45 | 46 | class MasterFactory { 47 | static Master *server; 48 | public: 49 | static Server* GetServer() { 50 | if (server) 51 | return server; 52 | else 53 | throw SERVER_NOT_EXIST_EXCEPTION; 54 | } 55 | static Master* CreateServer(const Conf& conf) { 56 | if (server) 57 | throw SERVER_ALREADY_EXIST_EXCEPTION; 58 | server = new Master(conf); 59 | return server; 60 | } 61 | ~MasterFactory() { 62 | if (server) 63 | delete server; 64 | } 65 | }; 66 | 67 | #endif /* INCLUDE_MASTER_H_ */ 68 | -------------------------------------------------------------------------------- /include/murmur_hasher.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_MURMUR_HASHER_H_ 5 | #define INCLUDE_MURMUR_HASHER_H_ 6 | 7 | #include 8 | #include "MurmurHash.h" 9 | 10 | static uint32_t seed = 123456; 11 | 12 | /*! CityHasher is a std::hash-style wrapper around CityHash. We 13 | * encourage using CityHasher instead of the default std::hash if 14 | * possible. */ 15 | template 16 | class MurmurHasher { 17 | public: 18 | size_t operator()(const Key& k) const { 19 | return MurmurHash2((const char*) &k, sizeof(k), seed); 20 | } 21 | }; 22 | 23 | /*! This is a template specialization of CityHasher for 24 | * std::string. */ 25 | template<> 26 | class MurmurHasher { 27 | public: 28 | size_t operator()(const std::string& k) const { 29 | return MurmurHash2(k.c_str(), k.size(), seed); 30 | } 31 | }; 32 | 33 | #endif /* INCLUDE_MURMUR_HASHER_H_ */ 34 | -------------------------------------------------------------------------------- /include/server.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef SERVER_H_ 4 | #define SERVER_H_ 5 | 6 | #include 7 | 8 | #include "client.h" 9 | #include "settings.h" 10 | #include "rdma.h" 11 | #include "workrequest.h" 12 | #include "structure.h" 13 | #include "ae.h" 14 | #include "hashtable.h" 15 | #include "locked_unordered_map.h" 16 | #include "map.h" 17 | 18 | class ServerFactory; 19 | class Server; 20 | class Client; 21 | 22 | class Server { 23 | private: 24 | //unordered_map qpCliMap; /* rdma clients */ 25 | //unordered_map widCliMap; //map from worker id to region 26 | HashTable qpCliMap { "qpCliMap" }; //thread-safe as it is dynamic 27 | HashTable widCliMap { "widCliMap" }; //store all the wid -> Client map 28 | UnorderedMap widCliMapWorker { "widCliMapWorker" }; //only store the wid -> Client map excluding ClientServer 29 | RdmaResource* resource; 30 | aeEventLoop* el; 31 | int sockfd; 32 | const Conf* conf; 33 | 34 | friend class ServerFactory; 35 | friend class Master; 36 | friend class Worker; 37 | friend class Cache; 38 | 39 | public: 40 | Client* NewClient(bool isMaster, const char* rdmaConn = nullptr); 41 | Client* NewClient(const char*); 42 | Client* NewClient(); 43 | 44 | virtual bool IsMaster() = 0; 45 | virtual int GetWorkerId() = 0; 46 | 47 | void RmClient(Client *); 48 | 49 | Client* FindClient(uint32_t qpn); 50 | void UpdateWidMap(Client* cli); 51 | Client* FindClientWid(int wid); 52 | inline int GetClusterSize() { 53 | return widCliMap.size(); 54 | } 55 | 56 | void ProcessRdmaRequest(); 57 | void ProcessRdmaRequest(ibv_wc& wc); 58 | virtual int PostAcceptWorker(int, void*) { 59 | return 0; 60 | } 61 | virtual int PostConnectMaster(int, void*) { 62 | return 0; 63 | } 64 | virtual void ProcessRequest(Client* client, WorkRequest* wr) = 0; 65 | virtual void ProcessRequest(Client* client, unsigned int id) {} 66 | virtual void CompletionCheck(unsigned int id) {} 67 | 68 | const string& GetIP() const { 69 | return conf->worker_ip; 70 | } 71 | 72 | int GetPort() const { 73 | return conf->worker_port; 74 | } 75 | 76 | virtual ~Server() { 77 | aeDeleteEventLoop(el); 78 | } 79 | }; 80 | #endif 81 | -------------------------------------------------------------------------------- /include/settings.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_SETTINGS_H_ 5 | #define INCLUDE_SETTINGS_H_ 6 | 7 | using namespace std; 8 | #define UNDER_DEVELOPMENT 9 | //#define DEBUG 10 | //#define LOCAL_MEMORY_HOOK 11 | 12 | //#define NDEBUG 13 | 14 | #define USE_SIMPLE_MAP 15 | 16 | #define HARDWARE_CACHE_LINE 64 17 | 18 | #define ASYNC_UNLOCK 19 | 20 | //#define FINE_SLAB_LOCK 21 | 22 | /* 23 | * test switch 24 | */ 25 | //#define WH_USE_LOCK 26 | #define USE_LRU 27 | #define LRU_NUM 10 28 | //#define USE_APPR_LRU 29 | //#define SELECTIVE_CACHING 30 | #define GFUNC_SUPPORT 31 | 32 | //#define USE_PIPE_W_TO_H 33 | //#define USE_PIPE_H_TO_W 34 | #define USE_BOOST_QUEUE 35 | //#define USE_BUF_ONLY 36 | #define MULTITHREAD 37 | #define USE_CITYHASH 38 | //#define USE_MURMURHASH 39 | 40 | #define MERGE_RDMA_REQUESTS 41 | //#define USE_PTHREAD_COND 42 | 43 | //#define USE_HUGEPAGE 44 | 45 | #define USE_COCKOOHASH 46 | 47 | //#define NOCACHE 48 | //#define ASYNC_RDMA_SEND 49 | 50 | #define RDMA_POLL 51 | //#define MULTITHREAD_RECV 52 | //#define USE_BOOST_THREADPOOL 53 | 54 | #define USE_ATOMIC 55 | 56 | #ifdef USE_PIPE_H_TO_W 57 | #define USE_LOCAL_TIME_EVENT 58 | #endif 59 | 60 | #define MAX_RW_TIME 20 //in microsecond 61 | 62 | #define MAX_SHARED_LOCK 254 //MAX(unsigned char)-1 63 | #define EXCLUSIVE_LOCK_TAG 255 //MAX(unsigned char) 64 | 65 | #define BLOCK_POWER 9 66 | #define BLOCK_MASK 0xFFFFFFFFFFFFFE00L 67 | #define BLOCK_SIZE (1 << BLOCK_POWER) 68 | 69 | #define RDMA_RESOURCE_EXCEPTION 1 70 | #define RDMA_CONTEXT_EXCEPTION 2 71 | #define SERVER_NOT_EXIST_EXCEPTION 3 72 | #define SERVER_ALREADY_EXIST_EXCEPTION 4 73 | 74 | #define MIN_RESERVED_FDS 32 75 | #define EVENTLOOP_FDSET_INCR (MIN_RESERVED_FDS+96) 76 | #define EVENTLOOP_FDSET_INCR (MIN_RESERVED_FDS+96) 77 | 78 | #define TCP_BACKLOG 511 /* TCP listen backlog */ 79 | #define IP_STR_LEN 46 /* INET6_ADDRSTRLEN is 46, but we need to be sure */ 80 | 81 | #define MAX_CQ_EVENTS 1024 82 | 83 | #define MAX_NUM_WORKER 20 84 | #define MAX_MASTER_PENDING_MSG 1024 85 | #define MAX_UNSIGNALED_MSG 512 86 | 87 | #define HW_MAX_PENDING 16351 88 | #define MAX_WORKER_PENDING_MSG 1024 89 | #define MASTER_RDMA_SRQ_RX_DEPTH \ 90 | (MAX_MASTER_PENDING_MSG * MAX_NUM_WORKER) 91 | #define WORKER_RDMA_SRQ_RX_DEPTH \ 92 | (MAX_WORKER_PENDING_MSG * (MAX_NUM_WORKER-1) + MAX_MASTER_PENDING_MSG) 93 | 94 | #define MAX_REQUEST_SIZE 1024 95 | #define WORKER_BUFFER_SIZE (MAX_WORKER_PENDING_MSG * MAX_REQUEST_SIZE) 96 | #define MASTER_BUFFER_SIZE (MAX_MASTER_PENDING_MSG * MAX_REQUEST_SIZE) 97 | 98 | #define MAX_IPPORT_STRLEN 21 //192.168.154.154:12345 99 | #define MAX_WORKERS_STRLEN (MAX_NUM_WORKER*MAX_IPPORT_STRLEN+MAX_NUM_WORKER-1) //192.168.154.154:12345, 100 | 101 | #define INIT_WORKQ_SIZE 2000 102 | 103 | #define MAX_MEM_STATS_SIZE 64 //10+10+20+20+4 in decimal 104 | 105 | #endif /* INCLUDE_SETTINGS_H_ */ 106 | -------------------------------------------------------------------------------- /include/structure.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_STRUCTURE_H_ 5 | #define INCLUDE_STRUCTURE_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include "settings.h" 11 | #include "log.h" 12 | #include "locked_unordered_map.h" 13 | 14 | typedef size_t Size; 15 | typedef unsigned char byte; 16 | 17 | #define DEFAULT_SPLIT_CHAR ':' 18 | 19 | #define ALLOCATOR_ALREADY_EXIST_EXCEPTION 1 20 | #define ALLOCATOR_NOT_EXIST_EXECEPTION 2 21 | 22 | typedef uint64_t ptr_t; 23 | 24 | typedef uint64_t Key; 25 | typedef uint64_t GAddr; 26 | #define OFF_MASK 0xFFFFFFFFFFFFL 27 | #define WID(gaddr) ((gaddr) >> 48) 28 | #define OFF(gaddr) ((gaddr) & OFF_MASK) 29 | #define TO_GLOB(addr, base, wid) ((ptr_t)(addr) - (ptr_t)(base) + ((ptr_t)(wid) << 48)) 30 | #define EMPTY_GLOB(wid) ((ptr_t)(wid) << 48) 31 | 32 | #define GADD(addr, off) ((addr)+(off)) //for now, we don't have any check for address overflow 33 | #define GMINUS(a, b) ((a)-(b)) //need to guarantee WID(a) == WID(b) 34 | #define TOBLOCK(x) (((ptr_t)x) & BLOCK_MASK) 35 | #define BLOCK_ALIGNED(x) (!((x) & ~BLOCK_MASK)) 36 | #define BADD(addr, i) TOBLOCK((addr) + (i)*BLOCK_SIZE) //return an addr 37 | #define BMINUS(i, j) (((i)-(j))>>BLOCK_POWER) 38 | #define TO_LOCAL(gaddr, base) (void*)(OFF(gaddr) + (ptr_t)(base)) 39 | #define Gnullptr 0 40 | 41 | struct Conf { 42 | bool is_master = true; //mark whether current process is the master (obtained from conf and the current ip) 43 | int master_port = 12345; 44 | std::string master_ip = "localhost"; 45 | std::string master_bindaddr; 46 | int worker_port = 12346; 47 | std::string worker_bindaddr; 48 | std::string worker_ip = "localhost"; 49 | Size size = 1024 * 1024L * 512; //per-server size of memory pre-allocated 50 | Size ghost_th = 1024 * 1024; 51 | double cache_th = 0.15; //if free mem is below this threshold, we start to allocate memory from remote nodes 52 | int unsynced_th = 1; 53 | double factor = 1.25; 54 | int maxclients = 1024; 55 | int maxthreads = 10; 56 | int backlog = TCP_BACKLOG; 57 | int loglevel = LOG_WARNING; 58 | std::string* logfile = nullptr; 59 | int timeout = 10; //ms 60 | int eviction_period = 100; //ms 61 | }; 62 | 63 | typedef int PostProcessFunc(int, void*); 64 | 65 | #define LOCK_MICRO(table, key) do {((table).lock(key));} while(0) 66 | #define UNLOCK_MICRO(table, key) ((table).unlock(key)) 67 | 68 | #endif /* INCLUDE_STRUCTURE_H_ */ 69 | -------------------------------------------------------------------------------- /include/tcp.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #ifndef CALLBACK_H_ 4 | #define CALLBACK_H_ 5 | 6 | #include "ae.h" 7 | #include "rdma.h" 8 | #include "settings.h" 9 | 10 | /** 11 | * callback for connection 12 | * For tcp case (a remote client), the established tcp connection is only used 13 | * to exchange rdma parameters, and will be immediately closed after 14 | * the exchangement. 15 | * 16 | * For Unix Socket case, the incoming client is local, and the established 17 | * socket will be used for messaging with this client. The first 4 bytes of 18 | * each message is same as the immediate data associated with 19 | * RDMA_WRITE_WITH_IMM verbs. 20 | */ 21 | 22 | aeFileProc AcceptTcpClientHandle; 23 | aeFileProc ProcessRdmaRequestHandle; 24 | 25 | #endif 26 | 27 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_UTIL_H_ 5 | #define INCLUDE_UTIL_H_ 6 | 7 | #include 8 | #include 9 | #include 10 | #include "settings.h" 11 | #include "structure.h" 12 | 13 | template 14 | inline vector& Split(stringstream& ss, vector& elems, char delim) { 15 | T item; 16 | while (ss >> item) { 17 | elems.push_back(item); 18 | if (ss.peek() == delim) 19 | ss.ignore(); 20 | } 21 | return elems; 22 | } 23 | 24 | template 25 | inline vector& Split(string& s, vector& elems, char delim = 26 | DEFAULT_SPLIT_CHAR) { 27 | stringstream ss(s); 28 | return Split(ss, elems, delim); 29 | } 30 | 31 | template 32 | inline vector& Split(char *s, vector& elems, char delim = 33 | DEFAULT_SPLIT_CHAR) { 34 | stringstream ss(s); 35 | return Split(ss, elems, delim); 36 | } 37 | 38 | template<> 39 | vector& Split(stringstream& ss, vector& elems, 40 | char delim); 41 | 42 | string get_local_ip(const char* iface = nullptr); 43 | inline string get_local_ip(const string iface = "") { 44 | return get_local_ip(iface.empty() ? nullptr : iface.c_str()); 45 | } 46 | 47 | long get_time(); 48 | uint64_t rdtsc(); 49 | 50 | #define atomic_add(v, i) __sync_fetch_and_add((v), (i)) 51 | #define atomic_read(v) __sync_fetch_and_add((v), (0)) 52 | 53 | inline int GetRandom(int min, int max) { 54 | static __thread unsigned int tid = (unsigned int) syscall(SYS_gettid); 55 | epicLog(LOG_DEBUG, "tid = %d", tid); 56 | int ret = (rand_r(&tid) % (max - min)) + min; 57 | return ret; 58 | } 59 | 60 | inline uint64_t ceil_divide(uint64_t a, uint64_t b) { 61 | return (a + b - 1) / b; 62 | } 63 | 64 | inline int GetRandom(int min, int max, unsigned int* seedp) { 65 | int ret = (rand_r(seedp) % (max - min)) + min; 66 | return ret; 67 | } 68 | 69 | /* 70 | * only support basic types 71 | */ 72 | template 73 | inline T1 force_cast(T2 t2) { 74 | assert(sizeof(T1) == sizeof(T2)); 75 | T1 t1; 76 | memcpy(&t1, &t2, sizeof(T1)); 77 | return t1; 78 | } 79 | 80 | #endif /* INCLUDE_UTIL_H_ */ 81 | -------------------------------------------------------------------------------- /include/worker_handle.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #ifndef INCLUDE_WORKER_HANDLE_H_ 5 | #define INCLUDE_WORKER_HANDLE_H_ 6 | 7 | #include 8 | #include 9 | 10 | #include "lockwrapper.h" 11 | #include "worker.h" 12 | #include "workrequest.h" 13 | 14 | class WorkerHandle { 15 | boost::lockfree::queue* wqueue; //work queue used to communicate with worker 16 | Worker* worker; 17 | //app-side pipe fd 18 | int send_pipe[2]; 19 | int recv_pipe[2]; 20 | static LockWrapper lock; 21 | #ifdef USE_PTHREAD_COND 22 | pthread_mutex_t cond_lock; 23 | pthread_cond_t cond; 24 | #endif 25 | #if !(defined(USE_PIPE_W_TO_H) && defined(USE_PIPE_H_TO_W)) 26 | volatile int* notify_buf; 27 | int notify_buf_size; 28 | #endif 29 | public: 30 | WorkerHandle(Worker* w); 31 | void RegisterThread(); 32 | void DeRegisterThread(); 33 | int SendRequest(WorkRequest* wr); 34 | inline int GetWorkerId() { 35 | return worker->GetWorkerId(); 36 | } 37 | int GetWorkersSize() { 38 | return worker->GetWorkersSize(); 39 | } 40 | inline void* GetLocal(GAddr addr) { 41 | return worker->ToLocal(addr); 42 | } 43 | 44 | void ReportCacheStatistics(); 45 | void ResetCacheStatistics(); 46 | 47 | ~WorkerHandle(); 48 | }; 49 | 50 | #endif /* INCLUDE_WORKER_HANDLE_H_ */ 51 | -------------------------------------------------------------------------------- /include/zmalloc.h: -------------------------------------------------------------------------------- 1 | /* zmalloc - total amount of allocated memory aware version of malloc() 2 | * 3 | * Copyright (c) 2009-2010, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef __ZMALLOC_H 32 | #define __ZMALLOC_H 33 | 34 | ///* Double expansion needed for stringification of macro values. */ 35 | //#define __xstr(s) __str(s) 36 | //#define __str(s) #s 37 | 38 | #if defined(USE_TCMALLOC) 39 | #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) 40 | #include 41 | #if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) 42 | #define HAVE_MALLOC_SIZE 1 43 | #define zmalloc_size(p) tc_malloc_size(p) 44 | #else 45 | #error "Newer version of tcmalloc required" 46 | #endif 47 | 48 | #elif defined(USE_JEMALLOC) 49 | #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) 50 | #include 51 | #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) 52 | #define HAVE_MALLOC_SIZE 1 53 | #define zmalloc_size(p) je_malloc_usable_size(p) 54 | #else 55 | #error "Newer version of jemalloc required" 56 | #endif 57 | 58 | #elif defined(__APPLE__) 59 | #include 60 | #define HAVE_MALLOC_SIZE 1 61 | #define zmalloc_size(p) malloc_size(p) 62 | #endif 63 | 64 | #ifndef ZMALLOC_LIB 65 | #define ZMALLOC_LIB "libc" 66 | #endif 67 | 68 | void *zmalloc(size_t size); 69 | void *zcalloc(size_t size); 70 | void *zrealloc(void *ptr, size_t size); 71 | void zfree(void *ptr); 72 | char *zstrdup(const char *s); 73 | size_t zmalloc_used_memory(void); 74 | void zmalloc_enable_thread_safeness(void); 75 | void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); 76 | float zmalloc_get_fragmentation_ratio(size_t rss); 77 | size_t zmalloc_get_rss(void); 78 | size_t zmalloc_get_private_dirty(void); 79 | size_t zmalloc_get_smap_bytes_by_field(char *field); 80 | void zlibc_free(void *ptr); 81 | 82 | #ifndef HAVE_MALLOC_SIZE 83 | size_t zmalloc_size(void *ptr); 84 | #endif 85 | 86 | #endif /* __ZMALLOC_H */ 87 | -------------------------------------------------------------------------------- /lib/libcuckoo/.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.in 3 | *.la 4 | *.lo 5 | *.log 6 | *.o 7 | *.out 8 | *.trs 9 | *~ 10 | .DS_Store 11 | .deps 12 | .libs 13 | Makefile 14 | aclocal.m4 15 | autom4te.cache 16 | cityhash_unittest 17 | compile 18 | config.guess 19 | config.h 20 | config.log 21 | config.status 22 | config.sub 23 | config.sub 24 | configure 25 | depcomp 26 | depcomp 27 | examples/count_freq 28 | examples/hellohash 29 | examples/nested_table 30 | install-sh 31 | libtool 32 | libtool.m4 33 | lt*.m4 34 | ltmain.sh 35 | missing 36 | stamp-h1 37 | test-driver -------------------------------------------------------------------------------- /lib/libcuckoo/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2013, Carnegie Mellon University and Intel Corporation 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | --------------------------- 16 | 17 | CityHash (lib/city.h, lib/city.cc) is Copyright (c) Google, Inc. and 18 | has its own license, as detailed in the source files. 19 | -------------------------------------------------------------------------------- /lib/libcuckoo/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = cityhash-1.1.1 src tests examples 2 | CLEANFILES = core *.core *~ 3 | DISTCLEANFILES = autom4te*.cache config.status config.log 4 | MAINTAINERCLEANFILES = aclocal.m4 install-sh mkinstalldirs \ 5 | missing configure config.guess config.sub config.h.in \ 6 | ltconfig ltmain.sh Makefile.in stamp-h.in depcomp m4/*.m4 7 | 8 | ACLOCAL_AMFLAGS = -I m4 9 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/COPYING: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/Makefile.am: -------------------------------------------------------------------------------- 1 | ACLOCAL_AMFLAGS = -I m4 2 | SUBDIRS = src 3 | 4 | dist_doc_DATA = README NEWS COPYING 5 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/NEWS: -------------------------------------------------------------------------------- 1 | CityHash v1.1.1, June 17, 2013 2 | 3 | * Fix CityHash32() so platforms that disagree about whether 'char' is signed 4 | use the same mathematical function. For most people this changes nothing 5 | as most popular platforms agree with x86 about whether 'char' is signed. 6 | * No changes to any of the functions, unless you had been using CityHash32() 7 | despite "make check" reporting a failure on your platform. 8 | * Slightly modernize and improve configuration and portability. 9 | 10 | CityHash v1.1, October 22, 2012 11 | 12 | * Add CityHash32(), intended for 32-bit platforms. 13 | * Change existing functions to improve their hash quality and/or speed. Most 14 | of the changes were minor, but CityHashCrc* was substantially reworked 15 | (and made perhaps 10% slower, unfortunately). 16 | * Improve README. 17 | 18 | CityHash v1.0.3, October 6, 2011 19 | 20 | * Change all the functions to improve their hash quality. Most of the 21 | changes were minor. Special thanks to Bob Jenkins for reporting some 22 | issues that he'd found. The speed of the functions after these changes is 23 | roughly unchanged, except that CityHash128() and CityHash128WithSeed() are 24 | slower. 25 | * To improve portability, replace the one use of ssize_t with signed long. 26 | * Improve README. 27 | 28 | CityHash v1.0.2, May 8, 2011 29 | 30 | * Correct a problem in CityHashCrc256(); for inputs under 240 bytes the 31 | scheme of padding to 240 bytes was causing the empty string and an input 32 | of exactly 240 NULs to have the same hash code. That is now fixed. 33 | Most strings less than 240 bytes long will have a different hash than 34 | they did in v1.0.1. 35 | * Other hash functions are unchanged. 36 | * Minor corrections and improvements to README. 37 | 38 | CityHash v1.0.1, April 28, 2011 39 | 40 | * Added README, NEWS, and COPYING. The README contains installation and 41 | usage instructions, information on "hash quality," and other goodies. 42 | * Improved how CityHash128() and CityHash128WithSeed() handle very short 43 | input strings. 44 | * Added new functions that are faster on long strings on 64-bit CPUs with 45 | a CRC32 instruction: CityHashCrc128(), CityHashCrc128WithSeed(), and 46 | CityHashCrc256(). 47 | * Removed our assumption that "*(const uint64*)p" and such is safe. Now we 48 | memcpy() instead. It has no speed penalty and makes our intent explicit 49 | to the compiler. 50 | * Changed #include "city.h" to #include , suggested by Robert Escriva. 51 | * Added build system, a modified version of one contributed by Robert Escriva. 52 | * We now use __builtin_expect if the configure script can make it work, rather 53 | than just on gcc. 54 | * Added a test: use "make check" to run it. If you compile CityHash in a way 55 | that causes it to return unexpected results, the test should fail. 56 | * Added untested big-endian support. Please let us know if you try it! 57 | * Other than CityHash128() and CityHash128WithSeed(), the hash functions in 58 | the previous release are unchanged. 59 | 60 | CityHash v1, April 11, 2011 61 | 62 | * Initial release 63 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/configure.ac: -------------------------------------------------------------------------------- 1 | m4_define([cityhash_major], [1]) 2 | m4_define([cityhash_minor], [1]) 3 | m4_define([cityhash_patchlevel], [1]) 4 | 5 | # Libtool shared library interface versions (current:revision:age) 6 | # Update this value for every release! (A:B:C will map to foo.so.(A-C).C.B) 7 | # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html 8 | m4_define([cityhash_ltversion], [4:0:0]) 9 | 10 | AC_PREREQ([2.65]) 11 | AC_INIT([CityHash], [cityhash_major.cityhash_minor.cityhash_patchlevel], [cityhash-discuss@googlegroups.com]) 12 | AC_CONFIG_HEADERS([config.h]) 13 | AM_INIT_AUTOMAKE([1.10 no-define foreign]) 14 | LT_PREREQ([2.2]) 15 | LT_INIT 16 | 17 | AC_CONFIG_FILES([Makefile 18 | src/Makefile]) 19 | AC_CONFIG_SRCDIR([src/city.h]) 20 | AC_CONFIG_MACRO_DIR([m4]) 21 | 22 | AC_ARG_ENABLE([sse4.2], 23 | AS_HELP_STRING("Build CityHash variants that depend on the _mm_crc32_u64 intrinsic."), 24 | [ cityhash_sse42=true ], 25 | []) 26 | AM_CONDITIONAL([SSE42], [test x$cityhash_sse42 = xtrue ]) 27 | 28 | # Checks for programs. 29 | AC_PROG_CXX 30 | AC_LANG([C++]) 31 | AC_C_BIGENDIAN 32 | 33 | # Checks for libraries. 34 | 35 | # Checks for header files. 36 | AC_CHECK_HEADERS([stdint.h stdlib.h]) 37 | 38 | # Checks for typedefs, structures, and compiler characteristics. 39 | AC_C_INLINE 40 | AC_TYPE_SIZE_T 41 | AC_TYPE_SSIZE_T 42 | AC_TYPE_UINT32_T 43 | AC_TYPE_UINT64_T 44 | AC_TYPE_UINT8_T 45 | 46 | # Check for __builtin_expect 47 | AC_MSG_CHECKING([if the compiler supports __builtin_expect]) 48 | AC_COMPILE_IFELSE( 49 | [AC_LANG_PROGRAM(, [[return __builtin_expect(1, 1) ? 1 : 0;]])], 50 | [ 51 | cityhash_have_builtin_expect=yes 52 | AC_MSG_RESULT([yes]) 53 | ], [ 54 | cityhash_have_builtin_expect=no 55 | AC_MSG_RESULT([no]) 56 | ]) 57 | if test x$cityhash_have_builtin_expect = xyes ; then 58 | AC_DEFINE([HAVE_BUILTIN_EXPECT], [1], [Define to 1 if the compiler supports __builtin_expect.]) 59 | fi 60 | 61 | AC_OUTPUT 62 | 63 | echo \ 64 | "------------------------------------------------- 65 | 66 | ${PACKAGE_NAME} Version ${PACKAGE_VERSION} 67 | 68 | Prefix: '${prefix}'. 69 | Compiler: '${CXX} ${CXXFLAGS}' 70 | 71 | Now type 'make @<:@@:>@' 72 | where the optional is: 73 | all - build everything 74 | check - build and run tests 75 | install - install everything 76 | 77 | --------------------------------------------------" 78 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/m4/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ooibc88/gam/456b3f8c15f3052a19c69cc585318e4c61e592b5/lib/libcuckoo/cityhash-1.1.1/m4/.gitkeep -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/src/Makefile.am: -------------------------------------------------------------------------------- 1 | # library 2 | lib_LTLIBRARIES = libcityhash.la 3 | libcityhash_la_SOURCES = city.cc 4 | if SSE42 5 | include_HEADERS = city.h citycrc.h 6 | else 7 | include_HEADERS = city.h 8 | endif 9 | 10 | # test 11 | cityhash_unittest_SOURCES = city-test.cc 12 | cityhash_unittest_LDADD = libcityhash.la 13 | TESTS = cityhash_unittest 14 | noinst_PROGRAMS = $(TESTS) 15 | -------------------------------------------------------------------------------- /lib/libcuckoo/cityhash-1.1.1/src/citycrc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /lib/libcuckoo/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.67]) 5 | AC_INIT([libcuckoo], [1.0], [fawn-dev@mailman.srv.cs.cmu.edu]) 6 | AC_CONFIG_HEADERS([config.h]) 7 | AM_INIT_AUTOMAKE([foreign]) 8 | 9 | AC_CONFIG_FILES([ Makefile 10 | src/Makefile 11 | examples/Makefile 12 | ]) 13 | AC_CONFIG_SRCDIR([src/cuckoohash_map.hh]) 14 | AC_CONFIG_SUBDIRS([cityhash-1.1.1 tests]) 15 | AC_CONFIG_MACRO_DIR([m4]) 16 | 17 | # Empty defaults for flags 18 | : ${CFLAGS=""} 19 | : ${CXXFLAGS=""} 20 | AC_PROG_CC 21 | AC_PROG_CXX 22 | AC_PROG_LIBTOOL 23 | 24 | AX_PTHREAD 25 | LIBS="$PTHREAD_LIBS $LIBS" 26 | CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" 27 | CC="$PTHREAD_CC" 28 | 29 | AX_CXX_COMPILE_STDCXX_11(,[mandatory]) 30 | 31 | CXXFLAGS="$CXXFLAGS -Wall -Wextra -Wshadow" 32 | 33 | AC_OUTPUT 34 | -------------------------------------------------------------------------------- /lib/libcuckoo/examples/Makefile.am: -------------------------------------------------------------------------------- 1 | CITYHASH_DIR = ../cityhash-1.1.1 2 | AM_LDFLAGS=-L$(CITYHASH_DIR)/src -lcityhash 3 | AM_CXXFLAGS=-I$(CITYHASH_DIR)/src 4 | 5 | noinst_PROGRAMS = hellohash count_freq nested_table 6 | 7 | hellohash_SOURCES = hellohash.cc 8 | count_freq_SOURCES = count_freq.cc 9 | nested_table_SOURCES = nested_table.cc 10 | -------------------------------------------------------------------------------- /lib/libcuckoo/examples/count_freq.cc: -------------------------------------------------------------------------------- 1 | /* A simple example of using the hash table that counts the 2 | * frequencies of a sequence of random numbers. */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "../src/cuckoohash_map.hh" 15 | #include "../src/city_hasher.hh" 16 | 17 | typedef uint32_t KeyType; 18 | typedef cuckoohash_map > Table; 19 | const size_t thread_num = 8; 20 | const size_t total_inserts = 10000000; 21 | 22 | void do_inserts(Table& freq_map) { 23 | std::mt19937_64 gen( 24 | std::chrono::system_clock::now().time_since_epoch().count()); 25 | std::uniform_int_distribution dist( 26 | std::numeric_limits::min(), 27 | std::numeric_limits::max()); 28 | auto updatefn = [](size_t& num) { ++num; }; 29 | for (size_t i = 0; i < total_inserts/thread_num; i++) { 30 | KeyType num = dist(gen); 31 | // If the number is already in the table, it will increment 32 | // its count by one. Otherwise it will insert a new entry in 33 | // the table with count one. 34 | freq_map.upsert(num, updatefn, 1); 35 | } 36 | } 37 | 38 | int main() { 39 | Table freq_map; 40 | freq_map.reserve(total_inserts); 41 | // Run the inserts in thread_num threads 42 | std::vector threads; 43 | for (size_t i = 0; i < thread_num; i++) { 44 | threads.emplace_back(do_inserts, std::ref(freq_map)); 45 | } 46 | for (size_t i = 0; i < thread_num; i++) { 47 | threads[i].join(); 48 | } 49 | 50 | // We iterate through the table and print out the element with the 51 | // maximum number of occurrences. 52 | KeyType maxkey; 53 | size_t maxval = 0; 54 | { 55 | auto lt = freq_map.lock_table(); 56 | for (const auto& it : lt) { 57 | if (it.second > maxval) { 58 | maxkey = it.first; 59 | maxval = it.second; 60 | } 61 | } 62 | } 63 | 64 | std::cout << maxkey << " occurred " << maxval << " times." << std::endl; 65 | 66 | // Print some information about the table 67 | std::cout << "Table size: " << freq_map.size() << std::endl; 68 | std::cout << "Bucket count: " << freq_map.bucket_count() << std::endl; 69 | std::cout << "Load factor: " << freq_map.load_factor() << std::endl; 70 | } 71 | -------------------------------------------------------------------------------- /lib/libcuckoo/examples/hellohash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "../src/cuckoohash_map.hh" 6 | #include "../src/city_hasher.hh" 7 | 8 | int main() { 9 | cuckoohash_map >Table; 10 | 11 | for (int i = 0; i < 100; i++) { 12 | Table[i] = "hello"+std::to_string(i); 13 | } 14 | 15 | for (int i = 0; i < 101; i++) { 16 | std::string out; 17 | 18 | if (Table.find(i, out)) { 19 | std::cout << i << " " << out << std::endl; 20 | } else { 21 | std::cout << i << " NOT FOUND" << std::endl; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /lib/libcuckoo/examples/nested_table.cc: -------------------------------------------------------------------------------- 1 | /* We demonstrate how to nest hash tables within one another, to store 2 | * unstructured data, kind of like JSON. There's still the limitation that it's 3 | * statically typed. */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "../src/cuckoohash_map.hh" 11 | 12 | typedef cuckoohash_map InnerTable; 13 | typedef cuckoohash_map> OuterTable; 14 | 15 | int main() { 16 | OuterTable tbl; 17 | 18 | tbl.insert("bob", std::unique_ptr(new InnerTable)); 19 | tbl.update_fn("bob", [] (std::unique_ptr& innerTbl) { 20 | innerTbl->insert("nickname", "jimmy"); 21 | innerTbl->insert("pet", "dog"); 22 | innerTbl->insert("food", "bagels"); 23 | }); 24 | 25 | tbl.insert("jack", std::unique_ptr(new InnerTable)); 26 | tbl.update_fn("jack", [] (std::unique_ptr& innerTbl) { 27 | innerTbl->insert("friend", "bob"); 28 | innerTbl->insert("activity", "sleeping"); 29 | innerTbl->insert("language", "javascript"); 30 | }); 31 | 32 | { 33 | auto lt = tbl.lock_table(); 34 | for (const auto& item : lt) { 35 | std::cout << "Properties for " << item.first << std::endl; 36 | auto innerLt = item.second->lock_table(); 37 | for (auto innerItem : innerLt) { 38 | std::cout << "\t" << innerItem.first << " = " 39 | << innerItem.second << std::endl; 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/Makefile.am: -------------------------------------------------------------------------------- 1 | libcuckooincludedir = $(includedir)/libcuckoo 2 | libcuckooinclude_HEADERS = city_hasher.hh default_hasher.hh cuckoohash_map.hh \ 3 | cuckoohash_config.hh cuckoohash_util.hh lazy_array.hh 4 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/city_hasher.hh: -------------------------------------------------------------------------------- 1 | #ifndef _CITY_HASHER_HH 2 | #define _CITY_HASHER_HH 3 | 4 | #include 5 | #include 6 | 7 | /*! CityHasher is a std::hash-style wrapper around CityHash. We 8 | * encourage using CityHasher instead of the default std::hash if 9 | * possible. */ 10 | template 11 | class CityHasher { 12 | public: 13 | size_t operator()(const Key& k) const { 14 | return CityHash64((const char*) &k, sizeof(k)); 15 | } 16 | }; 17 | 18 | /*! This is a template specialization of CityHasher for 19 | * std::string. */ 20 | template <> 21 | class CityHasher { 22 | public: 23 | size_t operator()(const std::string& k) const { 24 | return CityHash64(k.c_str(), k.size()); 25 | } 26 | }; 27 | 28 | #endif // _CITY_HASHER_HH 29 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/cuckoohash_config.hh: -------------------------------------------------------------------------------- 1 | /** \file */ 2 | 3 | #ifndef _CUCKOOHASH_CONFIG_HH 4 | #define _CUCKOOHASH_CONFIG_HH 5 | 6 | #include 7 | 8 | //! The default maximum number of keys per bucket 9 | const size_t DEFAULT_SLOT_PER_BUCKET = 4; 10 | 11 | //! The default number of elements in an empty hash table 12 | const size_t DEFAULT_SIZE = (1U << 16) * DEFAULT_SLOT_PER_BUCKET; 13 | 14 | //! On a scale of 0 to 16, the memory granularity of the locks array. 0 is the 15 | //! least granular, meaning the array is a contiguous array and thus offers the 16 | //! best performance but the greatest memory overhead. 16 is the most granular, 17 | //! offering the least memory overhead but worse performance. 18 | const size_t LOCK_ARRAY_GRANULARITY = 0; 19 | 20 | //! The default minimum load factor that the table allows for automatic 21 | //! expansion. It must be a number between 0.0 and 1.0. The table will throw 22 | //! libcuckoo_load_factor_too_low if the load factor falls below this value 23 | //! during an automatic expansion. 24 | const double DEFAULT_MINIMUM_LOAD_FACTOR = 0.05; 25 | 26 | //! An alias for the value that sets no limit on the maximum hashpower. If this 27 | //! value is set as the maximum hashpower limit, there will be no limit. Since 0 28 | //! is the only hashpower that can never occur, it should stay at 0. 29 | const size_t NO_MAXIMUM_HASHPOWER = 0; 30 | 31 | //! set LIBCUCKOO_DEBUG to 1 to enable debug output 32 | #define LIBCUCKOO_DEBUG 0 33 | 34 | #endif // _CUCKOOHASH_CONFIG_HH 35 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/default_hasher.hh: -------------------------------------------------------------------------------- 1 | #ifndef _DEFAULT_HASHER_HH 2 | #define _DEFAULT_HASHER_HH 3 | 4 | #include 5 | #include 6 | 7 | /*! DefaultHasher is the default hash class used in the table. It overloads a 8 | * few types that std::hash does badly on (namely integers), and falls back to 9 | * std::hash for anything else. */ 10 | template 11 | class DefaultHasher { 12 | std::hash fallback; 13 | 14 | public: 15 | template 16 | typename std::enable_if::value, size_t>::type 17 | operator()(const Key& k) const { 18 | // This constant is found in the CityHash code 19 | return k * 0x9ddfea08eb382d69ULL; 20 | } 21 | 22 | template 23 | typename std::enable_if::value, size_t>::type 24 | operator()(const Key& k) const { 25 | return fallback(k); 26 | } 27 | }; 28 | 29 | #endif // _DEFAULT_HASHER_HH 30 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/lazy_array.hh: -------------------------------------------------------------------------------- 1 | /** \file */ 2 | 3 | #ifndef _LAZY_ARRAY_HH 4 | #define _LAZY_ARRAY_HH 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "cuckoohash_util.hh" 12 | 13 | // lazy array. A fixed-size array, broken up into segments that are dynamically 14 | // allocated, only when requested. The array size and segment size are 15 | // pre-defined, and are powers of two. The user must make sure the necessary 16 | // segments are allocated before accessing the array. 17 | template 19 | > 20 | class lazy_array { 21 | static_assert(SEGMENT_BITS + OFFSET_BITS <= sizeof(size_t)*8, 22 | "The number of segment and offset bits cannot exceed " 23 | " the number of bits in a size_t"); 24 | private: 25 | static const size_t SEGMENT_SIZE = 1UL << OFFSET_BITS; 26 | static const size_t NUM_SEGMENTS = 1UL << SEGMENT_BITS; 27 | // The segments array itself is mutable, so that the const subscript 28 | // operator can still add segments 29 | mutable std::array segments_; 30 | 31 | void move_other_array(lazy_array&& arr) { 32 | clear(); 33 | std::copy(arr.segments_.begin(), arr.segments_.end(), 34 | segments_.begin()); 35 | std::fill(arr.segments_.begin(), arr.segments_.end(), nullptr); 36 | } 37 | 38 | inline size_t get_segment(size_t i) { 39 | return i >> OFFSET_BITS; 40 | } 41 | 42 | static const size_t OFFSET_MASK = ((1UL << OFFSET_BITS) - 1); 43 | inline size_t get_offset(size_t i) { 44 | return i & OFFSET_MASK; 45 | } 46 | 47 | public: 48 | lazy_array(): segments_{{nullptr}} {} 49 | 50 | // No copying 51 | lazy_array(const lazy_array&) = delete; 52 | lazy_array& operator=(const lazy_array&) = delete; 53 | 54 | // Moving is allowed 55 | lazy_array(lazy_array&& arr) : segments_{{nullptr}} { 56 | move_other_array(std::move(arr)); 57 | } 58 | lazy_array& operator=(lazy_array&& arr) { 59 | move_other_vector(std::move(arr)); 60 | return *this; 61 | } 62 | 63 | ~lazy_array() { 64 | clear(); 65 | } 66 | 67 | void clear() { 68 | for (size_t i = 0; i < segments_.size(); ++i) { 69 | if (segments_[i] != nullptr) { 70 | destroy_array(segments_[i], SEGMENT_SIZE); 71 | segments_[i] = nullptr; 72 | } 73 | } 74 | } 75 | 76 | T& operator[](size_t i) { 77 | assert(segments_[get_segment(i)] != nullptr); 78 | return segments_[get_segment(i)][get_offset(i)]; 79 | } 80 | 81 | const T& operator[](size_t i) const { 82 | assert(segments_[get_segment(i)] != nullptr); 83 | return segments_[get_segment(i)][get_offset(i)]; 84 | } 85 | 86 | // Ensures that the array has enough segments to index target elements, not 87 | // exceeding the total size. The user must ensure that the array is properly 88 | // allocated before accessing a certain index. This saves having to check 89 | // every index operation. 90 | void allocate(size_t target) { 91 | assert(target <= size()); 92 | if (target == 0) { 93 | return; 94 | } 95 | const size_t last_segment = get_segment(target - 1); 96 | for (size_t i = 0; i <= last_segment; ++i) { 97 | if (segments_[i] == nullptr) { 98 | segments_[i] = create_array(SEGMENT_SIZE); 99 | } 100 | } 101 | } 102 | 103 | // Returns the number of elements in the array that can be indexed, starting 104 | // contiguously from the beginning. 105 | size_t allocated_size() const { 106 | size_t num_allocated_segments = 0; 107 | for (; segments_[num_allocated_segments] != nullptr && 108 | num_allocated_segments < NUM_SEGMENTS; 109 | ++num_allocated_segments) {} 110 | return num_allocated_segments * SEGMENT_SIZE; 111 | } 112 | 113 | static constexpr size_t size() { 114 | return 1UL << (OFFSET_BITS + SEGMENT_BITS); 115 | } 116 | }; 117 | 118 | #endif // _LAZY_ARRAY_HH 119 | -------------------------------------------------------------------------------- /lib/libcuckoo/src/mainpage.dox: -------------------------------------------------------------------------------- 1 | /*! \mainpage libcuckoo Documentation 2 | * 3 | * libcuckoo is a high-performance, memory efficient hash table that 4 | * supports concurrent reads and writes. 5 | * 6 | * \ref cuckoohash_map is the class of the hash table. Its interface 7 | * resembles that of STL's unordered_map but does contain some 8 | * important differences. 9 | * 10 | * Internally, the hash table is partitioned into an array of 11 | * buckets, each of which contains \ref SLOT_PER_BUCKET slots to 12 | * store items. 13 | * 14 | * Each bucket has a lock to ensure multiple threads don't modify the 15 | * same elements. Most operations will lock no more than two buckets 16 | * at a time, thereby allowing for concurrent reads and writes. 17 | * 18 | * The library also contains \ref CityHasher, which is a simple 19 | * wrapper around Google's CityHash for the std::hash type. 20 | */ 21 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = benchmarks stress-tests unit-tests 2 | EXTRA_DIST = test_util.hh 3 | ACLOCAL_AMFLAGS = -I ../m4 4 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/benchmarks/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CXXFLAGS = -O3 -DNDEBUG 2 | 3 | TESTS = benchmark-driver.sh 4 | noinst_PROGRAMS = insert_throughput.out read_throughput.out \ 5 | read_insert_throughput.out 6 | insert_throughput_out_SOURCES = insert_throughput.cc 7 | read_throughput_out_SOURCES = read_throughput.cc 8 | read_insert_throughput_out_SOURCES = read_insert_throughput.cc 9 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/benchmarks/benchmark-driver.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | ./insert_throughput.out --table-capacity 1 3 | ./read_throughput.out 4 | ./read_insert_throughput.out --table-capacity 1 5 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.67]) 5 | AC_INIT([libcuckoo-tests], [1.0]) 6 | AM_INIT_AUTOMAKE([foreign -Wall -Werror]) 7 | 8 | AC_CONFIG_FILES([ Makefile 9 | benchmarks/Makefile 10 | stress-tests/Makefile 11 | unit-tests/Makefile 12 | ]) 13 | AC_CONFIG_MACRO_DIR([../m4]) 14 | 15 | # Empty defaults for flags 16 | : ${CFLAGS=""} 17 | : ${CXXFLAGS=""} 18 | AC_PROG_CC 19 | AC_PROG_CXX 20 | AC_PROG_LIBTOOL 21 | 22 | AX_PTHREAD 23 | LIBS="$PTHREAD_LIBS $LIBS" 24 | CFLAGS="$CFLAGS $PTHREAD_CFLAGS" 25 | CXXFLAGS="$CFLAGS $PTHREAD_CFLAGS" 26 | CC="$PTHREAD_CC" 27 | 28 | AX_CXX_COMPILE_STDCXX_11(,[mandatory]) 29 | 30 | AC_OUTPUT 31 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/m4/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ooibc88/gam/456b3f8c15f3052a19c69cc585318e4c61e592b5/lib/libcuckoo/tests/m4/.gitkeep -------------------------------------------------------------------------------- /lib/libcuckoo/tests/stress-tests/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CXXFLAGS = -O3 -DNDEBUG 2 | 3 | TESTS = stress_checked.out stress_unchecked.out 4 | noinst_PROGRAMS = $(TESTS) 5 | stress_checked_out_SOURCES = stress_checked.cc 6 | stress_unchecked_out_SOURCES = stress_unchecked.cc 7 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CXXFLAGS = -g -O0 2 | 3 | TESTS = test_runner.out 4 | noinst_PROGRAMS = $(TESTS) 5 | test_runner_out_SOURCES = \ 6 | test_iterator.cc \ 7 | test_bracket_operator.cc \ 8 | test_resize.cc \ 9 | test_constructor.cc \ 10 | test_minimum_load_factor.cc \ 11 | test_maximum_hashpower.cc \ 12 | test_hash_properties.cc \ 13 | test_noncopyable_types.cc \ 14 | unit_test_util.cc \ 15 | test_runner.cc catch.hpp 16 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_bracket_operator.cc: -------------------------------------------------------------------------------- 1 | #include "catch.hpp" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "../../src/cuckoohash_map.hh" 10 | #include "unit_test_util.hh" 11 | 12 | TEST_CASE("bracket find empty table", "[bracket]") { 13 | IntIntTable table; 14 | auto ref = table[10]; 15 | REQUIRE_THROWS_AS((void) ((int) ref), std::out_of_range); 16 | } 17 | 18 | TEST_CASE("bracket find filled table", "[bracket]") { 19 | StringIntTable table; 20 | for (int i = 0; i < 10; ++i) { 21 | table.insert(std::to_string(i), i); 22 | } 23 | 24 | for (int i = 0; i < 10; ++i) { 25 | REQUIRE(table[std::to_string(i)] == i); 26 | } 27 | } 28 | 29 | TEST_CASE("bracket insert", "[bracket]") { 30 | IntIntTable table; 31 | for (int i = 0; i < 10; ++i) { 32 | table[i] = i + 1; 33 | } 34 | for (int i = 0; i < 10; ++i) { 35 | REQUIRE(table[i] == i + 1); 36 | } 37 | } 38 | 39 | TEST_CASE("bracket assign to reference", "[bracket]") { 40 | IntIntTable table; 41 | table.insert(0, 0); 42 | for (int i = 1; i < 10; ++i) { 43 | table[i] = table[i - 1]; 44 | } 45 | for (int i = 0; i < 10; ++i) { 46 | REQUIRE(table[i] == 0); 47 | } 48 | } 49 | 50 | TEST_CASE("bracket assign updates", "[bracket]") { 51 | IntIntTable table; 52 | table.insert(0, 0); 53 | REQUIRE(table[0] == 0); 54 | table[0] = 10; 55 | REQUIRE(table[0] == 10); 56 | } 57 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_constructor.cc: -------------------------------------------------------------------------------- 1 | #include "catch.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include "../../src/cuckoohash_map.hh" 7 | #include "unit_test_util.hh" 8 | 9 | TEST_CASE("default size", "[constructor]") { 10 | IntIntTable tbl; 11 | REQUIRE(tbl.size() == 0); 12 | REQUIRE(tbl.empty()); 13 | REQUIRE(tbl.hashpower() == (size_t) log2(DEFAULT_SIZE / 4)); 14 | REQUIRE(tbl.bucket_count() == DEFAULT_SIZE / 4); 15 | REQUIRE(tbl.load_factor() == 0); 16 | } 17 | 18 | TEST_CASE("given size", "[constructor]") { 19 | IntIntTable tbl(1); 20 | REQUIRE(tbl.size() == 0); 21 | REQUIRE(tbl.empty()); 22 | REQUIRE(tbl.hashpower() == 1); 23 | REQUIRE(tbl.bucket_count() == 2); 24 | REQUIRE(tbl.load_factor() == 0); 25 | } 26 | 27 | TEST_CASE("frees even with exceptions", "[constructor]") { 28 | typedef IntIntTableWithAlloc< TrackingAllocator> no_space_table; 29 | // Should throw when allocating the TableInfo struct 30 | REQUIRE_THROWS_AS(no_space_table(1), std::bad_alloc); 31 | REQUIRE(get_unfreed_bytes() == 0); 32 | 33 | typedef IntIntTableWithAlloc< 34 | TrackingAllocator> 35 | some_space_table; 36 | // Should throw when allocating the counters, after the buckets 37 | REQUIRE_THROWS_AS(some_space_table(1), std::bad_alloc); 38 | REQUIRE(get_unfreed_bytes() == 0); 39 | } 40 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_hash_properties.cc: -------------------------------------------------------------------------------- 1 | #include "catch.hpp" 2 | 3 | #include "../../src/cuckoohash_map.hh" 4 | #include "unit_test_util.hh" 5 | 6 | // Checks that the alt index function returns a different bucket, and can 7 | // recover the old bucket when called with the alternate bucket as the index. 8 | template 9 | void check_key(size_t hashpower, 10 | const typename CuckoohashMap::key_type& key) { 11 | auto hashfn = typename CuckoohashMap::hasher(); 12 | size_t hv = hashfn(key); 13 | auto partial = UnitTestInternalAccess::partial_key(hv); 14 | size_t bucket = UnitTestInternalAccess::index_hash( 15 | hashpower, hv); 16 | size_t alt_bucket = UnitTestInternalAccess::alt_index( 17 | hashpower, partial, bucket); 18 | size_t orig_bucket = UnitTestInternalAccess::alt_index( 19 | hashpower, partial, alt_bucket); 20 | 21 | REQUIRE(bucket != alt_bucket); 22 | REQUIRE(bucket == orig_bucket); 23 | } 24 | 25 | TEST_CASE("int alt index works correctly", "[hash properties]") { 26 | for (size_t hashpower = 10; hashpower < 15; ++hashpower) { 27 | for (int key = 0; key < 10000; ++key) { 28 | check_key(hashpower, key); 29 | } 30 | } 31 | } 32 | 33 | TEST_CASE("string alt index works correctly", "[hash properties]") { 34 | for (size_t hashpower = 10; hashpower < 15; ++hashpower) { 35 | for (int key = 0; key < 10000; ++key) { 36 | check_key(hashpower, std::to_string(key)); 37 | } 38 | } 39 | } 40 | 41 | TEST_CASE("hash with larger hashpower only adds top bits", 42 | "[hash properties]") { 43 | std::string key = "abc"; 44 | size_t hv = StringIntTable::hasher()(key); 45 | auto partial = UnitTestInternalAccess::partial_key(hv); 46 | for (size_t hashpower = 1; hashpower < 30; ++hashpower) { 47 | size_t index_bucket1 = UnitTestInternalAccess::index_hash< 48 | StringIntTable>(hashpower, hv); 49 | size_t index_bucket2 = UnitTestInternalAccess::index_hash< 50 | StringIntTable>(hashpower+1, hv); 51 | CHECK((index_bucket2 & ~(1L << hashpower)) == index_bucket1); 52 | 53 | size_t alt_bucket1 = UnitTestInternalAccess::alt_index< 54 | StringIntTable>(hashpower, partial, index_bucket1); 55 | size_t alt_bucket2 = UnitTestInternalAccess::alt_index< 56 | StringIntTable>(hashpower, partial, index_bucket2); 57 | 58 | CHECK((alt_bucket2 & ~(1L << hashpower)) == alt_bucket1); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_maximum_hashpower.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "catch.hpp" 4 | 5 | #include "../../src/cuckoohash_map.hh" 6 | #include "unit_test_util.hh" 7 | 8 | TEST_CASE("caps any expansion", "[maximum hash power]") { 9 | IntIntTable tbl(1, DEFAULT_MINIMUM_LOAD_FACTOR, 1); 10 | for (size_t i = 0; i < 2 * tbl.slot_per_bucket; ++i) { 11 | tbl.insert(i, i); 12 | } 13 | 14 | REQUIRE(tbl.hashpower() == 1); 15 | REQUIRE_THROWS_AS(tbl.insert(2*tbl.slot_per_bucket, 0), 16 | libcuckoo_maximum_hashpower_exceeded); 17 | REQUIRE_THROWS_AS(tbl.rehash(2), libcuckoo_maximum_hashpower_exceeded); 18 | REQUIRE_THROWS_AS(tbl.reserve(4*tbl.slot_per_bucket), 19 | libcuckoo_maximum_hashpower_exceeded); 20 | 21 | } 22 | 23 | TEST_CASE("no maximum hash power", "[maximum hash power]") { 24 | // It's difficult to check that we actually don't ever set a maximum hash 25 | // power, but if we explicitly unset it, we should be able to expand beyond 26 | // the limit that we had previously set. 27 | 28 | IntIntTable tbl(1, DEFAULT_MINIMUM_LOAD_FACTOR, 1); 29 | REQUIRE_THROWS_AS(tbl.rehash(2), libcuckoo_maximum_hashpower_exceeded); 30 | 31 | tbl.maximum_hashpower(2); 32 | tbl.rehash(2); 33 | REQUIRE(tbl.hashpower() == 2); 34 | REQUIRE_THROWS_AS(tbl.rehash(3), libcuckoo_maximum_hashpower_exceeded); 35 | 36 | tbl.maximum_hashpower(NO_MAXIMUM_HASHPOWER); 37 | tbl.rehash(10); 38 | REQUIRE(tbl.hashpower() == 10); 39 | } 40 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_minimum_load_factor.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "catch.hpp" 4 | 5 | #include "../../src/cuckoohash_map.hh" 6 | #include "unit_test_util.hh" 7 | 8 | class BadHashFunction { 9 | public: 10 | int operator()(int key) { 11 | return 0; 12 | } 13 | }; 14 | 15 | TEST_CASE("caps automatic expansion", "[minimum load fator]") { 16 | const size_t slot_per_bucket = 4; 17 | cuckoohash_map, 18 | std::allocator>, 19 | slot_per_bucket> tbl(16); 20 | tbl.minimum_load_factor(0.6); 21 | 22 | for (size_t i = 0; i < 2 * slot_per_bucket; ++i) { 23 | tbl.insert(i, i); 24 | } 25 | 26 | REQUIRE_THROWS_AS(tbl.insert(2*slot_per_bucket, 0), 27 | libcuckoo_load_factor_too_low); 28 | } 29 | 30 | TEST_CASE("invalid minimum load factor", "[minimum load factor]") { 31 | REQUIRE_THROWS_AS(IntIntTable(5, -0.01), std::invalid_argument); 32 | REQUIRE_THROWS_AS(IntIntTable(5, 1.01), std::invalid_argument); 33 | 34 | IntIntTable t; 35 | REQUIRE(t.minimum_load_factor() == DEFAULT_MINIMUM_LOAD_FACTOR); 36 | REQUIRE_THROWS_AS(t.minimum_load_factor(-0.01), std::invalid_argument); 37 | REQUIRE_THROWS_AS(t.minimum_load_factor(1.01), std::invalid_argument); 38 | } 39 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_noncopyable_types.cc: -------------------------------------------------------------------------------- 1 | #include "catch.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include "../../src/cuckoohash_map.hh" 7 | #include "unit_test_util.hh" 8 | 9 | typedef std::unique_ptr uptr; 10 | struct uptr_hash { 11 | size_t operator()(const uptr& ptr) const { 12 | return (*ptr) * 0xc6a4a7935bd1e995; 13 | } 14 | }; 15 | 16 | struct uptr_eq { 17 | bool operator()(const uptr& ptr1, const uptr& ptr2) const { 18 | return *ptr1 == *ptr2; 19 | } 20 | }; 21 | 22 | typedef cuckoohash_map uptr_tbl; 23 | 24 | const size_t TBL_INIT = 1; 25 | const size_t TBL_SIZE = TBL_INIT * uptr_tbl::slot_per_bucket * 2; 26 | 27 | void check_key_eq(uptr_tbl& tbl, int key, int expected_val) { 28 | REQUIRE(tbl.contains(uptr(new int(key)))); 29 | tbl.update_fn(uptr(new int(key)), [expected_val](const uptr& ptr) { 30 | REQUIRE(*ptr == expected_val); 31 | }); 32 | } 33 | 34 | TEST_CASE("noncopyable insert and update", "[noncopyable]") { 35 | uptr_tbl tbl(TBL_INIT); 36 | for (size_t i = 0; i < TBL_SIZE; ++i) { 37 | tbl.insert(uptr(new int(i)), uptr(new int(i))); 38 | } 39 | for (size_t i = 0; i < TBL_SIZE; ++i) { 40 | check_key_eq(tbl, i, i); 41 | } 42 | for (size_t i = 0; i < TBL_SIZE; ++i) { 43 | tbl.update(uptr(new int(i)), uptr(new int(i+1))); 44 | } 45 | for (size_t i = 0; i < TBL_SIZE; ++i) { 46 | check_key_eq(tbl, i, i+1); 47 | } 48 | } 49 | 50 | TEST_CASE("noncopyable upsert", "[noncopyable]") { 51 | uptr_tbl tbl(TBL_INIT); 52 | auto increment = [](uptr& ptr) { 53 | *ptr += 1; 54 | }; 55 | for (size_t i = 0; i < TBL_SIZE; ++i) { 56 | tbl.upsert(uptr(new int(i)), increment, uptr(new int(i))); 57 | } 58 | for (size_t i = 0; i < TBL_SIZE; ++i) { 59 | check_key_eq(tbl, i, i); 60 | } 61 | for (size_t i = 0; i < TBL_SIZE; ++i) { 62 | tbl.upsert(uptr(new int(i)), increment, uptr(new int(i))); 63 | } 64 | for (size_t i = 0; i < TBL_SIZE; ++i) { 65 | check_key_eq(tbl, i, i+1); 66 | } 67 | } 68 | 69 | TEST_CASE("noncopyable iteration", "[noncopyable]") { 70 | uptr_tbl tbl(TBL_INIT); 71 | for (size_t i = 0; i < TBL_SIZE; ++i) { 72 | tbl.insert(uptr(new int(i)), uptr(new int(i))); 73 | } 74 | { 75 | auto locked_tbl = tbl.lock_table(); 76 | for (auto& kv : locked_tbl) { 77 | REQUIRE(*kv.first == *kv.second); 78 | *kv.second += 1; 79 | } 80 | } 81 | { 82 | auto locked_tbl = tbl.lock_table(); 83 | for (auto& kv : locked_tbl) { 84 | REQUIRE(*kv.first == *kv.second - 1); 85 | } 86 | } 87 | } 88 | 89 | TEST_CASE("nested table", "[noncopyable]") { 90 | typedef cuckoohash_map inner_tbl; 91 | typedef cuckoohash_map> nested_tbl; 92 | nested_tbl tbl; 93 | std::string keys[] = {"abc", "def"}; 94 | for (std::string& k : keys) { 95 | tbl.insert(std::string(k), nested_tbl::mapped_type(new inner_tbl)); 96 | tbl.update_fn(k, [&k](nested_tbl::mapped_type& t) { 97 | for (char c : k) { 98 | t->insert(c, std::string(k)); 99 | } 100 | }); 101 | } 102 | for (std::string& k : keys) { 103 | REQUIRE(tbl.contains(k)); 104 | tbl.update_fn(k, [&k](nested_tbl::mapped_type& t) { 105 | for (char c : k) { 106 | REQUIRE(t->find(c) == k); 107 | } 108 | }); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_resize.cc: -------------------------------------------------------------------------------- 1 | #include "catch.hpp" 2 | 3 | #include "../../src/cuckoohash_map.hh" 4 | #include "unit_test_util.hh" 5 | 6 | TEST_CASE("rehash empty table", "[resize]") { 7 | IntIntTable table(1); 8 | REQUIRE(table.hashpower() == 1); 9 | 10 | table.rehash(20); 11 | REQUIRE(table.hashpower() == 20); 12 | 13 | table.rehash(1); 14 | REQUIRE(table.hashpower() == 1); 15 | } 16 | 17 | TEST_CASE("reserve empty table", "[resize]") { 18 | IntIntTable table(1); 19 | table.reserve(100); 20 | REQUIRE(table.hashpower() == 5); 21 | 22 | table.reserve(1); 23 | REQUIRE(table.hashpower() == 1); 24 | 25 | table.reserve(2); 26 | REQUIRE(table.hashpower() == 1); 27 | } 28 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/test_runner.cc: -------------------------------------------------------------------------------- 1 | // This file will be the entry point for the test runner 2 | #define CATCH_CONFIG_MAIN 3 | #include "catch.hpp" 4 | -------------------------------------------------------------------------------- /lib/libcuckoo/tests/unit-tests/unit_test_util.cc: -------------------------------------------------------------------------------- 1 | #include "unit_test_util.hh" 2 | 3 | std::atomic& get_unfreed_bytes() { 4 | static std::atomic unfreed_bytes(0L); 5 | return unfreed_bytes; 6 | } 7 | -------------------------------------------------------------------------------- /scripts/slaves: -------------------------------------------------------------------------------- 1 | ciidaa-a02 1234 2 | ciidaa-a03 1234 3 | ciidaa-a04 1234 4 | ciidaa-a05 1234 5 | ciidaa-a06 1234 6 | ciidaa-a07 1234 7 | ciidaa-a08 1234 8 | ciidaa-a09 1234 9 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | CPP = g++ -std=c++11 -O3 -DNDEBUG 2 | INCLUDE = -I../include -I../lib/libcuckoo/src -I../lib/libcuckoo/cityhash-1.1.1/src 3 | #INCLUDE = -I../include -I../lib/libcuckoo/src -I../lib/libcuckoo/cityhash-1.1.1/src -I/users/zhongle/local/include 4 | #LIBS = ../src/libgalloc.a -libverbs -lpthread ../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a /users/zhongle/local/lib/libboost_thread.so /users/zhongle/local/lib/libboost_system.so 5 | LIBS = ./libgalloc.a -libverbs -lpthread ../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a -lboost_thread -lboost_system 6 | CFLAGS += -rdynamic 7 | ifdef dht 8 | CFLAGS += -DDHT 9 | endif 10 | 11 | all: libgalloc.a 12 | test: libgalloc.a lock_test example example-r worker master rw_test fence_test benchmark map_test gfunc_test 13 | build: libgalloc.a libgalloc.so 14 | 15 | SRC = gfunc.cc ae.cc client.cc server.cc worker.cc gallocator.cc master.cc tcp.cc worker_handle.cc anet.cc rdma.cc util.cc zmalloc.cc log.cc slabs.cc workrequest.cc cache.cc directory.cc local_request.cc remote_request.cc pending_request.cc MurmurHash.cc logging.cc 16 | OBJ = gfunc.o ae.o client.o server.o worker.o gallocator.o master.o tcp.o worker_handle.o anet.o rdma.o util.o zmalloc.o log.o slabs.o workrequest.o cache.o directory.o remote_request.o local_request.o pending_request.o MurmurHash.o logging.o 17 | 18 | libgalloc.so: $(SRC) 19 | $(CPP) $(CFLAGS) $(INCLUDE) -fPIC -shared -o $@ $^ $(LIBS) 20 | 21 | $(OBJ): %.o : %.cc 22 | $(CPP) $< $(CFLAGS) $(INCLUDE) -g -c -o $@ 23 | 24 | libgalloc.a: $(OBJ) 25 | ar rcs $@ $^ 26 | 27 | benchmark: ../test/benchmark.cc 28 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 29 | 30 | lock_test: ../test/lock_test.cc 31 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 32 | 33 | example: ../test/example.cc 34 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 35 | 36 | example-r: ../test/example-r.cc 37 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 38 | 39 | worker: ../test/worker_test.cc 40 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 41 | 42 | master: ../test/master_test.cc 43 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 44 | 45 | rw_test: ../test/rw_test.cc 46 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 47 | 48 | fence_test: ../test/fence_test.cc 49 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 50 | 51 | map_test: ../test/map_test.cc 52 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 53 | 54 | gfunc_test: ../test/gfunc_test.cc 55 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 56 | 57 | clean_test: 58 | rm -rf benchmark example worker master example-r rw_test fence_test lock_test 59 | 60 | clean: 61 | rm -rf gfunc_test benchmark example worker master example-r rw_test fence_test lock_test map_test *.o *.a *.so *.d 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/MurmurHash.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * MurmurHash.c 3 | * MYUtilities 4 | * 5 | * This file created by Jens Alfke on 3/17/08. 6 | * Algorithm & source code by Austin Appleby, released to public domain. 7 | * 8 | * Downloaded 3/16/2008. 9 | * Modified slightly by Jens Alfke (use standard uint32_t and size_t types; 10 | * change 'm' and 'r' to #defines for better C compatibility.) 11 | * 12 | */ 13 | 14 | #include "MurmurHash.h" 15 | 16 | //----------------------------------------------------------------------------- 17 | // MurmurHash2, by Austin Appleby 18 | 19 | // Note - This code makes a few assumptions about how your machine behaves - 20 | 21 | // 1. We can read a 4-byte value from any address without crashing 22 | // 2. sizeof(int) == 4 **Jens: I fixed this by changing 'unsigned int' to 'uint32_t'** 23 | 24 | // And it has a few limitations - 25 | 26 | // 1. It will not work incrementally. 27 | // 2. It will not produce the same results on little-endian and big-endian 28 | // machines. 29 | 30 | uint32_t MurmurHash2(const void * key, size_t len, uint32_t seed) { 31 | // 'm' and 'r' are mixing constants generated offline. 32 | // They're not really 'magic', they just happen to work well. 33 | 34 | #define m 0x5bd1e995 35 | #define r 24 36 | 37 | // Initialize the hash to a 'random' value 38 | 39 | uint32_t h = seed ^ len; 40 | 41 | // Mix 4 bytes at a time into the hash 42 | 43 | const unsigned char * data = (const unsigned char *) key; 44 | 45 | while (len >= 4) { 46 | uint32_t k = *(uint32_t *) data; 47 | 48 | k *= m; 49 | k ^= k >> r; 50 | k *= m; 51 | 52 | h *= m; 53 | h ^= k; 54 | 55 | data += 4; 56 | len -= 4; 57 | } 58 | 59 | // Handle the last few bytes of the input array 60 | 61 | switch (len) { 62 | case 3: 63 | h ^= data[2] << 16; 64 | case 2: 65 | h ^= data[1] << 8; 66 | case 1: 67 | h ^= data[0]; 68 | h *= m; 69 | }; 70 | 71 | // Do a few final mixes of the hash to ensure the last few 72 | // bytes are well-incorporated. 73 | 74 | h ^= h >> 13; 75 | h *= m; 76 | h ^= h >> 15; 77 | 78 | return h; 79 | } 80 | 81 | -------------------------------------------------------------------------------- /src/ae_select.cc: -------------------------------------------------------------------------------- 1 | /* Select()-based ae.c module. 2 | * 3 | * Copyright (c) 2009-2012, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #include 32 | 33 | typedef struct aeApiState { 34 | fd_set rfds, wfds; 35 | /* We need to have a copy of the fd sets as it's not safe to reuse 36 | * FD sets after select(). */ 37 | fd_set _rfds, _wfds; 38 | } aeApiState; 39 | 40 | static int aeApiCreate(aeEventLoop *eventLoop) { 41 | aeApiState *state = zmalloc(sizeof(aeApiState)); 42 | 43 | if (!state) 44 | return -1; 45 | FD_ZERO(&state->rfds); 46 | FD_ZERO(&state->wfds); 47 | eventLoop->apidata = state; 48 | return 0; 49 | } 50 | 51 | static int aeApiResize(aeEventLoop *eventLoop, int setsize) { 52 | /* Just ensure we have enough room in the fd_set type. */ 53 | if (setsize >= FD_SETSIZE) 54 | return -1; 55 | return 0; 56 | } 57 | 58 | static void aeApiFree(aeEventLoop *eventLoop) { 59 | zfree(eventLoop->apidata); 60 | } 61 | 62 | static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { 63 | aeApiState *state = eventLoop->apidata; 64 | 65 | if (mask & AE_READABLE) 66 | FD_SET(fd, &state->rfds); 67 | if (mask & AE_WRITABLE) 68 | FD_SET(fd, &state->wfds); 69 | return 0; 70 | } 71 | 72 | static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { 73 | aeApiState *state = eventLoop->apidata; 74 | 75 | if (mask & AE_READABLE) 76 | FD_CLR(fd, &state->rfds); 77 | if (mask & AE_WRITABLE) 78 | FD_CLR(fd, &state->wfds); 79 | } 80 | 81 | static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { 82 | aeApiState *state = eventLoop->apidata; 83 | int retval, j, numevents = 0; 84 | 85 | memcpy(&state->_rfds, &state->rfds, sizeof(fd_set)); 86 | memcpy(&state->_wfds, &state->wfds, sizeof(fd_set)); 87 | 88 | retval = select(eventLoop->maxfd + 1, &state->_rfds, &state->_wfds, NULL, 89 | tvp); 90 | if (retval > 0) { 91 | for (j = 0; j <= eventLoop->maxfd; j++) { 92 | int mask = 0; 93 | aeFileEvent *fe = &eventLoop->events[j]; 94 | 95 | if (fe->mask == AE_NONE) 96 | continue; 97 | if (fe->mask & AE_READABLE && FD_ISSET(j, &state->_rfds)) 98 | mask |= AE_READABLE; 99 | if (fe->mask & AE_WRITABLE && FD_ISSET(j, &state->_wfds)) 100 | mask |= AE_WRITABLE; 101 | eventLoop->fired[numevents].fd = j; 102 | eventLoop->fired[numevents].mask = mask; 103 | numevents++; 104 | } 105 | } 106 | return numevents; 107 | } 108 | 109 | static char *aeApiName(void) { 110 | return "select"; 111 | } 112 | -------------------------------------------------------------------------------- /src/client.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "rdma.h" 9 | #include "anet.h" 10 | #include "log.h" 11 | #include "client.h" 12 | #include "server.h" 13 | #include "zmalloc.h" 14 | 15 | Client::Client(RdmaResource* res, bool isForMaster, const char* rdmaConnStr) 16 | : resource(res) { 17 | wid = free = size = 0; 18 | this->ctx = res->NewRdmaContext(isForMaster); 19 | if (rdmaConnStr) 20 | this->SetRemoteConnParam(rdmaConnStr); 21 | } 22 | 23 | int Client::ExchConnParam(const char* ip, int port, Server* server) { 24 | //open the socket to exch rdma resouces 25 | char neterr[ANET_ERR_LEN]; 26 | int sockfd = anetTcpConnect(neterr, const_cast(ip), port); 27 | if (sockfd < 0) { 28 | epicLog(LOG_WARNING, "Connecting to %s:%d %s", ip, port, neterr); 29 | exit(1); 30 | } 31 | 32 | const char* conn_str = GetConnString(server->GetWorkerId()); 33 | int conn_len = strlen(conn_str); 34 | if (write(sockfd, conn_str, conn_len) != conn_len) { 35 | return -1; 36 | } 37 | 38 | char msg[conn_len + 1]; 39 | /* waiting for server's response */ 40 | int n = read(sockfd, msg, conn_len); 41 | if (n != conn_len) { 42 | epicLog(LOG_WARNING, 43 | "Failed to read conn param from server (%s; read %d bytes)\n", 44 | strerror(errno), n); 45 | return -1; 46 | } 47 | msg[n] = '\0'; 48 | epicLog(LOG_INFO, "received conn string %s\n", msg); 49 | 50 | SetRemoteConnParam(msg); 51 | server->UpdateWidMap(this); 52 | 53 | /* 54 | * TODO: check whether this is needed! 55 | */ 56 | // if (write(sockfd, ACK_CONN_STRING, sizeof ACK_CONN_STRING)) { 57 | // /*do nothing */ 58 | // } 59 | if (IsForMaster()) 60 | server->PostConnectMaster(sockfd, server); 61 | 62 | close(sockfd); 63 | return 0; 64 | } 65 | 66 | int Client::SetRemoteConnParam(const char *conn) { 67 | const char* p = conn; 68 | if (resource->IsMaster()) { //in the Master thread, connected to worker 69 | wid = resource->GetCounter(); 70 | } else if (IsForMaster()) { //in the worker thread, but connected to Master 71 | sscanf(conn, "%x:", &wid); 72 | } else if (!resource->IsMaster()) { //in the worker thread, and connected to worker 73 | sscanf(conn, "%x:", &wid); 74 | } else { 75 | epicLog(LOG_WARNING, "undefined cases"); 76 | } 77 | p = strchr(conn, ':'); 78 | p++; 79 | return ctx->SetRemoteConnParam(p); 80 | } 81 | 82 | const char* Client::GetConnString(int workerid) { 83 | const char* rdmaConn = ctx->GetRdmaConnString(); 84 | if (!connstr) 85 | connstr = (char*) zmalloc(MAX_CONN_STRLEN + 1); 86 | 87 | if (resource->IsMaster()) { //in the Master thread 88 | sprintf(connstr, "%04x:%s", wid, rdmaConn); //wid is already set 89 | epicLog(LOG_DEBUG, "master to worker here"); 90 | } else if (IsForMaster()) { //in the worker thread, but connected to Master 91 | sprintf(connstr, "%04x:%s", 0, rdmaConn); 92 | epicLog(LOG_DEBUG, "worker to master here"); 93 | } else if (!resource->IsMaster()) { //in the worker thread, and connected to worker 94 | epicAssert(workerid != 0); 95 | sprintf(connstr, "%04x:%s", workerid, rdmaConn); //wid is the current worker id (not the remote pair's) 96 | } else { 97 | epicLog(LOG_WARNING, "undefined cases"); 98 | } 99 | epicLog(LOG_DEBUG, "conn str %s\n", connstr); 100 | 101 | /* FIXME: rdmaConn does not get freed? rdmaConn is not null-terminated 102 | * string; will print it using %s format cause problems? */ 103 | return connstr; 104 | } 105 | 106 | Client::~Client() { 107 | resource->DeleteRdmaContext(ctx); 108 | } 109 | -------------------------------------------------------------------------------- /src/gfunc.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include "settings.h" 4 | #ifdef GFUNC_SUPPORT 5 | 6 | #include "gfunc.h" 7 | #include "log.h" 8 | #include "gallocator.h" 9 | #include "util.h" 10 | 11 | void IncrDouble(void* ptr, uint64_t arg) { 12 | double a = force_cast(arg); 13 | epicLog(LOG_WARNING, "a = %lf", a); 14 | if (arg == 0) { 15 | (*(char*) ptr)++; 16 | } else { 17 | (*(char*) ptr) += a; 18 | } 19 | } 20 | 21 | void Incr(void* ptr, uint64_t arg) { 22 | if (arg == 0) { 23 | (*(char*) ptr)++; 24 | } else { 25 | (*(char*) ptr) += arg; 26 | } 27 | } 28 | 29 | /** import from Graph to support GFunc that can operate on Vertex 30 | * TODO: move Storage of graph engine to GAM 31 | */ 32 | struct Vertex { 33 | // general 34 | uint64_t vertex_id;bool active, active_minor_step; 35 | // currently assume data type is only double 36 | // app data 37 | double value; 38 | // vertex program 39 | double accumulator; 40 | // for scatter 41 | double delta; 42 | }; 43 | 44 | void GatherPagerank(void *ptr, uint64_t argc) { 45 | // for simplicity and performance, we use push-based weight propagation instead of pull-based gathering 46 | // here, reduce value in vertex.accumulator atomically 47 | Vertex *vertex = reinterpret_cast(ptr); 48 | double value = force_cast(argc); 49 | vertex->accumulator += value; 50 | } 51 | 52 | void ApplyPagerank(void *ptr, uint64_t argc) { 53 | const static double REST_PROB = 0.15; 54 | const static double TOLERANCE = 0.01; 55 | Vertex *vertex_data = reinterpret_cast(ptr); 56 | if (vertex_data->active) { 57 | vertex_data->active = false; 58 | double new_pagerank = vertex_data->accumulator * (1.0 - REST_PROB) 59 | + REST_PROB; 60 | vertex_data->delta = new_pagerank - vertex_data->value; 61 | vertex_data->active_minor_step = std::abs(vertex_data->delta) > TOLERANCE; 62 | vertex_data->value = new_pagerank; 63 | } else { 64 | vertex_data->active_minor_step = false; 65 | } 66 | } 67 | void ScatterPagerank(void *ptr, uint64_t argc) { 68 | double delta = force_cast(argc); 69 | Vertex *vertex_data = reinterpret_cast(ptr); 70 | vertex_data->accumulator += delta; 71 | vertex_data->active = true; 72 | } 73 | 74 | int GetGFuncID(GFunc* gfunc) { 75 | if (!gfunc) 76 | return -1; 77 | for (int i = 0; i < sizeof(GAllocFactory::gfuncs); i++) { 78 | if (GAllocFactory::gfuncs[i] == gfunc) { 79 | return i; 80 | } 81 | } 82 | epicLog(LOG_WARNING, "cannot find the gfunc in the gfunc directory"); 83 | return -1; 84 | } 85 | 86 | GFunc* GetGFunc(int id) { 87 | if (id == -1) 88 | return nullptr; 89 | epicAssert(id < sizeof(GAllocFactory::gfuncs)); 90 | return GAllocFactory::gfuncs[id]; 91 | } 92 | 93 | #endif 94 | 95 | -------------------------------------------------------------------------------- /src/log.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "log.h" 10 | #include "gallocator.h" 11 | 12 | void _epicLogRaw(int level, const char* msg) { 13 | const char *c = ".-*#"; 14 | FILE *fp; 15 | char buf[128]; 16 | 17 | fp = 18 | (GAllocFactory::LogFile() == nullptr) ? 19 | (level <= LOG_FATAL ? stderr : stdout) : 20 | fopen(GAllocFactory::LogFile()->c_str(), "a"); 21 | if (!fp) 22 | return; 23 | 24 | int off; 25 | struct timeval tv; 26 | 27 | gettimeofday(&tv, NULL); 28 | off = strftime(buf, sizeof(buf), "%d %b %H:%M:%S.", localtime(&tv.tv_sec)); 29 | snprintf(buf + off, sizeof(buf) - off, "%03d", (int) tv.tv_usec / 1000); 30 | //fprintf(fp,"[%d] %s %c %s\n",(int)getpid(),buf,c[level],msg); 31 | fprintf(fp, "[%d] %s %c %s\n", (int) syscall(SYS_gettid), buf, c[level], msg); 32 | 33 | fflush(fp); 34 | 35 | if (GAllocFactory::LogFile()) 36 | fclose(fp); 37 | } 38 | 39 | void _epicLog(char* file, char* func, int lineno, int level, const char *fmt, 40 | ...) { 41 | if (level > GAllocFactory::LogLevel()) 42 | return; 43 | 44 | va_list ap; 45 | char msg[MAX_LOGMSG_LEN]; 46 | 47 | int n = sprintf(msg, "[%s:%d-%s()] ", file, lineno, func); 48 | va_start(ap, fmt); 49 | vsnprintf(msg + n, MAX_LOGMSG_LEN - n, fmt, ap); 50 | va_end(ap); 51 | 52 | _epicLogRaw(level, msg); 53 | } 54 | 55 | void PrintStackTrace() { 56 | printf("\n***************Start Stack Trace******************\n"); 57 | int size = 100; 58 | void *buffer[100]; 59 | char **strings; 60 | int j, nptrs; 61 | nptrs = backtrace(buffer, size); 62 | printf("backtrace() returned %d addresses\n", nptrs); 63 | strings = backtrace_symbols(buffer, nptrs); 64 | if (strings == NULL) { 65 | perror("backtrace_symbols"); 66 | exit(EXIT_FAILURE); 67 | } 68 | for (j = 0; j < nptrs; j++) { 69 | printf("%s\n", strings[j]); 70 | } 71 | free(strings); 72 | 73 | printf("\n***************End Stack Trace******************\n"); 74 | } 75 | 76 | -------------------------------------------------------------------------------- /src/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include "logging.h" 10 | #include "log.h" 11 | 12 | constexpr Size BUF_SIZE = 128 * 1024 * 1024; 13 | 14 | static Size distance(int start, int end) { 15 | return (start < end) ? end - start : end + BUF_SIZE - start; 16 | } 17 | 18 | Log::Log(void* base): base_(base) 19 | , spos_(0) 20 | , epos_(BUF_SIZE) 21 | , running_(false) 22 | , buf_(new char[BUF_SIZE]) 23 | , thread_(std::this_thread::get_id()) 24 | { 25 | fd_ = open("/data/caiqc/gam.log", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 26 | } 27 | 28 | Log::~Log() { 29 | close(fd_); 30 | delete buf_; 31 | } 32 | 33 | int Log::reserve(Size sz) { 34 | 35 | bool reserved = false; 36 | while (!reserved) { 37 | bool fvalue = false; 38 | if (distance(spos_, epos_) >= sz) { 39 | uint_fast32_t spos = spos_.load(); 40 | uint_fast32_t npos = (spos + sz) % BUF_SIZE; 41 | if (spos_.compare_exchange_strong(spos, npos)) { 42 | if (distance(spos_, epos_) < BUF_SIZE / 2) { 43 | if (running_.compare_exchange_strong(fvalue, true)) { 44 | //epicLog(LOG_WARNING, "start reclaim, running_ = %d", running_ ? 1 : 0); 45 | async_ = std::async(std::launch::async, &Log::write, this); 46 | } 47 | } 48 | return spos; 49 | } 50 | } else { 51 | if (running_.compare_exchange_strong(fvalue, true)) { 52 | write(); 53 | } 54 | } 55 | } 56 | } 57 | 58 | void Log::write() { 59 | 60 | // append [epos, spos) of buf to the log file 61 | int start = epos_ % BUF_SIZE; 62 | int end = spos_; 63 | 64 | Size avail = distance(start, end); 65 | 66 | epicLog(LOG_WARNING, " start reclaim: avail = %llu MB", avail >> 20); 67 | 68 | if (start < end) { 69 | ::write(fd_, buf_ + start, end - start); 70 | } else { 71 | ::write(fd_, buf_ + end, BUF_SIZE - end); 72 | ::write(fd_, buf_, start); 73 | } 74 | 75 | fsync(fd_); 76 | epos_.store(end); 77 | running_.store(false); 78 | } 79 | 80 | int Log::writeToBuf(void* ptr, Size size, int spos) { 81 | if (spos + size < BUF_SIZE) { 82 | memcpy(spos + buf_, ptr, size); 83 | return spos + size; 84 | } else { 85 | int bytesToEnd = BUF_SIZE - spos; 86 | memcpy(spos + buf_, ptr, bytesToEnd); 87 | memcpy(buf_, ptr + bytesToEnd, size - bytesToEnd); 88 | return size - bytesToEnd; 89 | } 90 | } 91 | 92 | void Log::logWrite(GAddr addr, Size size, const void* content) { 93 | 94 | int sz = size + sizeof addr + sizeof size; 95 | int spos = reserve(sz); 96 | 97 | // simply memory copy 98 | void* p = TO_LOCAL(addr, base_); 99 | 100 | spos = writeToBuf(&addr, sizeof(GAddr), spos); 101 | spos = writeToBuf(&size, sizeof(Size), spos); 102 | spos = writeToBuf(p, size, spos); 103 | 104 | //if (distance(spos, epos_) < BUF_SIZE / 2) { 105 | // if (running_.compare_exchange_strong(FALSE_VALUE, true)) { 106 | // async_ = std::async(std::launch::async, &Log::write, this); 107 | // } 108 | //} 109 | } 110 | 111 | void Log::logOwner(int node, GAddr addr) { 112 | Size id = node; 113 | int sz = sizeof addr + sizeof id; 114 | int spos = reserve(sz); 115 | 116 | id |= (1ul << (8*(sizeof(Size)) - 1)); 117 | spos = writeToBuf(&addr, sizeof(GAddr), spos); 118 | spos = writeToBuf(&id, sizeof(id), spos); 119 | } 120 | -------------------------------------------------------------------------------- /src/tcp.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include "tcp.h" 4 | #include "rdma.h" 5 | #include "log.h" 6 | #include "server.h" 7 | #include "client.h" 8 | #include "anet.h" 9 | #include "kernel.h" 10 | #include "zmalloc.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | void AcceptTcpClientHandle(aeEventLoop *el, int fd, void *data, int mask) { 17 | epicAssert(data != nullptr); 18 | Server *server = (Server*) (data); 19 | char msg[MAX_CONN_STRLEN + 1]; 20 | int n; 21 | const char *p; 22 | Client *cli; 23 | char neterr[ANET_ERR_LEN]; 24 | char cip[IP_STR_LEN]; 25 | int cfd, cport; 26 | 27 | cfd = anetTcpAccept(neterr, fd, cip, sizeof(cip), &cport); 28 | if (cfd == ANET_ERR) { 29 | if (errno != EWOULDBLOCK) 30 | epicLog(LOG_WARNING, "Accepting client connection: %s", neterr); 31 | return; 32 | } 33 | epicLog(LOG_INFO, "Accepted %s:%d", cip, cport); 34 | 35 | if (mask & AE_READABLE) { 36 | n = read(cfd, msg, sizeof msg); 37 | if (unlikely(n <= 0)) { 38 | epicLog(LOG_WARNING, "Unable to read conn string\n"); 39 | goto out; 40 | } 41 | msg[n] = '\0'; 42 | epicLog(LOG_INFO, "conn string %s\n", msg); 43 | } 44 | 45 | if (unlikely(!(cli = server->NewClient(msg)))) { 46 | goto out; 47 | } 48 | server->UpdateWidMap(cli); 49 | 50 | if (unlikely(!(p = cli->GetConnString(server->GetWorkerId())))) { 51 | goto out; 52 | } 53 | 54 | server->UpdateWidMap(cli); 55 | 56 | n = write(cfd, p, strlen(p)); 57 | 58 | if (unlikely(n < strlen(p))) { 59 | epicLog(LOG_WARNING, "Unable to send conn string\n"); 60 | server->RmClient(cli); 61 | } 62 | 63 | if (server->IsMaster()) 64 | server->PostAcceptWorker(cfd, server); 65 | 66 | out: close(cfd); 67 | } 68 | 69 | void ProcessRdmaRequestHandle(aeEventLoop *el, int fd, void *data, int mask) { 70 | ((Server *) data)->ProcessRdmaRequest(); 71 | } 72 | -------------------------------------------------------------------------------- /src/util.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "util.h" 15 | 16 | template<> 17 | vector& Split(stringstream& ss, vector& elems, 18 | char delim) { 19 | string item; 20 | while (getline(ss, item, delim)) { 21 | if (!item.empty()) 22 | elems.push_back(item); 23 | } 24 | return elems; 25 | } 26 | 27 | struct timespec init_time; 28 | void init() __attribute__ ((constructor)); 29 | void fini() __attribute__ ((destructor)); 30 | void init() { 31 | clock_gettime(CLOCK_REALTIME, &init_time); 32 | } 33 | 34 | void fini() { 35 | } 36 | 37 | /* 38 | * initial time that is used to avoid long overflow 39 | * return the current time in nanoseconds 40 | */ 41 | long get_time() { 42 | // struct timeval start; 43 | // gettimeofday(&start, NULL); 44 | // return start.tv_sec*1000l*1000+start.tv_usec; 45 | struct timespec start; 46 | clock_gettime(CLOCK_REALTIME, &start); 47 | return (start.tv_sec - init_time.tv_sec) * 1000l * 1000 * 1000 48 | + (start.tv_nsec - init_time.tv_nsec);; 49 | } 50 | 51 | //get the ip address of the first interface 52 | string get_local_ip(const char* iface) { 53 | int MAXINTERFACES = 16; 54 | char *ip = NULL; 55 | int fd, intrface; 56 | struct ifreq buf[MAXINTERFACES]; 57 | struct ifconf ifc; 58 | 59 | if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) >= 0) { 60 | ifc.ifc_len = sizeof(buf); 61 | ifc.ifc_buf = (caddr_t) buf; 62 | if (!ioctl(fd, SIOCGIFCONF, (char *) &ifc)) { 63 | intrface = ifc.ifc_len / sizeof(struct ifreq); 64 | 65 | while (intrface-- > 0) { 66 | if (!(ioctl(fd, SIOCGIFADDR, (char *) &buf[intrface]))) { 67 | ip = (inet_ntoa( 68 | ((struct sockaddr_in*) (&buf[intrface].ifr_addr))->sin_addr)); 69 | if (!iface || strcmp(buf[intrface].ifr_name, iface) == 0) { 70 | break; 71 | } 72 | } 73 | } 74 | } 75 | close(fd); 76 | } 77 | return string(ip); 78 | } 79 | 80 | unsigned char* mac_eth(char *eth) { 81 | #define HWADDR_len 6 82 | int s, i; 83 | int mac_len = 17; 84 | unsigned char * MAC_str = (unsigned char*) malloc(sizeof(char) * 18); //13 85 | struct ifreq ifr; 86 | s = socket(AF_INET, SOCK_DGRAM, 0); 87 | strcpy(ifr.ifr_name, eth); 88 | ioctl(s, SIOCGIFHWADDR, &ifr); 89 | for (i = 0; i < HWADDR_len - 1; i++) { 90 | sprintf((char*) &MAC_str[i * 3], "%02x:", 91 | ((char*) ifr.ifr_hwaddr.sa_data)[i]); 92 | } 93 | sprintf((char*) &MAC_str[i * 3], "%02x", ((char*) ifr.ifr_hwaddr.sa_data)[i]); 94 | MAC_str[mac_len] = '\0'; 95 | return MAC_str; 96 | } 97 | 98 | char* get_hostname() { 99 | char *Name = (char*) malloc(150); 100 | memset(Name, 0, 150); 101 | gethostname(Name, 150); 102 | return Name; 103 | } 104 | 105 | char* get_ipbyname(char* name) { 106 | struct hostent *h; 107 | char * ip; 108 | 109 | /* get the host info */ 110 | if ((h = gethostbyname(name)) == NULL) { 111 | herror("gethostbyname(): "); 112 | exit(1); 113 | } else { 114 | ip = inet_ntoa(*((struct in_addr *) h->h_addr)); 115 | } 116 | return ip; 117 | } 118 | 119 | // Windows 120 | #ifdef _WIN32 121 | 122 | #include 123 | uint64_t rdtsc() { 124 | return __rdtsc(); 125 | } 126 | 127 | // Linux/GCC 128 | #else 129 | 130 | uint64_t rdtsc() { 131 | unsigned int lo, hi; 132 | __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); 133 | return ((uint64_t) hi << 32) | lo; 134 | } 135 | 136 | #endif 137 | 138 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | CPP = g++ -std=c++11 -O3 2 | INCLUDE = -I../include -I../lib/libcuckoo/src -I../lib/libcuckoo/cityhash-1.1.1/src 3 | LIBS = ../src/libgalloc.a -libverbs -lpthread ../lib/libcuckoo/cityhash-1.1.1/src/.libs/libcityhash.a -lboost_thread -lboost_system 4 | CFLAGS += -g -rdynamic 5 | 6 | build: benchmark lru_test example example-r worker master slab_test rw_test fence_test lock_test hashtable_test hashtable_throw_test garray_test cs_test 7 | 8 | benchmark: benchmark.cc 9 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 10 | 11 | lru_test: lru_test.cc 12 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 13 | 14 | lock_test: lock_test.cc 15 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 16 | 17 | fence_test: fence_test.cc 18 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 19 | 20 | rw_test: rw_test.cc 21 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 22 | 23 | slab_test: slab_test.cc 24 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 25 | 26 | example: example.cc 27 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 28 | 29 | example-r: example-r.cc 30 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 31 | 32 | worker: worker_test.cc 33 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 34 | 35 | master: master_test.cc 36 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 37 | 38 | hashtable_test: hashtable_test.cc 39 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 40 | 41 | hashtable_throw_test: hashtable_throw_test.cc 42 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 43 | 44 | garray_test: garray_test.cc 45 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 46 | 47 | cs_test: cs_test.cc 48 | $(CPP) $(CFLAGS) $(INCLUDE) -o $@ $^ $(LIBS) 49 | 50 | gfunc_test: ../test/gfunc_test.cc 51 | $(CPP) $(CFLAGS) $(INCLUDE) -g -o $@ $^ $(LIBS) 52 | 53 | clean: 54 | rm -rf example example-r worker master slab_test rw_test lock_test lru_test fence_test benchmark hashtable_test hashtable_throw_test garray_test cs_test gfunc_test 55 | -------------------------------------------------------------------------------- /test/cs_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include "structure.h" 7 | #include "worker.h" 8 | #include "settings.h" 9 | #include "worker_handle.h" 10 | #include "master.h" 11 | #include "gallocator.h" 12 | #include "workrequest.h" 13 | #include "util.h" 14 | 15 | int main() { 16 | ibv_device **list = ibv_get_device_list(NULL); 17 | int level = LOG_WARNING; 18 | 19 | //master 20 | Conf* conf = new Conf(); 21 | conf->loglevel = level; 22 | GAllocFactory::SetConf(conf); 23 | Master* master = new Master(*conf); 24 | 25 | //worker1 26 | conf = new Conf(); 27 | conf->loglevel = level; 28 | RdmaResource* res = new RdmaResource(list[0], false); 29 | Worker *worker1, *worker2, *worker3; 30 | WorkerHandle *wh1, *wh2, *wh3; 31 | worker1 = new Worker(*conf, res); 32 | wh1 = new WorkerHandle(worker1); 33 | 34 | //worker2 35 | conf = new Conf(); 36 | conf->loglevel = level; 37 | res = new RdmaResource(list[0], false); 38 | conf->worker_port += 1; 39 | worker2 = new Worker(*conf, res); 40 | wh2 = new WorkerHandle(worker2); 41 | 42 | //client1 43 | conf = new Conf(); 44 | conf->loglevel = level; 45 | conf->cache_th = 1; 46 | res = new RdmaResource(list[0], false); 47 | conf->worker_port += 2; 48 | worker3 = new Worker(*conf, res); 49 | wh3 = new WorkerHandle(worker3); 50 | 51 | sleep(2); 52 | 53 | WorkRequest wr { }; 54 | Size size = 10; 55 | GAddr gaddr; 56 | int i = 0; 57 | char buf[size]; 58 | char wbuf[size]; 59 | char local_init = 1; 60 | char remote_init = 2; 61 | 62 | for (i = 0; i < size; i++) { 63 | wr.Reset(); 64 | wr.op = MALLOC; 65 | wr.size = size; 66 | wr.flag = RANDOM; 67 | if (wh1->SendRequest(&wr)) { 68 | epicLog(LOG_WARNING, "send request failed"); 69 | } 70 | gaddr = wr.addr; 71 | epicAssert(WID(wr.addr) != 3); 72 | epicLog(LOG_WARNING, "allocated %ld at %lx", size, gaddr); 73 | 74 | wr.Reset(); 75 | wr.addr = gaddr; 76 | wr.op = FREE; 77 | if (wh1->SendRequest(&wr)) { 78 | epicLog(LOG_WARNING, "send request failed"); 79 | } 80 | } 81 | 82 | for (i = 0; i < size; i++) { 83 | wr.Reset(); 84 | wr.op = MALLOC; 85 | wr.size = size; 86 | wr.flag = REMOTE; 87 | if (wh1->SendRequest(&wr)) { 88 | epicLog(LOG_WARNING, "send request failed"); 89 | } 90 | gaddr = wr.addr; 91 | epicAssert(WID(wr.addr) != 3); 92 | epicLog(LOG_WARNING, "allocated %ld at %lx", size, gaddr); 93 | 94 | wr.Reset(); 95 | wr.addr = gaddr; 96 | wr.op = FREE; 97 | if (wh1->SendRequest(&wr)) { 98 | epicLog(LOG_WARNING, "send request failed"); 99 | } 100 | } 101 | 102 | for (i = 0; i < size; i++) { 103 | wr.Reset(); 104 | wr.op = MALLOC; 105 | wr.size = size; 106 | wr.flag = RANDOM; 107 | if (wh3->SendRequest(&wr)) { 108 | epicLog(LOG_WARNING, "send request failed"); 109 | } 110 | gaddr = wr.addr; 111 | epicAssert(WID(wr.addr) != 3); 112 | epicLog(LOG_WARNING, "allocated %ld at %lx", size, gaddr); 113 | 114 | wr.Reset(); 115 | wr.addr = gaddr; 116 | wr.op = FREE; 117 | if (wh3->SendRequest(&wr)) { 118 | epicLog(LOG_WARNING, "send request failed"); 119 | } 120 | } 121 | 122 | for (i = 0; i < size; i++) { 123 | wr.Reset(); 124 | wr.op = MALLOC; 125 | wr.size = size; 126 | wr.flag = REMOTE; 127 | if (wh3->SendRequest(&wr)) { 128 | epicLog(LOG_WARNING, "send request failed"); 129 | } 130 | gaddr = wr.addr; 131 | epicAssert(WID(wr.addr) != 3); 132 | epicLog(LOG_WARNING, "allocated %ld at %lx", size, gaddr); 133 | 134 | wr.Reset(); 135 | wr.addr = gaddr; 136 | wr.op = FREE; 137 | if (wh3->SendRequest(&wr)) { 138 | epicLog(LOG_WARNING, "send request failed"); 139 | } 140 | } 141 | 142 | sleep(2); 143 | epicLog(LOG_WARNING, "test done"); 144 | 145 | return 0; 146 | } 147 | 148 | -------------------------------------------------------------------------------- /test/example-r.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include "gallocator.h" 8 | #include "log.h" 9 | 10 | // 11 | using namespace std; 12 | 13 | int main() { 14 | Conf conf; 15 | conf.loglevel = LOG_WARNING; 16 | int size = 4097; 17 | int iteration = 10000; 18 | 19 | GAlloc* allocator = GAllocFactory::CreateAllocator(&conf); 20 | GAlloc* allocator2 = GAllocFactory::CreateAllocator(); 21 | 22 | sleep(2); 23 | 24 | GAddr lptr = allocator->AlignedMalloc(sizeof(int)); 25 | int i = 2, j = 0; 26 | 27 | if (!allocator->Try_WLock(lptr, sizeof(int))) { 28 | allocator->Write(lptr, &i, sizeof(int)); 29 | allocator->Read(lptr, &j, sizeof(int)); 30 | epicLog(LOG_WARNING, "j = %d", j); 31 | allocator->UnLock(lptr, sizeof(int)); 32 | } 33 | 34 | GAddr lptr2 = allocator2->AlignedMalloc(size); 35 | epicLog(LOG_WARNING, "allocate addr at %lx", lptr2); 36 | char buf[size], wbuf[size]; 37 | for (int k = 0; k < size; k++) { 38 | wbuf[k] = k; 39 | } 40 | 41 | Key key = 1; 42 | GAddr rlptr2 = 0; 43 | allocator->Put(key, &lptr2, sizeof(GAddr)); 44 | int s = allocator->Get(key, &rlptr2); 45 | epicAssert(s == sizeof(GAddr)); 46 | epicLog(LOG_WARNING, "get key %ld: %lx", key, rlptr2); 47 | 48 | sleep(2); 49 | while (iteration--) { 50 | while (allocator->Try_WLock(lptr2, size)) 51 | ; 52 | allocator->Write(lptr2, wbuf, size); 53 | allocator->Read(lptr2, buf, size); 54 | allocator->UnLock(lptr2, size); 55 | int k; 56 | for (k = 0; k < size; k++) { 57 | if (buf[k] != wbuf[k]) { 58 | epicLog(LOG_WARNING, "read failed buf[%d] (%d) != wbuf[%d] (%d)", k, 59 | buf[k], k, wbuf[k]); 60 | exit(-1); 61 | } 62 | } 63 | if (k == size) { 64 | cout << "read succeed (iteration = " << iteration << ")" << endl; 65 | } else { 66 | cout << "read failed (iteration = " << iteration << ")" << endl; 67 | exit(1); 68 | } 69 | } 70 | 71 | cout << "end" << endl; 72 | sleep(10); 73 | cout << "done" << endl; 74 | 75 | return 0; 76 | } 77 | 78 | -------------------------------------------------------------------------------- /test/example.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include "gallocator.h" 8 | #ifdef GFUNC_SUPPORT 9 | #include "gfunc.h" 10 | #endif 11 | #include "util.h" 12 | 13 | using namespace std; 14 | 15 | int main() { 16 | Conf conf; 17 | conf.loglevel = LOG_WARNING; 18 | conf.is_master = true; 19 | conf.worker_port = 12347; 20 | 21 | GAlloc* allocator = GAllocFactory::CreateAllocator(&conf); 22 | GAddr lptr = allocator->Malloc(sizeof(int), RANDOM); 23 | printf("%lx is local? %s, local addr = %p\n", lptr, 24 | allocator->IsLocal(lptr) == true ? "true" : "false", 25 | allocator->GetLocal(lptr)); 26 | int i = 2, j = 0; 27 | allocator->Write(lptr, &i, sizeof(int)); 28 | allocator->Read(lptr, &j, sizeof(int)); 29 | cout << "j = " << j << endl; 30 | 31 | #ifdef GFUNC_SUPPORT 32 | double inc = 2.2; 33 | uint64_t incl = force_cast(inc); 34 | allocator->Write(lptr, &i, sizeof(int), IncrDouble, incl); 35 | allocator->Read(lptr, &j, sizeof(int)); 36 | cout << "j = " << j << endl; 37 | 38 | allocator->Write(lptr, &i, sizeof(int), IncrDouble, 39 | force_cast(2.0)); 40 | allocator->Read(lptr, &j, sizeof(int)); 41 | cout << "j = " << j << endl; 42 | #endif 43 | 44 | allocator->Free(lptr); 45 | 46 | // Size size = 4097; 47 | // char buf[size]; 48 | // char wbuf[size]; 49 | // for(int i = 0; i < size; i++) { 50 | // wbuf[i] = (char)i; 51 | // } 52 | // 53 | // GAddr rptr = 0; // allocator->AlignedMalloc(size, REMOTE); 54 | // Key key = 1; 55 | // int s = allocator->Get(key, &rptr); 56 | // epicAssert(s == sizeof(GAddr)); 57 | // epicLog(LOG_WARNING, "get key %ld: %lx", key, rptr); 58 | // 59 | // sleep(1); 60 | // 61 | // while(allocator->Try_RLock(rptr, size)); 62 | // allocator->Read(rptr, buf, size); 63 | // for(i = 0; i < size; i++) { 64 | // if(buf[i] != wbuf[i]) { 65 | // cout << "Error" << endl; 66 | // printf("read buf[%d:%d] = %d\n", i, (char)i, buf[i]); 67 | // break; 68 | // } 69 | // } 70 | // if(i == size) cout << "first read succeed!" << endl; 71 | // allocator->UnLock(rptr, size); 72 | // 73 | // while(iteration--) { 74 | // while (allocator->Try_WLock(rptr, size)); 75 | // memset(buf, 0, size); 76 | // allocator->Write(rptr, wbuf, size); 77 | // for (int i = 0; i < size; i++) { 78 | // wbuf[i]++; 79 | // } 80 | // allocator->Write(rptr, wbuf, size); 81 | // allocator->Read(rptr, buf, size); 82 | // allocator->Read(rptr, buf, size); 83 | // for (i = 0; i < size; i++) { 84 | // if (buf[i] != wbuf[i]) { 85 | // cout << "Error" << endl; 86 | // printf("read buf[%d:%d] = %d\n", i, (char) i, buf[i]); 87 | // break; 88 | // } 89 | // } 90 | // if (i == size) { 91 | // cout << "read succeed (iteration = " << iteration << ")" << endl; 92 | // } else { 93 | // cout << "read failed (iteration = " << iteration << ")" << endl; 94 | // exit(1); 95 | // } 96 | // allocator->UnLock(rptr, size); 97 | // } 98 | // 99 | // while(allocator->Try_WLock(rptr, size)); 100 | // memset(buf, 0, size); 101 | // for(i = 0; i < size; i++) { 102 | // j = i+1; 103 | // allocator->Write(GADD(rptr, i), &j , sizeof(char)); 104 | // j++; 105 | // allocator->Write(GADD(rptr, i), &j , sizeof(char)); 106 | // allocator->Read(GADD(rptr, i), &buf[i], sizeof(char)); 107 | // if(buf[i] != (char)j) { 108 | // cout << "Error" << endl; 109 | // printf("read buf[%d:%d] = %d\n", i, (char)j, buf[i]); 110 | // break; 111 | // } 112 | // } 113 | // if(i == size) { 114 | // cout << "third read succeed!" << endl; 115 | // } else { 116 | // cout << "third read failed!" << endl; 117 | // exit(1); 118 | // } 119 | // allocator->UnLock(rptr, size); 120 | // 121 | // allocator->Free(lptr); 122 | // allocator->Free(rptr); 123 | // sleep(10); 124 | // cout << "done" << endl; 125 | 126 | return 0; 127 | } 128 | 129 | -------------------------------------------------------------------------------- /test/hashtable_throw_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "hashtable.h" 8 | 9 | int main() { 10 | HashTable hashtable; 11 | hashtable[0] = "string_0"; 12 | try { 13 | std::string x = hashtable[0]; 14 | std::cout << x << std::endl; 15 | } catch (const exception& e) { 16 | std::cout << e.what() << std::endl; 17 | } 18 | try { 19 | std::string x = hashtable.at(0); 20 | std::cout << x << std::endl; 21 | } catch (const exception& e) { 22 | std::cout << e.what() << std::endl; 23 | } 24 | try { 25 | std::string x = hashtable[1]; 26 | std::cout << x << std::endl; 27 | } catch (const exception& e) { 28 | std::cout << e.what() << std::endl; 29 | } 30 | try { 31 | std::string x = hashtable.at(1); 32 | std::cout << x << std::endl; 33 | } catch (const exception& e) { 34 | std::cout << e.what() << std::endl; 35 | } 36 | hashtable[1] = "string_1"; 37 | try { 38 | std::string x = hashtable[1]; 39 | std::cout << x << std::endl; 40 | } catch (const exception& e) { 41 | std::cout << e.what() << std::endl; 42 | } 43 | try { 44 | std::string x = hashtable.at(1); 45 | std::cout << x << std::endl; 46 | } catch (const exception& e) { 47 | std::cout << e.what() << std::endl; 48 | } 49 | hashtable.at(1) = "string_1_1"; 50 | try { 51 | std::string x = hashtable[1]; 52 | std::cout << x << std::endl; 53 | } catch (const exception& e) { 54 | std::cout << e.what() << std::endl; 55 | } 56 | try { 57 | std::string x = hashtable.at(1); 58 | std::cout << x << std::endl; 59 | } catch (const exception& e) { 60 | std::cout << e.what() << std::endl; 61 | } 62 | return EXIT_SUCCESS; 63 | } 64 | -------------------------------------------------------------------------------- /test/map_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include "settings.h" 7 | #include "map.h" 8 | 9 | int main() { 10 | Map m { "m" }; 11 | int iter = 1; 12 | for (int i = 0; i < iter; i++) { 13 | m[i] = i - 1; 14 | } 15 | for (int i = 0; i < iter; i++) { 16 | assert(m[i] == i - 1); 17 | } 18 | printf("int check succeeded!!\n"); 19 | 20 | Map sm("sm"); 21 | for (int i = 0; i < iter; i++) { 22 | sm[std::to_string(i)] = std::to_string(i + 1); 23 | } 24 | for (int i = 0; i < iter; i++) { 25 | assert(sm[std::to_string(i)] == std::to_string(i + 1)); 26 | } 27 | printf("string check succeeded!!\n"); 28 | 29 | try { 30 | sm.at(std::to_string(iter)); 31 | } catch (std::exception& e) { 32 | printf("key %d not exists (err = %s)\n", iter, e.what()); 33 | } 34 | 35 | size_t hv = m.lock(1); 36 | m[1] = 100; 37 | m.unlock_hv(hv); 38 | 39 | m.lock(1); 40 | assert(m[1] == 100); 41 | m.unlock(1); 42 | return 0; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /test/master_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include "structure.h" 4 | #include "worker.h" 5 | #include "settings.h" 6 | #include "worker_handle.h" 7 | #include "master.h" 8 | #include "gallocator.h" 9 | 10 | int main() { 11 | Conf* conf = new Conf(); 12 | GAllocFactory::SetConf(conf); 13 | Master* master = new Master(*conf); 14 | master->Join(); 15 | return 0; 16 | } 17 | -------------------------------------------------------------------------------- /test/slab_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include "slabs.h" 8 | #include "gallocator.h" 9 | #include "structure.h" 10 | #include 11 | using namespace std; 12 | 13 | int main() { 14 | const Conf* conf = GAllocFactory::InitConf(); 15 | SlabAllocator sb { }; 16 | void* base = sb.slabs_init(conf->size, conf->factor, true); 17 | int max = 1024 * 1024; 18 | int step = 5; 19 | char* buf = (char*) sb.sb_aligned_malloc(1); 20 | epicAssert((uint64_t)buf % BLOCK_SIZE == 0); 21 | 22 | size_t avail = sb.get_avail(); 23 | cout << "avail memory: " << sb.get_avail() << endl; 24 | 25 | /* 26 | * check the functionality of sb_allocator 27 | */ 28 | int i; 29 | for (i = 1; i < max; i += step) { 30 | if (sb.get_avail() != avail) { 31 | printf("error %dth time avail = %ld\n", i, sb.get_avail()); 32 | break; 33 | } 34 | void* buf = sb.sb_malloc(i); 35 | if (!buf) { 36 | cout << "malloc error for size = " << i << endl; 37 | break; 38 | } 39 | //memset(buf, 1, i); 40 | epicAssert(!sb.is_free(buf)); 41 | epicAssert(sb.get_size(buf) == i); 42 | sb.sb_free(buf); 43 | epicAssert(sb.is_free(buf)); 44 | } 45 | 46 | if (i == max) { 47 | cout << "First test: succeed!" << endl; 48 | } else { 49 | cout << "First test: failed" << endl; 50 | } 51 | 52 | /* 53 | * check that the behavior of sb_allocator will not touch the memory to be allocated! 54 | * Otherwise, it will generate segmentation fault 55 | */ 56 | mprotect(base, conf->size, PROT_NONE); 57 | for (i = 1; i < max; i += step) { 58 | if (sb.get_avail() != avail) { 59 | cout << i << endl; 60 | cout << sb.get_avail() << endl; 61 | break; 62 | } 63 | void* buf = sb.sb_malloc(i); 64 | if (!buf) { 65 | cout << "malloc error for size = " << i << endl; 66 | break; 67 | } 68 | sb.sb_free(buf); 69 | } 70 | if (i == max) { 71 | cout << "Second test: succeed!" << endl; 72 | } else { 73 | cout << "Second test: failed" << endl; 74 | } 75 | return 0; 76 | } 77 | 78 | -------------------------------------------------------------------------------- /test/worker_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The GAM Authors 2 | 3 | #include "structure.h" 4 | #include "worker.h" 5 | #include "settings.h" 6 | #include "worker_handle.h" 7 | #include "master.h" 8 | #include "gallocator.h" 9 | 10 | int main() { 11 | ibv_device **list = ibv_get_device_list(NULL); 12 | //worker1 13 | Conf* conf = new Conf(); 14 | GAllocFactory::SetConf(conf); 15 | RdmaResource* res = new RdmaResource(list[0], false); 16 | Worker* worker1 = new Worker(*conf, res); 17 | 18 | //worker2 19 | conf = new Conf(); 20 | res = new RdmaResource(list[0], false); 21 | conf->worker_port += 1; 22 | Worker* worker2 = new Worker(*conf, res); 23 | 24 | //worker3 25 | conf = new Conf(); 26 | res = new RdmaResource(list[0], false); 27 | conf->worker_port += 2; 28 | Worker* worker3 = new Worker(*conf, res); 29 | 30 | worker1->Join(); 31 | worker2->Join(); 32 | worker3->Join(); 33 | return 0; 34 | } 35 | --------------------------------------------------------------------------------