├── .gitmodules ├── CMakeLists.txt ├── README.md ├── cmake ├── Dependencies.cmake ├── rocc.cmake └── tpce.cmake ├── docs └── rnic.md ├── scripts ├── aws-config.py ├── awsrun.sh ├── cmd.py ├── config.xml ├── count_line.sh ├── hosts.xml ├── module_test.py ├── run-aws.py ├── run2.py ├── run_batch_aws.py ├── run_test3.py ├── run_util.py ├── runner.py ├── sync_to_aws.sh └── sync_to_server.sh ├── src ├── all.h ├── app │ ├── README │ ├── graph │ │ ├── base_distribution.hpp │ │ ├── grap_main.cc │ │ ├── graph.h │ │ ├── graph_constants.h │ │ ├── graph_scheme.h │ │ ├── graph_util.hpp │ │ └── real_distribution.hpp │ ├── micro │ │ ├── micro.cc │ │ ├── micro_rdma.cc │ │ ├── micro_rpc.cc │ │ └── micro_worker.h │ ├── smallbank │ │ ├── bank_main.cc │ │ ├── bank_schema.h │ │ ├── bank_worker.cc │ │ ├── bank_worker.h │ │ └── bank_worker_new.cc │ ├── tpcc │ │ ├── tpcc_cache.cc │ │ ├── tpcc_loader.cc │ │ ├── tpcc_main.cc │ │ ├── tpcc_mixin.h │ │ ├── tpcc_schema.h │ │ ├── tpcc_worker.cc │ │ ├── tpcc_worker.h │ │ └── tpcc_worker_new.cc │ └── tpce │ │ ├── egen │ │ ├── AddressTable.cpp │ │ ├── AddressTable.h │ │ ├── BaseLoader.h │ │ ├── BaseLoaderFactory.h │ │ ├── BaseLogFormatter.h │ │ ├── BaseLogger.cpp │ │ ├── BaseLogger.h │ │ ├── BigMath.h │ │ ├── Brokers.h │ │ ├── CE.cpp │ │ ├── CE.h │ │ ├── CESUTInterface.h │ │ ├── CETxnInputGenerator.cpp │ │ ├── CETxnInputGenerator.h │ │ ├── CETxnMixGenerator.cpp │ │ ├── CETxnMixGenerator.h │ │ ├── ChargeTable.h │ │ ├── CommissionRateTable.h │ │ ├── CompanyCompetitorFile.h │ │ ├── CompanyCompetitorTable.h │ │ ├── CompanyFile.h │ │ ├── CompanyTable.h │ │ ├── CustomerAccountsAndPermissionsTable.h │ │ ├── CustomerSelection.cpp │ │ ├── CustomerSelection.h │ │ ├── CustomerTable.cpp │ │ ├── CustomerTable.h │ │ ├── CustomerTaxratesTable.h │ │ ├── DM.cpp │ │ ├── DM.h │ │ ├── DMSUTInterface.h │ │ ├── DailyMarketTable.h │ │ ├── DateTime.cpp │ │ ├── DateTime.h │ │ ├── DriverParamSettings.h │ │ ├── DriverTypes.h │ │ ├── EGenBaseLoader_stdafx.h │ │ ├── EGenError.h │ │ ├── EGenGenerateAndLoad.cpp │ │ ├── EGenGenerateAndLoad.h │ │ ├── EGenGenerateAndLoadBaseOutput.h │ │ ├── EGenGenerateAndLoadStandardOutput.h │ │ ├── EGenGenerateAndLoad_stdafx.h │ │ ├── EGenLoader.cxx │ │ ├── EGenLoader_stdafx.h │ │ ├── EGenLogFormatterTab.cpp │ │ ├── EGenLogFormatterTab.h │ │ ├── EGenLogger.h │ │ ├── EGenNullLoader_stdafx.h │ │ ├── EGenStandardTypes.h │ │ ├── EGenTables_common.h │ │ ├── EGenTables_stdafx.h │ │ ├── EGenUtilities_stdafx.h │ │ ├── EGenValidate.cxx │ │ ├── EGenVersion.cpp │ │ ├── EGenVersion.h │ │ ├── ExchangeTable.h │ │ ├── FinalTransform.h │ │ ├── FinancialTable.h │ │ ├── FixedArray.h │ │ ├── FixedMap.h │ │ ├── FlatAccountPermissionLoad.h │ │ ├── FlatAddressLoad.h │ │ ├── FlatBrokerLoad.h │ │ ├── FlatCashTransactionLoad.h │ │ ├── FlatChargeLoad.h │ │ ├── FlatCommissionRateLoad.h │ │ ├── FlatCompanyCompetitorLoad.h │ │ ├── FlatCompanyLoad.h │ │ ├── FlatCustomerAccountLoad.h │ │ ├── FlatCustomerLoad.h │ │ ├── FlatCustomerTaxrateLoad.h │ │ ├── FlatDailyMarketLoad.h │ │ ├── FlatExchangeLoad.h │ │ ├── FlatFile.h │ │ ├── FlatFileLoad_common.h │ │ ├── FlatFileLoad_stdafx.h │ │ ├── FlatFileLoader.cpp │ │ ├── FlatFileLoader.h │ │ ├── FlatFinancialLoad.h │ │ ├── FlatHoldingHistoryLoad.h │ │ ├── FlatHoldingLoad.h │ │ ├── FlatHoldingSummaryLoad.h │ │ ├── FlatIndustryLoad.h │ │ ├── FlatLastTradeLoad.h │ │ ├── FlatLoaderFactory.h │ │ ├── FlatNewsItemLoad.h │ │ ├── FlatNewsXRefLoad.h │ │ ├── FlatSectorLoad.h │ │ ├── FlatSecurityLoad.h │ │ ├── FlatSettlementLoad.h │ │ ├── FlatStatusTypeLoad.h │ │ ├── FlatTaxrateLoad.h │ │ ├── FlatTradeHistoryLoad.h │ │ ├── FlatTradeLoad.h │ │ ├── FlatTradeRequestLoad.h │ │ ├── FlatTradeTypeLoad.h │ │ ├── FlatWatchItemLoad.h │ │ ├── FlatWatchListLoad.h │ │ ├── FlatZipCodeLoad.h │ │ ├── HoldingsAndTradesTable.h │ │ ├── IndustryTable.h │ │ ├── InputFile.h │ │ ├── InputFileNoWeight.h │ │ ├── InputFlatFilesDeclarations.h │ │ ├── InputFlatFilesStructure.cpp │ │ ├── InputFlatFilesStructure.h │ │ ├── LastTradeTable.h │ │ ├── MEE.cxx │ │ ├── MEE.h │ │ ├── MEEPriceBoard.cpp │ │ ├── MEEPriceBoard.h │ │ ├── MEESUTInterface.h │ │ ├── MEESecurity.cpp │ │ ├── MEESecurity.h │ │ ├── MEETickerTape.cpp │ │ ├── MEETickerTape.h │ │ ├── MEETradeRequestActions.h │ │ ├── MEETradingFloor.cpp │ │ ├── MEETradingFloor.h │ │ ├── MiscConsts.h │ │ ├── Money.cpp │ │ ├── Money.h │ │ ├── NewsItemAndXRefTable.h │ │ ├── NullLoad_stdafx.h │ │ ├── NullLoader.h │ │ ├── NullLoaderFactory.h │ │ ├── Person.cpp │ │ ├── Person.h │ │ ├── RNGSeeds.h │ │ ├── Random.cpp │ │ ├── Random.h │ │ ├── ReadRowFunctions.cpp │ │ ├── ReadRowFunctions_istream.cpp │ │ ├── ReadRowFunctions_sscanf.cpp │ │ ├── SectorTable.h │ │ ├── SecurityFile.h │ │ ├── SecurityPriceRange.h │ │ ├── SecurityTable.h │ │ ├── StatusTypeTable.h │ │ ├── SyncLockInterface.h │ │ ├── TableConsts.h │ │ ├── TableTemplate.h │ │ ├── Table_Defs.h │ │ ├── TaxrateTable.h │ │ ├── TimerWheel.h │ │ ├── TimerWheelTimer.h │ │ ├── TradeGen.cpp │ │ ├── TradeGen.h │ │ ├── TradeTypeIDs.h │ │ ├── TradeTypeTable.h │ │ ├── TxnHarnessBrokerVolume.h │ │ ├── TxnHarnessCustomerPosition.h │ │ ├── TxnHarnessDBInterface.h │ │ ├── TxnHarnessDataMaintenance.h │ │ ├── TxnHarnessMarketFeed.h │ │ ├── TxnHarnessMarketWatch.h │ │ ├── TxnHarnessSecurityDetail.h │ │ ├── TxnHarnessSendToMarketInterface.h │ │ ├── TxnHarnessStructs.h │ │ ├── TxnHarnessTradeCleanup.h │ │ ├── TxnHarnessTradeLookup.h │ │ ├── TxnHarnessTradeOrder.h │ │ ├── TxnHarnessTradeResult.h │ │ ├── TxnHarnessTradeStatus.h │ │ ├── TxnHarnessTradeUpdate.h │ │ ├── WatchListsAndItemsTable.h │ │ ├── Wheel.h │ │ ├── WheelTime.cpp │ │ ├── WheelTime.h │ │ ├── ZipCodeTable.h │ │ ├── bucketsimulator.cpp │ │ ├── bucketsimulator.h │ │ ├── error.cpp │ │ ├── error.h │ │ ├── flat │ │ │ └── egen_flat_in │ │ │ │ ├── AreaCode.txt │ │ │ │ ├── Charge.txt │ │ │ │ ├── CommissionRate.txt │ │ │ │ ├── Company.txt │ │ │ │ ├── CompanyCompetitor.txt │ │ │ │ ├── CompanySPRate.txt │ │ │ │ ├── Exchange.txt │ │ │ │ ├── FemaleFirstName.txt │ │ │ │ ├── Industry.txt │ │ │ │ ├── LastName.txt │ │ │ │ ├── MaleFirstName.txt │ │ │ │ ├── NonTaxableAccountName.txt │ │ │ │ ├── Sector.txt │ │ │ │ ├── Security.txt │ │ │ │ ├── StatusType.txt │ │ │ │ ├── StreetName.txt │ │ │ │ ├── StreetSuffix.txt │ │ │ │ ├── TaxRatesCountry.txt │ │ │ │ ├── TaxRatesDivision.txt │ │ │ │ ├── TaxableAccountName.txt │ │ │ │ ├── TradeType.txt │ │ │ │ └── ZipCode.txt │ │ ├── locking.cpp │ │ ├── locking.h │ │ ├── progressmeter.cpp │ │ ├── progressmeter.h │ │ ├── progressmeterinterface.cpp │ │ ├── progressmeterinterface.h │ │ ├── shore_tpce_egen.h │ │ ├── strutil.cpp │ │ ├── strutil.h │ │ ├── threading.cpp │ │ ├── threading.h │ │ ├── threading_pthread.h │ │ ├── threading_single.h │ │ ├── unusedflag.h │ │ └── user_time.h │ │ ├── tpce_constants.h │ │ ├── tpce_loader.cc │ │ ├── tpce_loader_factory.h │ │ ├── tpce_main.cc │ │ ├── tpce_mixin.h │ │ ├── tpce_schema.h │ │ ├── tpce_worker.cc │ │ ├── tpce_worker.h │ │ └── tpce_worker_new.cc ├── core │ ├── common.h │ ├── commun_queue.hpp │ ├── config.hpp │ ├── logging.cc │ ├── logging.h │ ├── rdma_sched.cc │ ├── rdma_sched.h │ ├── routine.cc │ ├── routine.h │ ├── rpc_buf_allocator.hpp │ ├── rrpc.cc │ ├── rrpc.h │ ├── rworker.cc │ ├── rworker.h │ ├── rworker_yield_impl.hpp │ ├── tcp_adapter.cc │ ├── tcp_adapter.hpp │ └── utils │ │ ├── amd64.h │ │ ├── count_vector.hpp │ │ ├── latency_profier.h │ │ ├── macros.h │ │ ├── ndb_type_traits.h │ │ ├── records_buffer.hpp │ │ ├── scopedperf.hh │ │ ├── serializer.h │ │ ├── small_vector.h │ │ ├── spinbarrier.h │ │ ├── thread.cc │ │ ├── thread.h │ │ ├── util.h │ │ ├── varint.cc │ │ └── varint.h ├── framework │ ├── bench_listener.h │ ├── bench_listener2.cc │ ├── bench_reporter.cc │ ├── bench_reporter.h │ ├── bench_runner.cc │ ├── bench_runner.h │ ├── bench_worker.cc │ ├── bench_worker.h │ ├── config.cc │ ├── config.h │ ├── main.cc │ ├── req_buf_allocator.h │ ├── tpc_msg.h │ ├── utils │ │ ├── abstract_db.h │ │ ├── amd64.h │ │ ├── encoder.h │ │ ├── inline_str.h │ │ ├── macros.h │ │ ├── ndb_type_traits.h │ │ ├── scopedperf.hh │ │ ├── serializer.h │ │ ├── small_vector.h │ │ ├── spinbarrier.h │ │ ├── thread.cc │ │ ├── thread.h │ │ ├── util.h │ │ ├── varint.cc │ │ └── varint.h │ ├── view_manager.cc │ └── view_manager.h ├── memstore │ ├── cluster_chaining.hpp │ ├── cluster_chaining_remote_op.hpp │ ├── memdb.cc │ ├── memdb.h │ ├── memstore.cc │ ├── memstore.h │ ├── memstore_bplustree.cc │ ├── memstore_bplustree.h │ ├── memstore_hash.cc │ ├── memstore_hash.h │ ├── memstore_uint64bplustree.cc │ ├── memstore_uint64bplustree.h │ ├── rdma_chainhash.h │ ├── rdma_cuckoohash.h │ ├── rdma_hash.hpp │ ├── rdma_hashext.h │ └── simple_cache.hpp ├── port │ ├── README │ ├── atomic-template.h │ ├── atomic.h │ ├── atomic_pointer.h │ ├── port.h │ ├── port_example.h │ ├── port_posix.cc │ ├── port_posix.h │ ├── thread_annotations.h │ └── win │ │ └── stdint.h ├── rocc_config.h.in ├── rtx │ ├── batch_op_impl.hpp │ ├── checker.hpp │ ├── default_log_cleaner_impl.hpp │ ├── global_vars.cc │ ├── global_vars.h │ ├── local_op_impl.hpp │ ├── log_cleaner.hpp │ ├── log_mem_manager.hpp │ ├── log_store_manager.hpp │ ├── logger.hpp │ ├── logger_test.cxx │ ├── msg_format.hpp │ ├── occ.cc │ ├── occ.h │ ├── occ_inline.hpp │ ├── occ_internal_structure.h │ ├── occ_iterator.hpp │ ├── occ_rdma.cc │ ├── occ_rdma.h │ ├── occ_statistics.h │ ├── occ_variants.hpp │ ├── op_test.cxx │ ├── qp_selection_helper.h │ ├── rdma_logger_impl.hpp │ ├── rdma_op_impl.hpp │ ├── rdma_req_helper.hpp │ ├── rpc_logger_impl.hpp │ ├── tx_operator.hpp │ ├── view.cc │ └── view.h ├── tx_config.h.in └── util │ ├── Makefile │ ├── mapped_log.cc │ ├── mapped_log.h │ ├── msg_buf_allocator.h │ ├── mutexlock.h │ ├── random.h │ ├── rdtsc_converter.cxx │ ├── rtm.cc │ ├── rtm.h │ ├── simple_logger.h │ ├── spinlock.h │ ├── temp_log.h │ ├── timer.h │ ├── txprofile.h │ ├── util.cc │ └── util.h └── third_party └── micautil ├── city.h ├── city_mod.cc ├── citycrc.h ├── citycrc_mod.h ├── hash.h ├── rte_memcpy.h ├── rte_memcpy_mod.h └── table ├── fixedtable.h ├── fixedtable_impl ├── bucket.h ├── del.h ├── get.h ├── info.h ├── init.h ├── item.h ├── lock.h ├── lock_bkt.h ├── lock_bkt_and_get.h ├── lock_bkt_for_ins.h ├── prefetch.h ├── set.h ├── set_spinlock.h └── unlock_bkt.h ├── ltable.h ├── ltable_impl ├── bucket.h ├── del.h ├── get.h ├── increment.h ├── info.h ├── init.h ├── item.h ├── lock.h ├── move_to_head.h ├── prefetch.h ├── set.h ├── specialization.h └── test.h ├── table.h └── types.h /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/rdma_lib"] 2 | path = third_party/rdma_lib 3 | url = http://ipads.se.sjtu.edu.cn:1312/Windybeing/rdma_lib.git 4 | [submodule "third_party/cpuinfo"] 5 | path = third_party/cpuinfo 6 | url = https://github.com/pytorch/cpuinfo.git 7 | [submodule "third_party/rlib"] 8 | path = third_party/rlib 9 | url = https://github.com/wxdwfc/rlib.git 10 | -------------------------------------------------------------------------------- /cmake/rocc.cmake: -------------------------------------------------------------------------------- 1 | function(load_global_config) 2 | configure_file("src/rocc_config.h.in" "src/rocc_config.h") 3 | endfunction() 4 | 5 | function(load_tx_config) 6 | 7 | if(TX_LOG_STYLE) 8 | else() 9 | set(TX_LOG_STYLE 1) ## default uses RPC as logging 10 | endif() 11 | 12 | if(PA) 13 | else() 14 | set(PA NULL) ## default uses RPC as logging 15 | endif() 16 | 17 | ## whether to record staleness counter for timestamp based methods 18 | if(RECORD_STALE) 19 | else() 20 | set(RECORD_STALE NULL) ## default uses RPC as logging 21 | endif() 22 | 23 | 24 | if(TX_BACKUP_STORE) 25 | else() 26 | set(TX_BACKUP_STORE 1) 27 | endif() 28 | 29 | 30 | if(ONE_SIDED_READ) 31 | else() 32 | set(ONE_SIDED_READ NULL) 33 | endif() 34 | 35 | if(USE_RDMA_COMMIT) 36 | else() 37 | set(USE_RDMA_COMMIT NULL) 38 | endif() 39 | 40 | if(RDMA_CACHE) 41 | else() 42 | set(RDMA_CACHE 0) 43 | endif() 44 | 45 | if(RDMA_STORE_SIZE) 46 | else() 47 | if(ONE_SIDED_READ) 48 | set(RDMA_STORE_SIZE 0) 49 | else() 50 | set(RDMA_STORE_SIZE 8) 51 | endif() 52 | endif() 53 | 54 | 55 | configure_file("src/tx_config.h.in" "src/tx_config.h") 56 | endfunction() -------------------------------------------------------------------------------- /cmake/tpce.cmake: -------------------------------------------------------------------------------- 1 | ## TPC-E related configurations 2 | ## current it's not well since not all file in *egen* is necessary 3 | set(EGEN_PREFIX src/app/tpce/egen/) 4 | file(GLOB TPCE_SOURCES 5 | "${EGEN_PREFIX}DateTime.cpp" "${EGEN_PREFIX}error.cpp" 6 | "${EGEN_PREFIX}Random.cpp" "${EGEN_PREFIX}Money.cpp" 7 | "${EGEN_PREFIX}EGenVersion.cpp" "${EGEN_PREFIX}locking.cpp" 8 | "${EGEN_PREFIX}threading.cpp" "${EGEN_PREFIX}BaseLogger.cpp" "${EGEN_PREFIX}EGenLogFormatterTab.cpp" 9 | "${EGEN_PREFIX}MEEPriceBoard.cpp" "${EGEN_PREFIX}MEESecurity.cpp" "${EGEN_PREFIX}MEETickerTape.cpp" 10 | "${EGEN_PREFIX}MEETradingFloor.cpp" "${EGEN_PREFIX}WheelTime.cpp" "${EGEN_PREFIX}AddressTable.cpp" 11 | "${EGEN_PREFIX}CustomerSelection.cpp" "${EGEN_PREFIX}CustomerTable.cpp" "${EGEN_PREFIX}InputFlatFilesStructure.cpp" 12 | "${EGEN_PREFIX}Person.cpp" "${EGEN_PREFIX}ReadRowFunctions.cpp" 13 | "${EGEN_PREFIX}TradeGen.cpp" "${EGEN_PREFIX}FlatFileLoader.cpp" "${EGEN_PREFIX}CE.cpp" "${EGEN_PREFIX}CETxnInputGenerator.cpp" 14 | "${EGEN_PREFIX}CETxnMixGenerator.cpp" "${EGEN_PREFIX}DM.cpp" "${EGEN_PREFIX}EGenGenerateAndLoad.cpp" "${EGEN_PREFIX}strutil.cpp" 15 | "${EGEN_PREFIX}progressmeter.cpp" "${EGEN_PREFIX}progressmeterinterface.cpp" "${EGEN_PREFIX}bucketsimulator.cpp") 16 | -------------------------------------------------------------------------------- /docs/rnic.md: -------------------------------------------------------------------------------- 1 | Since this project uses new advanced hardware features like RDMA, we briefly describe the hardware setting which is suitable here. 2 | 3 | ## RDMA-enabled network card 4 | 5 | **ROCC** can run most Mellanox RNICs (We say most because we have not tested on all of them yet :) ). It has been tested on `ConnctX-3`, `ConnectX-4`,`ConnectX-5`, 6 | both VPI and ROCE. 7 | 8 | Yet, we strongly suggest not to use `ConnectX-3 RNIC`, because its one-sided primitive requires 9 | *very very careful setting and tuning* to achieve a reasonable performance on **ROCC**. 10 | Later generations of RNICs (Like `ConnectX-4`) works well on **ROCC**. 11 | This is because `ConnectX-3 RNIC` has limited hardware resources such as the cache on the NIC. 12 | 13 | 14 | ## CPU 15 | 16 | **DrTM+H** requires Intel's restricted transactional memory~(RTM) for its concurrently data stores. 17 | It can call illegal instructions fault on CPU without this support on some workloads. Yet, **ROCC** does not depends on RTM. 18 | -------------------------------------------------------------------------------- /scripts/awsrun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | aws ec2 run-instances \ 4 | --image-id "ami-41695a24" \ 5 | --instance-type "r4.2xlarge" \ 6 | --key-name "tp" \ 7 | --monitoring '{"Enabled":false}' \ 8 | --security-group-ids "sg-2131064a" \ 9 | --instance-initiated-shutdown-behavior "stop" \ 10 | --subnet-id "subnet-fec85696" \ 11 | --count 5 12 | -------------------------------------------------------------------------------- /scripts/cmd.py: -------------------------------------------------------------------------------- 1 | ### 2 | ## This file provides wrapper to help execute command at remote servers 3 | ### 4 | 5 | from subprocess import Popen, PIPE 6 | import subprocess # execute commands 7 | import os # change dir 8 | 9 | import paramiko 10 | import getpass 11 | 12 | FNULL = open(os.devnull, 'w') 13 | 14 | class ConnectProxy: 15 | def __init__(self,mac,user=""): 16 | if user == "": 17 | user = getpass.getuser() 18 | self.ssh = paramiko.SSHClient() 19 | 20 | self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 21 | self.user = user 22 | self.mac = mac 23 | self.sftp = None 24 | 25 | def connect(self,pwd): 26 | self.ssh.connect(hostname = self.mac,username = self.user, password = pwd) 27 | 28 | def execute(self,cmd,pty=False): 29 | return self.ssh.exec_command(cmd,get_pty=pty) 30 | 31 | def copy_file(self,f,dst_dir = ""): 32 | if self.sftp == None: 33 | self.sftp = paramiko.SFTPClient.from_transport(self.ssh.get_transport()) 34 | self.sftp.put(f, dst_dir + f) 35 | 36 | def get_file(self,remote_path,f): 37 | if self.sftp == None: 38 | self.sftp = paramiko.SFTPClient.from_transport(self.ssh.get_transport()) 39 | self.sftp.get(remote_path + "/" + f,f) 40 | 41 | def close(self): 42 | if self.sftp != None: 43 | self.sftp.close() 44 | self.ssh.close() 45 | 46 | def execute_at(mac,cmd): 47 | subprocess.call(["ssh", "-n","-f", mac, cmd]) 48 | 49 | def slient_execute_at(mac,cmd): 50 | subprocess.call(["ssh", "-n","-f", mac, cmd],stdout=FNULL,stderr=subprocess.STDOUT) 51 | 52 | def get_w_execute(mac,cmd,timeout = 2): 53 | stdout, stderr = Popen(['ssh',"-o","ConnectTimeout=" + int(timeout),mac,cmd], 54 | stdout=PIPE).communicate() 55 | return stdout,stderr 56 | 57 | def get_w_execute_aws(mac,cmd,timeout = 2,pem = "../aws/tp.pem"): 58 | stdout, stderr = Popen(['ssh',"-i",pem,"-o","ConnectTimeout=" + str(timeout),mac,cmd], 59 | stdout=PIPE).communicate() 60 | return stdout,stderr 61 | 62 | def execute_at_aws(mac,cmd,pem = "../aws/tp.pem"): 63 | ssh_base = ["ssh","-i",pem,"-o","StrictHostKeyChecking=no","-n","-f"] 64 | subprocess.call(ssh_base + [mac,cmd]) 65 | 66 | def copy_to(f,mac,dst="~"): 67 | subprocess.call(["scp",f,"%s:%s" % (mac,dst)],stdout=FNULL, stderr=subprocess.STDOUT) 68 | 69 | def copy_to_aws(f,mac,dst = "~",pem = "../aws/tp.pem"): 70 | subprocess.call(["scp","-i",pem,f,"%s:%s" % (mac,dst)],stdout=FNULL, stderr=subprocess.STDOUT) 71 | -------------------------------------------------------------------------------- /scripts/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 1 3 | 0 4 | 2 5 | 6 | 25 7 | 15 8 | 15 9 | 15 10 | 15 11 | 15 12 | 13 | 14 | 15 | 100 16 | 0 17 | 0 18 | 0 19 | 0 20 | 21 | 22 | -------------------------------------------------------------------------------- /scripts/count_line.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find "../src" -path '../' \ 3 | -prune -o -path '../src/app/tpce/egen' \ 4 | -prune -o -path '../src/port' \ 5 | -prune -o -path '../src/tests' \ 6 | -prune -o -path '../src/app' \ 7 | -prune -o -print \ 8 | | egrep '\.php|\.as|\.sql|\.css|\.js|\.cc|\.h|.cpp|.c ' \ 9 | | grep -v '\.svn' | xargs cat | sed '/^\s*$/d' \ 10 | | wc -l 11 | -------------------------------------------------------------------------------- /scripts/hosts.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | val01 5 | val02 6 | val03 7 | val04 8 | val05 9 | val06 10 | val08 11 | val09 12 | val10 13 | val11 14 | val12 15 | val13 16 | val14 17 | val15 18 | val07 19 | val00 20 | 21 | 22 | val02 23 | 24 | 25 | 123 26 | 27 | -------------------------------------------------------------------------------- /scripts/module_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | from run_test3 import calculate_results 5 | import sys 6 | 7 | 8 | if __name__ == "__main__": 9 | assert(len(sys.argv) > 1) 10 | log_file_name = sys.argv[1] 11 | 12 | print(calculate_results(log_file_name)) 13 | -------------------------------------------------------------------------------- /scripts/run_util.py: -------------------------------------------------------------------------------- 1 | ## helper functions for various script 2 | 3 | import os # for chdir 4 | 5 | ## print with color helper class 6 | class bcolors: 7 | HEADER = '\033[5;%s;2m' 8 | OKBLUE = '34' 9 | OKRED = '31' 10 | OKGREEN = '\033[92m' 11 | WARNING = '\033[93m' 12 | FAIL = '\033[91m' 13 | ENDC = '\033[0m' 14 | BOLD = '\033[1m' 15 | UNDERLINE = '\033[4m' 16 | 17 | @staticmethod 18 | def wrap_text_with_color(txt,c): 19 | return ("%s%s%s" % ((bcolors.HEADER % c),txt,bcolors.ENDC)) 20 | 21 | def print_with_tag(tag,s): 22 | tag = bcolors.wrap_text_with_color(tag,bcolors.OKRED) 23 | print "[%s] %s" % (tag,s) 24 | 25 | 26 | class PrintTee(object): 27 | def __init__(self,*files): 28 | self.files = files 29 | 30 | def write(self, obj): 31 | for f in self.files: 32 | f.write(obj) 33 | 34 | ## end class bcolors 35 | 36 | def change_to_parent(): 37 | os.chdir("..") 38 | return 39 | -------------------------------------------------------------------------------- /scripts/sync_to_aws.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | target="$1" 4 | ## this script will sync the project to the remote server 5 | #rsync -i -rtuv $PWD/../ $target:~/nocc --exclude ./pre-data/ 6 | rsync -e "ssh -i ../aws/tp.pem" -rtuv $PWD/../ ubuntu@$target:~/nocc --exclude ./pre-data/ --exclude .git 7 | -------------------------------------------------------------------------------- /scripts/sync_to_server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | target="$1" 4 | ## this script will sync the project to the remote server 5 | rsync -i -rtuv $PWD/../ $target:~/projects/nocc --exclude ./pre-data/ 6 | #rsync -e "ssh -i ../aws/tp.pem" -rtuv $PWD/../ $target:~/nocc --exclude ./pre-data/ 7 | -------------------------------------------------------------------------------- /src/all.h: -------------------------------------------------------------------------------- 1 | /* This file contains constants used in the framework. ************************/ 2 | 3 | #ifndef _ALL 4 | #define _ALL 5 | 6 | /* Coroutine related staff */ 7 | /* Using boost coroutine */ 8 | #include 9 | 10 | typedef boost::coroutines::symmetric_coroutine::call_type coroutine_func_t; 11 | typedef boost::coroutines::symmetric_coroutine::yield_type yield_func_t; 12 | 13 | namespace nocc { 14 | 15 | /***** hardware parameters ********/ 16 | #define CACHE_LINE_SZ 64 // cacheline size of x86 platform 17 | //#define MAX_MSG_SIZE 25600 // max msg size used 18 | #define MAX_MSG_SIZE 4096 19 | //#define MAX_MSG_SIZE 512 20 | 21 | #define HUGE_PAGE_SZ (2 * 1024 * 1024) // huge page size supported 22 | 23 | #define MAX_SERVERS 32 // maxium number of machines in the cluster 24 | #define MAX_SERVER_TO_SENT MAX_SERVERS // maxium broadcast number of one msg 25 | 26 | #define NOCC_BENCH_MAX_TX 16 27 | 28 | /**********************************/ 29 | 30 | 31 | /* some usefull macros ******************************************************/ 32 | #define unlikely(x) __builtin_expect(!!(x), 0) 33 | #define likely(x) __builtin_expect(!!(x), 1) 34 | 35 | #define NOCC_NOT_IMPLEMENT(fnc) { fprintf(stderr,"[NOCC] %s function not implemented.\n",fnc);assert(false); } 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/app/README: -------------------------------------------------------------------------------- 1 | This file mainly contains our benchmark code. 2 | TPCC's main TX's implementations are in tpcc/tpcc_worker_new.cc, while Smallbank's implementations are in smallbank/bank_worker_new.cc . -------------------------------------------------------------------------------- /src/app/graph/grap_main.cc: -------------------------------------------------------------------------------- 1 | #include "graph_constants.h" 2 | #include "graph.h" 3 | #include "graph_util.hpp" 4 | #include "real_distribution.hpp" 5 | 6 | #include "framework/bench_runner.h" 7 | 8 | #include "util/util.h" 9 | 10 | // for log normal distribution 11 | #include 12 | #include 13 | 14 | using namespace nocc::util; 15 | 16 | namespace nocc { 17 | namespace oltp { 18 | namespace link { // link benchmark 19 | 20 | class GraphLoader : public BenchLoader { 21 | MemDB *store_; 22 | uint64_t start_id_; uint64_t end_id_; 23 | double medium_data_size_; 24 | public: 25 | GraphLoader(uint64_t start_id,uint64_t end_id,double mdz, 26 | uint64_t seed,int partition,MemDB *store,bool is_primary) 27 | : start_id_(start_id), end_id_(end_id), medium_data_size_(mdz), 28 | BenchLoader(seed) 29 | { 30 | }; 31 | virtual void load(); 32 | }; 33 | 34 | class GraphRunner : public BenchRunner { 35 | public: 36 | GraphRunner(std::string &config_file) { 37 | 38 | } 39 | 40 | virtual std::vector make_loaders(int partition, MemDB* store = NULL); 41 | virtual std::vector make_workers(); 42 | virtual void init_store(MemDB* &store) { } 43 | virtual void init_backup_store(MemDB* &store) { } 44 | virtual void populate_cache() { } 45 | 46 | virtual void bootstrap_with_rdma(RdmaCtrl *r) { 47 | } 48 | 49 | virtual void warmup_buffer(char *buffer) { 50 | } 51 | }; 52 | 53 | void GraphTest(int argc,char **argv) { 54 | auto test = RealDistribution(); 55 | df_t res; 56 | 57 | std::ifstream ifs("../data/Distribution.dat", std::ifstream::in); 58 | auto title = test.get_cdf(res,ifs); 59 | pdf_t pdf; 60 | test.get_pdf(res,pdf); 61 | for(uint i = 0;i < 12;++i) { 62 | 63 | } 64 | //test.get_cdf(res,ifs); 65 | 66 | return; 67 | } 68 | 69 | std::vector GraphRunner::make_loaders(int partition, MemDB* store) { 70 | std::vector res; 71 | return res; 72 | } 73 | 74 | std::vector GraphRunner::make_workers() { 75 | std::vector res; 76 | return res; 77 | } 78 | 79 | void GraphLoader::load() { 80 | 81 | } 82 | 83 | 84 | }; // end namespace link 85 | }; // end namespace oltp 86 | }; // end namespace nocc 87 | -------------------------------------------------------------------------------- /src/app/graph/graph.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_APP_GRAPH_ 2 | #define NOCC_APP_GRAPH_ 3 | 4 | #include "all.h" 5 | #include "tx_config.h" 6 | 7 | namespace nocc { 8 | namespace oltp { 9 | namespace link { 10 | 11 | void GraphTest(int argc,char **argv); // main hook function 12 | 13 | }; // end namespace link 14 | }; // end namespace oltp 15 | }; 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /src/app/graph/graph_constants.h: -------------------------------------------------------------------------------- 1 | namespace nocc { 2 | 3 | namespace oltp { 4 | namespace link { 5 | int DEFAULT_LINK_TYPE = 123456789; 6 | int DEFAULT_NODE_TYPE = 2048; 7 | bool VISIBILITY_HIDDEN = 0; 8 | bool VISIBILITY_DEFAULT = 1; 9 | int DEFAULT_LIMIT = 10000; 10 | int MAX_LINK_DATA = 255; 11 | int DEFAULT_LIMI = 10000; 12 | 13 | double LINK_DATASIZE_SIGMA = 1.0; 14 | double NODE_DATASIZE_SIGMA = 1.0; 15 | }; 16 | }; 17 | }; 18 | -------------------------------------------------------------------------------- /src/app/graph/graph_scheme.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_APP_GRAPH_SCHEME_ 2 | #define NOCC_APP_GRAPH_SCHEME_ 3 | 4 | #include "all.h" 5 | 6 | namespace nocc { 7 | namespace oltp { 8 | namespace link { 9 | 10 | struct Node { 11 | uint64_t id; 12 | int type; 13 | uint64_t version; 14 | uint64_t time; 15 | char data[CACHE_LINE_SZ]; 16 | }; 17 | 18 | class Edge { 19 | int type; 20 | bool visiblilty; 21 | uint64_t version; 22 | char data[CACHE_LINE_SZ]; 23 | }; 24 | 25 | inline ALWAYS_INLINE 26 | uint64_t makeEdgeKey(uint64_t id0,uint64_t id1,uint64_t time) { 27 | uint64_t *sec = new uint64_t[3]; 28 | sec[0] = id0; sec[1] = id1;sec[2] = time; 29 | return (uint64_t )sec; 30 | } 31 | 32 | } // end namespace link 33 | } 34 | } 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /src/app/graph/graph_util.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // for std random generators 4 | #include 5 | #include 6 | #include // for logs 7 | 8 | namespace nocc { 9 | 10 | namespace oltp { 11 | 12 | namespace link { 13 | 14 | class LogNormalDistWrapper { 15 | 16 | std::mt19937 gen_; 17 | std::lognormal_distribution<> rng_; 18 | int min_; 19 | int max_; 20 | double mu_; 21 | double sigma_; 22 | 23 | public: 24 | LogNormalDistWrapper(int min,int max,int medium, double sigma,uint64_t seed) 25 | :gen_(seed), 26 | rng_(), 27 | min_(min), 28 | max_(max), 29 | sigma_(sigma), 30 | mu_(std::log(medium)) 31 | { 32 | } 33 | 34 | int choose() { 35 | double next_guassian = rng_(gen_); 36 | assert(0 <= next_guassian && next_guassian <= 1); 37 | double content = next_guassian * sigma_ + mu_; 38 | int choice = (int)(std::round(std::exp(content))); 39 | if(choice < min_) return min_; 40 | if(choice > max_) return max_ - 1; 41 | return choice; 42 | } 43 | }; // end class log normal wrapper 44 | }; 45 | }; 46 | 47 | }; 48 | -------------------------------------------------------------------------------- /src/app/graph/real_distribution.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "util/util.h" 6 | using namespace nocc::util; 7 | 8 | #include "base_distribution.hpp" 9 | 10 | namespace nocc { 11 | 12 | namespace oltp { 13 | 14 | namespace link { 15 | 16 | class RealDistribution : public BaseDistribution { 17 | // constants 18 | static const uint64_t NLINKS_SHUFFLER_SEED = 20343988438726021L; 19 | static const int NLINKS_SHUFFLER_GROUPS = 1024; 20 | static const uint64_t UNCORR_SHUFFLER_SEED = 53238253823453L; 21 | static const int UNCORR_SHUFFLER_GROUPS = 1024; 22 | static const uint64_t WRITE_CORR_SHUFFLER_SEED = NLINKS_SHUFFLER_SEED; 23 | static const int WRITE_CORR_SHUFFLER_GROUPS = NLINKS_SHUFFLER_GROUPS; 24 | 25 | static const uint64_t WRITE_UNCORR_SHUFFLER_SEED = UNCORR_SHUFFLER_SEED; 26 | static const int WRITE_UNCORR_SHUFFLER_GROUPS = UNCORR_SHUFFLER_GROUPS; 27 | 28 | static const uint64_t READ_UNCORR_SHUFFLER_SEED = UNCORR_SHUFFLER_SEED; 29 | static const int READ_UNCORR_SHUFFLER_GROUPS = UNCORR_SHUFFLER_GROUPS; 30 | 31 | static const uint64_t NODE_READ_SHUFFLER_SEED = 4766565305853767165L; 32 | static const int NODE_READ_SHUFFLER_GROUPS = 1024; 33 | static const uint64_t NODE_UPDATE_SHUFFLER_SEED = NODE_READ_SHUFFLER_SEED; 34 | static const int NODE_UPDATE_SHUFFLER_GROUPS = NODE_READ_SHUFFLER_GROUPS; 35 | 36 | static const uint64_t NODE_DELETE_SHUFFLER_SEED = NODE_READ_SHUFFLER_SEED; 37 | static const int NODE_DELETE_SHUFFLER_GROUPS = NODE_READ_SHUFFLER_GROUPS; 38 | 39 | enum TYPE { // types of distributions for different operation 40 | LINKS, // distribution for links 41 | LINKS_READ, 42 | LINKS_WRITE, 43 | NODE_READ, 44 | NODE_WRITE 45 | }; 46 | 47 | void init_link() { 48 | 49 | } 50 | 51 | uint64_t getNLinks(uint64_t id1,uint64_t start_id,uint64_t end_id) { 52 | return (uint64_t) expected_count(start_id,end_id,id1,nlinks_cdf_); 53 | } 54 | 55 | private: 56 | df_t nlinks_cdf_; 57 | }; // end class real 58 | 59 | }; // end class link 60 | }; 61 | 62 | }; 63 | -------------------------------------------------------------------------------- /src/app/micro/micro_rpc.cc: -------------------------------------------------------------------------------- 1 | #include "tx_config.h" 2 | #include "micro_worker.h" 3 | 4 | extern size_t distributed_ratio; // used for some app defined parameters 5 | extern size_t total_partition; 6 | 7 | namespace nocc { 8 | namespace oltp { 9 | 10 | extern __thread util::fast_random *random_generator; 11 | 12 | namespace micro { 13 | 14 | struct Req { 15 | uint8_t size = 0; 16 | }; 17 | 18 | // RPC IDs 19 | enum { 20 | REQ_ID = 0 21 | }; 22 | 23 | extern uint64_t working_space; 24 | txn_result_t MicroWorker::micro_rpc(yield_func_t &yield) { 25 | 26 | static const int num_nodes = total_partition; 27 | 28 | auto size = distributed_ratio; 29 | assert(size > 0 && size <= MAX_MSG_SIZE); 30 | 31 | int window_size = 10; 32 | ASSERT(window_size < 64) << "window size shall be smaller than 64"; 33 | static __thread char *req_buf = rpc_->get_static_buf(4096); 34 | static __thread char *reply_buf = (char *)malloc(1024); 35 | 36 | char *req_ptr = req_buf; 37 | ASSERT(size <= 4096); 38 | 39 | #if 1 40 | #if !PA 41 | rpc_->prepare_multi_req(reply_buf, window_size,cor_id_); 42 | #endif 43 | 44 | for (uint i = 0; i < window_size; ++i) { 45 | 46 | int pid = random_generator[cor_id_].next() % num_nodes; 47 | 48 | // prepare an RPC header 49 | Req *req = (Req *)(req_ptr); 50 | req->size = size; 51 | #if 1 52 | rpc_->append_pending_req((char *)req,REQ_ID,sizeof(Req),cor_id_,RRpc::REQ,pid); 53 | #else 54 | rpc_->append_req((char *)req_buf,REQ_ID,sizeof(Req),cor_id_,RRpc::REQ,pid); 55 | #endif 56 | } 57 | rpc_->flush_pending(); 58 | #endif 59 | 60 | #if !PA 61 | indirect_yield(yield); 62 | #endif 63 | ntxn_commits_ += (window_size - 1); 64 | 65 | return txn_result_t(true,1); 66 | } 67 | 68 | /** 69 | * RPC handlers 70 | */ 71 | 72 | void MicroWorker::register_callbacks() { 73 | ROCC_BIND_STUB(rpc_,&MicroWorker::nop_rpc_handler,this,REQ_ID); 74 | } 75 | 76 | void MicroWorker::nop_rpc_handler(int id,int cid,char *msg, void *arg) { 77 | char *reply_msg = rpc_->get_reply_buf(); 78 | Req *req = (Req *)msg; 79 | ASSERT(req->size <= ::rdmaio::UDRecvManager::MAX_PACKET_SIZE) 80 | << "req size "<< (int)(req->size) << " " << ::rdmaio::UDRecvManager::MAX_PACKET_SIZE; 81 | rpc_->send_reply(reply_msg,req->size,id,worker_id_,cid); // a dummy notification 82 | } 83 | 84 | } // end micro 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/app/micro/micro_worker.h: -------------------------------------------------------------------------------- 1 | #ifndef APP_MICRO 2 | #define APP_MICRO 3 | 4 | #include "tx_config.h" 5 | #include "framework/bench_worker.h" 6 | 7 | namespace nocc { 8 | namespace oltp { 9 | 10 | namespace micro { 11 | 12 | enum MICRO_TYPE { 13 | RPC = 0, 14 | RDMA_READ = 1, 15 | RDMA_WRITE = 15, 16 | RDMA_CAS = 3 17 | }; 18 | 19 | void MicroTest(int argc,char **argv); // main hook function 20 | 21 | class MicroWorker : public BenchWorker { 22 | public: 23 | MicroWorker(unsigned int worker_id,unsigned long seed,int micro_type,MemDB *store, 24 | uint64_t total_ops, spin_barrier *a,spin_barrier *b,BenchRunner *c); 25 | 26 | void register_callbacks(); 27 | void thread_local_init(); 28 | 29 | workload_desc_vec_t get_workload() const ; 30 | 31 | void workload_report() { 32 | REPORT(post); 33 | } 34 | 35 | /** 36 | * real worker function body 37 | */ 38 | txn_result_t micro_rpc(yield_func_t &yield); 39 | txn_result_t micro_rdma_read(yield_func_t &yield); 40 | txn_result_t micro_rdma_write(yield_func_t &yield); 41 | txn_result_t micro_rdma_atomic(yield_func_t &yield); 42 | // 43 | 44 | /** 45 | * RPC handlers 46 | */ 47 | void nop_rpc_handler(int id,int cid,char *msg,void *arg); 48 | // 49 | 50 | private: 51 | static workload_desc_vec_t _get_workload(); 52 | 53 | static txn_result_t MicroRpc(BenchWorker *w,yield_func_t &yield) { 54 | return static_cast(w)->micro_rpc(yield); 55 | } 56 | 57 | static txn_result_t MicroRdmaRead(BenchWorker *w,yield_func_t &yield) { 58 | return static_cast(w)->micro_rdma_read(yield); 59 | } 60 | 61 | static txn_result_t MicroRdmaWrite(BenchWorker *w,yield_func_t &yield) { 62 | return static_cast(w)->micro_rdma_write(yield); 63 | } 64 | 65 | static txn_result_t MicroRdmaAtomic(BenchWorker *w,yield_func_t &yield) { 66 | return static_cast(w)->micro_rdma_atomic(yield); 67 | } 68 | 69 | std::vector qp_vec_; 70 | 71 | // some performance counters 72 | LAT_VARS(post); 73 | }; // end class micro rpc 74 | 75 | } 76 | 77 | } // 78 | } 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /src/app/smallbank/bank_schema.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_FRAMEWORK_BANK_SCHEMA_H 2 | #define NOCC_FRAMEWORK_BANK_SCHEMA_H 3 | 4 | #include "framework/utils/encoder.h" 5 | #include "framework/utils/inline_str.h" 6 | #include "framework/utils/macros.h" 7 | 8 | #define ACCT 0 9 | #define SAV 1 10 | #define CHECK 2 11 | 12 | #define MIN_BALANCE 10000 13 | #define MAX_BALANCE 50000 14 | 15 | 16 | //#define BANK_NUM_ACCOUNTS 100000 17 | //define BANK_NUM_ACCOUNTS 18 | // 25% perct of the accounts are hot 19 | //#define BANK_NUM_HOT 4000 20 | 21 | //#ifdef BANK_NORMAL 22 | 23 | //#define BANK_NUM_HOT 0 24 | //#define BANK_NUM_HOT 25000 25 | //#define BANK_HOT_PECT 0 26 | //#else 27 | 28 | //#define BANK_NUM_HOT 2000 29 | //#define BANK_HOT_PECT 2 30 | //#define BANK_NUM_HOT 1000 31 | //#define BANK_HOT_PECT 1 32 | //#endif 33 | 34 | #define BANK_MIN_BALANCE 10000 35 | #define BANK_MAX_BALANCE 50000 36 | 37 | //#ifdef BANK_NORMAL 38 | //#define BANK_TX_HOT 0 39 | //#else 40 | // This number is defined in the original small bank paper 41 | //#define BANK_TX_HOT 90 42 | //#endif 43 | //#define BANK_TX_HOT 0 44 | 45 | 46 | /* table accounts */ 47 | 48 | #define ACCOUNTS_KEY_FIELDS(x,y) \ 49 | x(uint64_t,a_custid) 50 | 51 | // 52 | #define ACCOUNTS_VALUE_FIELDS(x,y)\ 53 | x(inline_str_16<64>,a_name) 54 | DO_STRUCT(account,ACCOUNTS_KEY_FIELDS,ACCOUNTS_VALUE_FIELDS) 55 | 56 | /* ------- */ 57 | 58 | /* table savings */ 59 | 60 | #define SAVINGS_KEY_FIELDS(x,y) \ 61 | x(uint64_t,s_cusitid) 62 | 63 | #define SAVINGS_VALUE_FIELDS(x,y) \ 64 | x(float,s_balance) 65 | DO_STRUCT(savings,SAVINGS_KEY_FIELDS,SAVINGS_VALUE_FIELDS) 66 | 67 | /* ------- */ 68 | 69 | 70 | /* table checking */ 71 | 72 | #define CHECKING_KEY_FIELDS(x,y) \ 73 | x(uint64_t ,c_custid) 74 | 75 | #define CHECKING_VALUE_FIELDS(x,y) \ 76 | x(float,c_balance) 77 | DO_STRUCT(checking,CHECKING_KEY_FIELDS,CHECKING_VALUE_FIELDS) 78 | 79 | /* ------- */ 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /src/app/tpcc/tpcc_cache.cc: -------------------------------------------------------------------------------- 1 | #include "tpcc_worker.h" 2 | #include "tpcc_schema.h" 3 | #include "tpcc_mixin.h" 4 | 5 | #include "memstore/memdb.h" 6 | 7 | #include "core/logging.h" 8 | 9 | using namespace rdmaio; 10 | 11 | 12 | extern size_t current_partition; 13 | extern size_t nthreads; 14 | 15 | namespace nocc { 16 | 17 | extern RdmaCtrl *cm; // global RDMA handler 18 | 19 | namespace oltp { 20 | namespace tpcc { 21 | 22 | void populate_ware(MemDB *db) { 23 | char *temp = (char *)Rmalloc(256); 24 | for(uint wid = 1; wid <= NumWarehouses();++wid) { 25 | int pid; 26 | if( (pid = WarehouseToPartition(wid)) != current_partition) { 27 | // fetch it 28 | auto off = db->stores_[WARE]->RemoteTraverse(wid,cm->get_rc_qp(create_rc_idx(pid,nthreads + nthreads + 1)),temp); 29 | assert(off != 0); 30 | 31 | } // end fetch 32 | PrintProgress((double)wid / NumWarehouses()); 33 | } // end iterating all warehouses 34 | Rfree(temp); 35 | } 36 | 37 | void populate_dist(MemDB *db) { 38 | int pid; 39 | char *temp = (char *)Rmalloc(256); 40 | for(uint wid = 1;wid <= NumWarehouses();++wid) { 41 | for(uint d = 1; d <= NumDistrictsPerWarehouse();++d) { 42 | if( (pid = WarehouseToPartition(wid)) == current_partition) 43 | continue; 44 | uint64_t key = makeDistrictKey(wid,d); 45 | auto off = db->stores_[DIST]->RemoteTraverse(key,cm->get_rc_qp(create_rc_idx(pid,nthreads + nthreads + 1)),temp); 46 | assert(off != 0); 47 | } // iterating all districts 48 | } 49 | Rfree(temp); 50 | } 51 | 52 | void populate_stock(MemDB *db) { 53 | 54 | char *temp = (char *)Rmalloc(256); 55 | int pid; 56 | for(uint wid = 1;wid <= NumWarehouses();++wid) { 57 | 58 | const size_t batchsize = NumItems() ; 59 | const size_t nbatches = (batchsize > NumItems()) ? 1 : (NumItems() / batchsize); 60 | 61 | for (uint b = 0; b < nbatches;) { 62 | const size_t iend = std::min((b + 1) * batchsize + 1, NumItems()); 63 | for (uint i = (b * batchsize + 1); i <= iend; i++) { 64 | 65 | uint64_t key = makeStockKey(wid,i); 66 | if((pid = WarehouseToPartition(wid)) == current_partition) 67 | continue; 68 | auto off = db->stores_[STOC]->RemoteTraverse(key,cm->get_rc_qp(create_rc_idx(pid,nthreads + nthreads + 1)),temp); 69 | assert(off != 0); 70 | } 71 | b++; 72 | } // end iterating all stocks 73 | PrintProgress((double)wid / NumWarehouses()); 74 | } 75 | Rfree(temp); 76 | LOG(2) << "total " << (NumWarehouses() * NumItems()) << " cache entries loaded."; 77 | } 78 | 79 | }; // namespace tpcc 80 | }; // namespace oltp 81 | 82 | }; // namespace nocc 83 | -------------------------------------------------------------------------------- /src/app/tpce/egen/DriverTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Matt Emmerton 35 | */ 36 | 37 | /****************************************************************************** 38 | * Description: Driver types and names 39 | ******************************************************************************/ 40 | 41 | #ifndef DRIVER_TYPES_H 42 | #define DRIVER_TYPES_H 43 | 44 | namespace TPCE 45 | { 46 | 47 | enum eDriverType 48 | { 49 | eDriverEGenLoader, 50 | eDriverAll, 51 | eDriverCE, 52 | eDriverMEE, 53 | eDriverDM, 54 | eDriverMax 55 | }; 56 | 57 | extern char szDriverTypeNames[eDriverMax][14]; 58 | } // namespace TPCE 59 | 60 | 61 | #endif // DRIVER_TYPES_H 62 | -------------------------------------------------------------------------------- /src/app/tpce/egen/EGenBaseLoader_stdafx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #ifndef EGEN_BASELOADER_STDAFX_H 38 | #define EGEN_BASELOADER_STDAFX_H 39 | 40 | #include "BaseLoader.h" 41 | #include "BaseLoaderFactory.h" 42 | 43 | #endif // #ifndef EGEN_BASELOADER_STDAFX_H 44 | -------------------------------------------------------------------------------- /src/app/tpce/egen/EGenGenerateAndLoad_stdafx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #ifndef EGEN_GENERATE_AND_LOAD_STDAFX_H 38 | #define EGEN_GENERATE_AND_LOAD_STDAFX_H 39 | 40 | #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | 51 | using namespace std; 52 | 53 | #include "EGenTables_stdafx.h" 54 | #include "EGenBaseLoader_stdafx.h" 55 | #include "EGenGenerateAndLoadBaseOutput.h" 56 | #include "EGenGenerateAndLoadStandardOutput.h" 57 | #include "DriverParamSettings.h" 58 | #include "EGenLogger.h" 59 | #include "EGenGenerateAndLoad.h" 60 | 61 | #endif // #ifndef EGEN_GENERATE_AND_LOAD_STDAFX_H 62 | -------------------------------------------------------------------------------- /src/app/tpce/egen/EGenNullLoader_stdafx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | // stdafx.h : include file for standard system include files, 38 | // or project specific include files that are used frequently, but 39 | // are changed infrequently 40 | // 41 | 42 | #ifndef EGEN_NULLLOADER_STDAFX_H 43 | #define EGEN_NULLLOADER_STDAFX_H 44 | 45 | #include "NullLoader.h" 46 | #include "NullLoaderFactory.h" 47 | 48 | #endif // #ifndef EGEN_NULLLOADER_STDAFX_H 49 | -------------------------------------------------------------------------------- /src/app/tpce/egen/EGenTables_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Chris Ruemmler 35 | */ 36 | 37 | #ifndef EGEN_TABLES_COMMON_H 38 | #define EGEN_TABLES_COMMON_H 39 | 40 | #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | 54 | using namespace std; 55 | 56 | #include "EGenStandardTypes.h" 57 | #include "EGenUtilities_stdafx.h" 58 | #include "Table_Defs.h" 59 | #include "TableTemplate.h" 60 | #include "InputFlatFilesDeclarations.h" 61 | #include "InputFlatFilesStructure.h" 62 | #include "InputFileNoWeight.h" 63 | #include "Person.h" 64 | #include "CustomerSelection.h" 65 | 66 | #endif // #ifndef EGEN_TABLES_COMMON_H 67 | -------------------------------------------------------------------------------- /src/app/tpce/egen/FlatFileLoad_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Chris Ruemmler 35 | */ 36 | 37 | #ifndef FLAT_FILE_LOAD_COMMON_H 38 | #define FLAT_FILE_LOAD_COMMON_H 39 | 40 | #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers 41 | 42 | #include 43 | #include 44 | #include 45 | 46 | #include "FlatFileLoader.h" 47 | #include "Table_Defs.h" 48 | #include "DateTime.h" 49 | #include "error.h" 50 | 51 | #endif // #ifndef FLAT_FILE_LOAD_COMMON_H 52 | -------------------------------------------------------------------------------- /src/app/tpce/egen/FlatSectorLoad.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Doug Johnson 35 | */ 36 | 37 | /* 38 | * Flat file loader for SECTOR. 39 | */ 40 | #ifndef FLAT_SECTOR_LOAD_H 41 | #define FLAT_SECTOR_LOAD_H 42 | 43 | #include "FlatFileLoad_common.h" 44 | 45 | namespace TPCE 46 | { 47 | 48 | class CFlatSectorLoad : public CFlatFileLoader 49 | { 50 | public: 51 | CFlatSectorLoad( char *szFileName, FlatFileOutputModes FlatFileOutputMode ) : CFlatFileLoader(szFileName, FlatFileOutputMode){}; 52 | 53 | /* 54 | * Writes a record to the file. 55 | */ 56 | void WriteNextRecord(PT next_record) 57 | { 58 | int rc = fprintf( hOutFile, SectorRowFmt, 59 | next_record->SC_ID, 60 | next_record->SC_NAME 61 | ); 62 | if (rc < 0) { 63 | throw CSystemErr(CSystemErr::eWriteFile, "CFlatSectorLoad::WriteNextRecord"); 64 | } 65 | } 66 | }; 67 | 68 | } // namespace TPCE 69 | 70 | #endif //FLAT_SECTOR_LOAD_H 71 | -------------------------------------------------------------------------------- /src/app/tpce/egen/MEETradeRequestActions.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Doug Johnson 35 | */ 36 | 37 | /****************************************************************************** 38 | * Description: These are the different actions the MEE can take 39 | * with an inbound trade request. 40 | ******************************************************************************/ 41 | 42 | #ifndef MEE_TRADE_REQUEST_ACTIONS_H 43 | #define MEE_TRADE_REQUEST_ACTIONS_H 44 | 45 | namespace TPCE 46 | { 47 | 48 | enum eMEETradeRequestAction 49 | { 50 | eMEEProcessOrder = 0, 51 | eMEESetLimitOrderTrigger 52 | }; 53 | 54 | } //namespace TPCE 55 | 56 | #endif //MEE_TRADE_REQUEST_ACTIONS_H 57 | -------------------------------------------------------------------------------- /src/app/tpce/egen/NullLoad_stdafx.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #ifndef EGEN_NULLLOADER_STDAFX_H 38 | #define EGEN_NULLLOADER_STDAFX_H 39 | 40 | #include "NullLoader.h" 41 | #include "NullLoaderFactory.h" 42 | 43 | #endif // #ifndef EGEN_NULLLOADER_STDAFX_H 44 | -------------------------------------------------------------------------------- /src/app/tpce/egen/ReadRowFunctions.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Matt Emmerton 35 | */ 36 | 37 | /* 38 | * This file allows compile-time selection of the type of ReadRowFunctions to 39 | * use at runtime - C++-style istream (the default) or C-style sscanf. 40 | */ 41 | 42 | #ifdef READROW_SSCANF 43 | #include "ReadRowFunctions_sscanf.cpp" 44 | #else 45 | #include "ReadRowFunctions_istream.cpp" 46 | #endif 47 | -------------------------------------------------------------------------------- /src/app/tpce/egen/SecurityPriceRange.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Doug Johnson 35 | */ 36 | 37 | /****************************************************************************** 38 | * Description: Minimum and maximum values for a security's price range 39 | ******************************************************************************/ 40 | 41 | #ifndef SECURITY_PRICE_RANGE_H 42 | #define SECURITY_PRICE_RANGE_H 43 | 44 | namespace TPCE 45 | { 46 | 47 | const double fMinSecPrice = 20.00; 48 | const double fMaxSecPrice = 30.00; 49 | 50 | } //namespace TPCE 51 | 52 | #endif //SECURITY_PRICE_RANGE_H 53 | -------------------------------------------------------------------------------- /src/app/tpce/egen/SyncLockInterface.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a preliminary 5 | * version of a benchmark specification being developed by the TPC. The 6 | * Work is being made available to the public for review and comment only. 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #ifndef SYNCLOCK_H 38 | #define SYNCLOCK_H 39 | 40 | #include 41 | 42 | /* 43 | * Syncronization lock that lets only one thread acquire it. 44 | */ 45 | 46 | namespace TPCE 47 | { 48 | 49 | class CSyncLock 50 | { 51 | pthread_mutex_t mutex; 52 | public: 53 | CSyncLock() { 54 | pthread_mutex_init(&mutex, NULL); 55 | } 56 | ~CSyncLock() { 57 | pthread_mutex_destroy(&mutex); 58 | } 59 | 60 | // Acquire lock or block until it is available 61 | void ClaimLock() { 62 | pthread_mutex_lock(&mutex); 63 | } 64 | 65 | // Release lock so that it can be acquired again 66 | void ReleaseLock() { 67 | pthread_mutex_unlock(&mutex); 68 | } 69 | }; 70 | 71 | } // namespace TPCE 72 | 73 | #endif // #ifndef SYNCLOCK_H 74 | -------------------------------------------------------------------------------- /src/app/tpce/egen/TradeTypeIDs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Doug Johnson 35 | */ 36 | 37 | /****************************************************************************** 38 | * Description: Trade Type IDs corresponding to the TradeType.txt flat 39 | * file. 40 | * Note: The order of enumeration members must match the 41 | * order of rows in the TradeType.txt flat file. 42 | ******************************************************************************/ 43 | 44 | #ifndef TRADE_TYPE_IDS_H 45 | #define TRADE_TYPE_IDS_H 46 | 47 | namespace TPCE 48 | { 49 | 50 | enum eTradeTypeID 51 | { 52 | eMarketBuy = 0, 53 | eMarketSell, 54 | eStopLoss, 55 | eLimitSell, 56 | eLimitBuy, 57 | 58 | eMaxTradeTypeID // should be the last - contains the number of items in the enumeration 59 | }; 60 | 61 | } //namespace TPCE 62 | 63 | #endif //TRADE_TYPE_IDS_H 64 | -------------------------------------------------------------------------------- /src/app/tpce/egen/TxnHarnessDataMaintenance.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #ifndef TXN_HARNESS_DATA_MAINTENANCE_H 38 | #define TXN_HARNESS_DATA_MAINTENANCE_H 39 | 40 | #include "TxnHarnessDBInterface.h" 41 | 42 | namespace TPCE 43 | { 44 | 45 | class CDataMaintenance 46 | { 47 | CDataMaintenanceDBInterface* m_db; 48 | 49 | public: 50 | CDataMaintenance(CDataMaintenanceDBInterface *pDB) 51 | : m_db(pDB) 52 | { 53 | }; 54 | 55 | rc_t DoTxn( PDataMaintenanceTxnInput pTxnInput, PDataMaintenanceTxnOutput pTxnOutput ) 56 | { 57 | TXN_HARNESS_SET_STATUS_SUCCESS; 58 | 59 | // Execute Frame 1 60 | try_return(m_db->DoDataMaintenanceFrame1(pTxnInput)); 61 | } 62 | }; 63 | 64 | } // namespace TPCE 65 | 66 | #endif //TXN_HARNESS_DATA_MAINTENANCE_H 67 | -------------------------------------------------------------------------------- /src/app/tpce/egen/TxnHarnessTradeCleanup.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Matt Emmerton 35 | */ 36 | 37 | #ifndef TXN_HARNESS_TRADE_CLEANUP_H 38 | #define TXN_HARNESS_TRADE_CLEANUP_H 39 | 40 | #include "TxnHarnessDBInterface.h" 41 | 42 | namespace TPCE 43 | { 44 | 45 | class CTradeCleanup 46 | { 47 | CTradeCleanupDBInterface* m_db; 48 | 49 | public: 50 | CTradeCleanup(CTradeCleanupDBInterface *pDB) 51 | : m_db(pDB) 52 | { 53 | } 54 | 55 | rc_t DoTxn( PTradeCleanupTxnInput pTxnInput, PTradeCleanupTxnOutput pTxnOutput) 56 | { 57 | TXN_HARNESS_SET_STATUS_SUCCESS; 58 | 59 | try_return(m_db->DoTradeCleanupFrame1(pTxnInput)); 60 | } 61 | }; 62 | 63 | } // namespace TPCE 64 | 65 | #endif //TXN_HARNESS_TRADE_CLEANUP_H 66 | -------------------------------------------------------------------------------- /src/app/tpce/egen/error.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Sergey Vasilevskiy 35 | */ 36 | 37 | #include "EGenUtilities_stdafx.h" // Windows-specific error file 38 | 39 | using namespace TPCE; 40 | 41 | CSystemErr::CSystemErr(Action eAction, char const * szLocation) 42 | : CBaseErr(szLocation) 43 | , m_eAction(eAction) 44 | { 45 | #ifdef WIN32 46 | m_idMsg = GetLastError(); //for Windows 47 | #elif (__unix) || (_AIX) 48 | m_idMsg = errno; //for Unix 49 | #else 50 | #error No system error routine defined. 51 | #endif 52 | } 53 | 54 | CSystemErr::CSystemErr(int iError, Action eAction, char const * szLocation) 55 | : CBaseErr(szLocation) 56 | , m_eAction(eAction) 57 | { 58 | // This constructor is provided for registry functions where the function return code 59 | // is the error code. 60 | m_idMsg = iError; 61 | } 62 | 63 | const char * CSystemErr::ErrorText() const 64 | { 65 | return strerror(m_idMsg); 66 | } 67 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/Charge.txt: -------------------------------------------------------------------------------- 1 | TMB 1 5 2 | TMB 2 4.5 3 | TMB 3 2.5 4 | TMS 1 5 5 | TMS 2 4.5 6 | TMS 3 2.5 7 | TSL 1 6.5 8 | TSL 2 5.5 9 | TSL 3 3.5 10 | TLS 1 6 11 | TLS 2 5 12 | TLS 3 3 13 | TLB 1 6 14 | TLB 2 5 15 | TLB 3 3 16 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/CompanySPRate.txt: -------------------------------------------------------------------------------- 1 | 1 A 2 | 1 B 3 | 1 C 4 | 1 AA 5 | 1 AB 6 | 1 AC 7 | 1 BA 8 | 1 BB 9 | 1 BC 10 | 1 CA 11 | 1 CB 12 | 1 CC 13 | 1 AAA 14 | 1 AAB 15 | 1 AAC 16 | 1 ABA 17 | 1 ABB 18 | 1 ABC 19 | 1 ACA 20 | 1 ACB 21 | 1 ACC 22 | 1 BAA 23 | 1 BAB 24 | 1 BAC 25 | 1 BBA 26 | 1 BBB 27 | 1 BBC 28 | 1 BCA 29 | 1 BCB 30 | 1 BCC 31 | 1 CAA 32 | 1 CAB 33 | 1 CAC 34 | 1 CBA 35 | 1 CBB 36 | 1 CBC 37 | 1 CCA 38 | 1 CCB 39 | 1 CCC 40 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/Exchange.txt: -------------------------------------------------------------------------------- 1 | NYSE New York Stock Exchange 0930 1600 Simulates the New York Stock Exchange 1 2 | NASDAQ NASDAQ (National Association of Security Dealers and Quotations) 0930 1600 Simulates the NASDAQ 2 3 | AMEX American Stock Exchange 0930 1600 Simulates the American Stock Exchange 3 4 | PCX Pacific Exchange 0930 1600 Simulates the Pacific Exchange 4 5 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/NonTaxableAccountName.txt: -------------------------------------------------------------------------------- 1 | Traditional IRA 2 | Roth IRA 3 | IRA-SEP 4 | 401(k) 5 | Pension Account 6 | Rollover IRA 7 | Retirement Fund 8 | Roth 401(K) 9 | Flexible Spending 10 | Health Savings 11 | Non-Taxable Trust 12 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/Sector.txt: -------------------------------------------------------------------------------- 1 | BM Basic Materials 2 | CG Capital Goods 3 | CO Conglomerates 4 | CC Consumer Cyclical 5 | CN Consumer Non-Cyclical 6 | EN Energy 7 | FN Financial 8 | HC Healthcare 9 | SV Services 10 | TC Technology 11 | TR Transportation 12 | UT Utilities 13 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/StatusType.txt: -------------------------------------------------------------------------------- 1 | CMPT Completed 2 | ACTV Active 3 | SBMT Submitted 4 | PNDG Pending 5 | CNCL Canceled 6 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/StreetSuffix.txt: -------------------------------------------------------------------------------- 1 | 1 Street 2 | 1 Road 3 | 1 Court 4 | 1 Boulevard 5 | 1 Way 6 | 1 Lane 7 | 1 Crescent 8 | 1 Park 9 | 1 Avenue 10 | 1 Rue 11 | 1 Drive 12 | 1 South 13 | 1 North 14 | 1 West 15 | 1 East 16 | 1 Upper 17 | 1 Lower 18 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/TaxRatesCountry.txt: -------------------------------------------------------------------------------- 1 | 1 US1 U.S. Income Tax Bracket for the poor 0.15 2 | 1 US2 U.S. Income Tax Bracket for the huddled masses 0.275 3 | 1 US3 U.S. Income Tax Bracket for the masses 0.305 4 | 1 US4 U.S. Income Tax Bracket for the well to do 0.355 5 | 1 US5 U.S. Income Tax Bracket for the filthy rich 0.391 6 | 2 CN1 Canadian Income Tax for the poor 0.16 7 | 2 CN2 Canadian Income Tax for the huddled masses 0.22 8 | 2 CN3 Canadian Income Tax for the well to do 0.26 9 | 2 CN4 Canadian Income Tax for the filthy rich 0.29 10 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/TaxableAccountName.txt: -------------------------------------------------------------------------------- 1 | Emergency Expenses 2 | Vacation Account 3 | Healthcare Fund 4 | New Car Account 5 | House Money 6 | Individual Account 7 | Family Trust 8 | Custodial Account 9 | Joint Account 10 | Business Account 11 | College Fund 12 | Savings Account 13 | Play Money 14 | -------------------------------------------------------------------------------- /src/app/tpce/egen/flat/egen_flat_in/TradeType.txt: -------------------------------------------------------------------------------- 1 | TMB Market-Buy 0 1 2 | TMS Market-Sell 1 1 3 | TSL Stop-Loss 1 0 4 | TLS Limit-Sell 1 0 5 | TLB Limit-Buy 0 0 6 | -------------------------------------------------------------------------------- /src/app/tpce/egen/locking.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Christopher Chan-Nui, Matt Emmerton 35 | */ 36 | 37 | #ifndef LOCKING_H_INCLUDED 38 | #define LOCKING_H_INCLUDED 39 | 40 | #include "EGenStandardTypes.h" 41 | 42 | namespace TPCE 43 | { 44 | 45 | // Standard mutex 46 | class CMutex 47 | { 48 | private: 49 | TMutex mutex_; 50 | TMutex* mutex(); 51 | public: 52 | CMutex(); 53 | ~CMutex(); 54 | void lock(); 55 | void unlock(); 56 | }; 57 | 58 | // Provide a RAII style lock for any class which supports 59 | // lock() and unlock() 60 | template 61 | class Locker 62 | { 63 | private: 64 | T& mutex_; 65 | 66 | public: 67 | explicit Locker(T& mutex) 68 | : mutex_(mutex) 69 | { 70 | mutex_.lock(); 71 | } 72 | 73 | ~Locker() { 74 | mutex_.unlock(); 75 | } 76 | }; 77 | 78 | } 79 | 80 | #endif // LOCKING_H_INCLUDED 81 | -------------------------------------------------------------------------------- /src/app/tpce/egen/progressmeterinterface.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Christopher Chan-Nui 35 | */ 36 | 37 | #include "progressmeterinterface.h" 38 | #include "unusedflag.h" 39 | 40 | namespace TPCE 41 | { 42 | 43 | // Dummy implementation in case someone wants to have a null progress meter 44 | ProgressMeterInterface::~ProgressMeterInterface() 45 | { 46 | } 47 | 48 | void ProgressMeterInterface::display() const 49 | { 50 | } 51 | 52 | void ProgressMeterInterface::inc(int count UNUSED) 53 | { 54 | } 55 | 56 | void ProgressMeterInterface::message(const std::string& mesg UNUSED, int level UNUSED) 57 | { 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/app/tpce/egen/progressmeterinterface.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Christopher Chan-Nui 35 | */ 36 | 37 | #ifndef PROGRESSMETERINTERFACE_H_INCLUDED 38 | #define PROGRESSMETERINTERFACE_H_INCLUDED 39 | 40 | #include 41 | 42 | namespace TPCE 43 | { 44 | 45 | // Interface to provide a simple progress indicator 46 | // Also an interface to display messages through. 47 | class ProgressMeterInterface 48 | { 49 | public: 50 | virtual ~ProgressMeterInterface(); 51 | 52 | // Display the current work status. 53 | virtual void display() const; 54 | 55 | // Mark some work done 56 | // count - amount of "work" completed 57 | virtual void inc(int count=1); 58 | 59 | // Print out an arbitrary message 60 | // mesg - message to display 61 | // level - level of message for verbosity filtering 62 | virtual void message(const std::string& mesg, int level=0); 63 | }; 64 | 65 | } 66 | 67 | #endif // PROGRESSMETER_H_INCLUDED 68 | -------------------------------------------------------------------------------- /src/app/tpce/egen/unusedflag.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Legal Notice 3 | * 4 | * This document and associated source code (the "Work") is a part of a 5 | * benchmark specification maintained by the TPC. 6 | * 7 | * The TPC reserves all right, title, and interest to the Work as provided 8 | * under U.S. and international laws, including without limitation all patent 9 | * and trademark rights therein. 10 | * 11 | * No Warranty 12 | * 13 | * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION 14 | * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE 15 | * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER 16 | * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY, 17 | * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES, 18 | * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR 19 | * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF 20 | * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE. 21 | * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT, 22 | * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT 23 | * WITH REGARD TO THE WORK. 24 | * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO 25 | * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE 26 | * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS 27 | * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT, 28 | * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY, 29 | * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT 30 | * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD 31 | * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. 32 | * 33 | * Contributors 34 | * - Christopher Chan-Nui 35 | */ 36 | /* 37 | * This file is only to provide a define which can hint some compilers that 38 | * certain parameters are unused. 39 | */ 40 | #ifndef UNUSEDFLAG_H_INCLUDED 41 | #define UNUSEDFLAG_H_INCLUDED 42 | #ifdef __GNUC__ 43 | #define UNUSED __attribute__((unused)) 44 | #else 45 | #define UNUSED 46 | #endif 47 | #endif // UNUSEDFLAG_H_INCLUDED 48 | -------------------------------------------------------------------------------- /src/app/tpce/tpce_constants.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_OLTP_TPCE_CON_H 2 | #define NOCC_OLTP_TPCE_CON_H 3 | 4 | // This file defines the constants used by tpce benchmark 5 | namespace nocc { 6 | namespace oltp { 7 | namespace tpce { 8 | 9 | const int accountPerPartition = 5000; 10 | const int caPerPartition = accountPerPartition * 10; 11 | const int companyPerPartition = 500 * (accountPerPartition / 1000); 12 | const int brokerPerPartition = accountPerPartition / 100; 13 | }; 14 | }; 15 | }; 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /src/core/common.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_COMMON_H 2 | #define NOCC_COMMON_H 3 | 4 | #define NOCC_DECLARE_typed_var(type, name) \ 5 | namespace nocc { \ 6 | extern type FLAGS_##name; \ 7 | } // namespace nocc 8 | 9 | 10 | namespace nocc { 11 | 12 | #define DISABLE_COPY_AND_ASSIGN(classname) \ 13 | private: \ 14 | classname(const classname&) = delete; \ 15 | classname& operator=(const classname&) = delete 16 | 17 | 18 | #define unlikely(x) __builtin_expect(!!(x), 0) 19 | #define likely(x) __builtin_expect(!!(x), 1) 20 | 21 | #define ALWAYS_INLINE __attribute__((always_inline)) 22 | 23 | }; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /src/core/config.hpp: -------------------------------------------------------------------------------- 1 | // This file contains the parsing of the framework 2 | 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | namespace nocc { 16 | 17 | namespace oltp { 18 | 19 | // return a network with @num@ servers given a host file 20 | std::vector parse_network(int num, std::string &hosts) { 21 | 22 | std::vector network; 23 | std::map black_list; 24 | network.clear(); 25 | black_list.clear(); 26 | 27 | using boost::property_tree::ptree; 28 | using namespace boost; 29 | using namespace property_tree; 30 | ptree pt; 31 | 32 | try { 33 | read_xml(hosts, pt); 34 | } catch (const ptree_error &e) { 35 | fprintf(stderr,"Failed to parse host file!"); 36 | assert(false); 37 | } 38 | 39 | try { 40 | // parse the black list of hosts 41 | BOOST_FOREACH(ptree::value_type &v, 42 | pt.get_child("hosts.black")) 43 | { 44 | std::string s = v.second.data(); 45 | boost::algorithm::trim_right(s); 46 | boost::algorithm::trim_left(s); 47 | 48 | if(black_list.find(s) == black_list.end()) 49 | black_list.insert(std::make_pair(s,true)); 50 | } 51 | } 52 | catch(const ptree_error &e) { 53 | // pass, no black list provided 54 | } 55 | // parse the hosts file 56 | try { 57 | 58 | // parse hosts 59 | BOOST_FOREACH(ptree::value_type &v, 60 | pt.get_child("hosts.macs")) 61 | { 62 | std::string s = v.second.data(); 63 | boost::algorithm::trim_right(s); 64 | boost::algorithm::trim_left(s); 65 | if(black_list.find(s) != black_list.end()) { 66 | 67 | } else { 68 | if(network.size() + 1 > num) 69 | break; 70 | network.push_back(s); 71 | } 72 | } // end iterating hosts 73 | 74 | } catch (const ptree_error &e) { 75 | assert(false); 76 | } 77 | 78 | assert(network.size() == num); 79 | return network; 80 | } 81 | 82 | }; // namespace oltp 83 | 84 | }; // namespace nocc 85 | -------------------------------------------------------------------------------- /src/core/logging.cc: -------------------------------------------------------------------------------- 1 | #include "logging.h" 2 | 3 | #include 4 | 5 | using namespace std; 6 | 7 | namespace nocc { 8 | 9 | // control flags for color 10 | #define R_BLACK 39 11 | #define R_RED 31 12 | #define R_GREEN 32 13 | #define R_YELLOW 33 14 | #define R_BLUE 34 15 | #define R_MAGENTA 35 16 | #define R_CYAN 36 17 | #define R_WHITE 37 18 | 19 | std::string StripBasename(const std::string &full_path) { 20 | const char kSeparator = '/'; 21 | size_t pos = full_path.rfind(kSeparator); 22 | if (pos != std::string::npos) { 23 | return full_path.substr(pos + 1, std::string::npos); 24 | } else { 25 | return full_path; 26 | } 27 | } 28 | 29 | inline string EndcolorFlag() { 30 | char flag[7]; 31 | snprintf(flag,7, "%c[0m", 0x1B); 32 | return string(flag); 33 | } 34 | 35 | const int RTX_DEBUG_LEVEL_COLOR[] = {R_BLACK,R_YELLOW,R_BLACK,R_GREEN,R_MAGENTA,R_RED,R_RED}; 36 | 37 | MessageLogger::MessageLogger(const char *file, int line, int level) 38 | :level_(level) { 39 | if(level_ < ROCC_LOG_LEVEL) 40 | return; 41 | stream_ << "[" << StripBasename(std::string(file)) << ":" << line << "] "; 42 | } 43 | 44 | 45 | MessageLogger::~MessageLogger() { 46 | if(level_ >= ROCC_LOG_LEVEL) { 47 | stream_ << "\n"; 48 | std::cout << "\033[" << RTX_DEBUG_LEVEL_COLOR[std::min(level_,6)] << "m" 49 | << stream_.str() << EndcolorFlag(); 50 | if(level_ >= LOG_FATAL) 51 | abort(); 52 | } 53 | } 54 | 55 | } // namespace nocc 56 | -------------------------------------------------------------------------------- /src/core/rdma_sched.cc: -------------------------------------------------------------------------------- 1 | #include "rdma_sched.h" 2 | #include "routine.h" // poll_comps will add_routine to scheduler 3 | 4 | #include "util/util.h" // for rdtsc() 5 | #include "logging.h" 6 | 7 | #include "rrpc.h" 8 | 9 | extern size_t nthreads; 10 | extern size_t current_partition; 11 | 12 | using namespace std; 13 | using namespace rdmaio; 14 | 15 | namespace nocc { 16 | 17 | namespace oltp { 18 | 19 | RScheduler::RScheduler() { 20 | 21 | } 22 | 23 | RScheduler::~RScheduler() { 24 | 25 | } 26 | 27 | void RScheduler::thread_local_init(int coroutines) { 28 | pending_counts_ = new int[coroutines + 1]; 29 | std::fill_n(pending_counts_,1 + coroutines,0); 30 | } 31 | 32 | 33 | void RScheduler::poll_comps() { 34 | 35 | for(auto it = pending_qps_.begin();it != pending_qps_.end();) { 36 | 37 | RCQP *qp = *it; 38 | auto poll_result = qp->poll_send_completion(wc_); 39 | 40 | if(poll_result == 0) { 41 | it++; 42 | continue; 43 | } 44 | 45 | if(unlikely(wc_.status != IBV_WC_SUCCESS)) { 46 | LOG(3) << "got bad completion with status: " << wc_.status << " with error " << ibv_wc_status_str(wc_.status) 47 | << ";@ node " << qp->idx_.node_id; 48 | if(wc_.status != IBV_WC_RETRY_EXC_ERR) 49 | assert(false); 50 | else { 51 | it++; 52 | continue; 53 | } 54 | } 55 | 56 | static_assert(sizeof(wc_.wr_id) == sizeof(uint64_t),"Un supported wr_id size!"); 57 | uint64_t low_watermark = decode_watermark(wc_.wr_id); 58 | 59 | ASSERT(low_watermark > qp->low_watermark_) << "encoded watermark: " << low_watermark 60 | << "; current watermark: " << qp->low_watermark_; 61 | qp->low_watermark_ = low_watermark; 62 | 63 | auto cor_id = decode_corid(wc_.wr_id); 64 | 65 | if(cor_id == 0) 66 | continue; // ignore null completion 67 | 68 | //LOG(2) << "polled " << cor_id << " low " << low_watermark; 69 | 70 | ASSERT(pending_counts_[cor_id] > 0) << "cor id " << cor_id 71 | << "; pendings " << pending_counts_[cor_id]; 72 | pending_counts_[cor_id] -= 1; 73 | 74 | if(pending_counts_[cor_id] == 0 && RRpc::reply_counts_[cor_id] == 0) { 75 | add_to_routine_list(cor_id); 76 | } 77 | 78 | // update the iterator 79 | it = pending_qps_.erase(it); 80 | } 81 | } 82 | 83 | void RScheduler::report() { 84 | } 85 | 86 | __thread int *RScheduler::pending_counts_ = NULL; 87 | 88 | 89 | } // namespace db 90 | 91 | } // namespace nocc 92 | -------------------------------------------------------------------------------- /src/core/routine.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "routine.h" 5 | #include "rworker.h" 6 | 7 | namespace nocc { 8 | 9 | // default routines to bind 10 | 11 | namespace oltp { 12 | 13 | __thread std::queue *ro_pool; 14 | 15 | // add a routine chain to RPC 16 | __thread RoutineMeta *next_routine_array; 17 | __thread RoutineMeta *routine_tailer; // the header of the scheduler 18 | __thread RoutineMeta *routine_header; // the tailer of the scheduler 19 | 20 | __thread RoutineMeta *one_shot_routine_pool; 21 | 22 | __thread one_shot_func_t *one_shot_callbacks; 23 | __thread RWorker *worker = NULL; 24 | 25 | bool inited = false; 26 | std::mutex routine_mtx; 27 | 28 | void one_shot_func(yield_func_t &yield,RoutineMeta *routine) { 29 | 30 | while(1) { 31 | // callback 32 | one_shot_callbacks[routine->info_.req_id](yield,routine->info_.from_mac, 33 | routine->info_.from_routine,routine->info_.msg, 34 | NULL); 35 | ro_pool->push(routine); 36 | free(routine->info_.msg); 37 | // yield back 38 | // need to reset-the context if necessary 39 | worker->indirect_must_yield(yield); 40 | } 41 | } 42 | 43 | void RoutineMeta::thread_local_init(int num_ros,int coroutines,coroutine_func_t *routines_,RWorker *w) { 44 | 45 | // init worker 46 | assert(worker == NULL); 47 | worker = w; 48 | 49 | // init next routine array 50 | next_routine_array = new RoutineMeta[coroutines + 1]; 51 | 52 | for(uint i = 0;i < coroutines + 1;++i) { 53 | next_routine_array[i].id_ = i; 54 | next_routine_array[i].routine_ = routines_ + i; 55 | } 56 | 57 | for(uint i = 0;i < coroutines + 1;++i) { 58 | next_routine_array[i].prev_ = next_routine_array + i - 1; 59 | next_routine_array[i].next_ = next_routine_array + i + 1; 60 | } 61 | 62 | routine_header = &(next_routine_array[0]); 63 | routine_tailer = &(next_routine_array[coroutines]); 64 | 65 | // set master routine's status 66 | next_routine_array[0].prev_ = routine_header; 67 | 68 | // set tail routine's chain 69 | next_routine_array[coroutines].next_ = routine_header; //loop back 70 | 71 | // init ro routine pool 72 | ro_pool = new std::queue(); 73 | for(uint i = 0;i < num_ros;++i) { 74 | RoutineMeta *meta = new RoutineMeta; 75 | meta->prev_ = NULL; meta->next_ = NULL; meta->id_ = 0; 76 | meta->routine_ = new coroutine_func_t(bind(one_shot_func,_1,meta)); 77 | ro_pool->push(meta); 78 | } 79 | 80 | one_shot_callbacks = new one_shot_func_t[MAX_ONE_SHOT_CALLBACK]; 81 | } 82 | 83 | void RoutineMeta::register_callback(one_shot_func_t callback,int id) { 84 | register_one_shot(callback,id); 85 | } 86 | 87 | 88 | }; 89 | 90 | }; 91 | -------------------------------------------------------------------------------- /src/core/rpc_buf_allocator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "ralloc/ralloc.h" // RDMA malloc 6 | 7 | namespace nocc { 8 | 9 | /** 10 | * The buffer used by rpc should be carefully allocated (from registered memory). 11 | * This class help this 12 | */ 13 | class RPCBufAllocator { 14 | public: 15 | RPCBufAllocator(int req_buf_num,int reply_buf_num,int req_msg_size,int reply_msg_size,int coroutines) { 16 | 17 | assert(req_buf_num > 0 && reply_buf_num > 0); 18 | 19 | RThreadLocalInit(); 20 | 21 | for(uint i = 0;i < coroutines + 1;++i) { 22 | req_buf_slots_.push_back(0); 23 | req_buf_pool_.push_back(std::vector ()); 24 | 25 | for(uint j = 0;j < req_buf_num;++j) { 26 | char *buf = (char *)Rmalloc(req_msg_size); 27 | req_buf_pool_[i].push_back(buf); 28 | } 29 | } 30 | 31 | for(uint i = 0;i < reply_buf_num;++i) { 32 | char *buf = (char *)Rmalloc(reply_msg_size); 33 | reply_buf_pool_.push_back(buf); 34 | } 35 | } 36 | 37 | // Get a reply buffer for the RPC handler 38 | inline char *get_reply_buf() { 39 | auto res = reply_buf_pool_[(reply_buf_slot_++) % reply_buf_pool_.size()]; 40 | return res; 41 | } 42 | 43 | // Get a buffer for sending RPC request. 44 | // This buffer can be re-used. 45 | inline char *get_fly_buf(int cid) { 46 | char *res = req_buf_pool_[cid][(req_buf_slots_[cid]++) % req_buf_pool_[cid].size()]; 47 | return res; 48 | } 49 | 50 | private: 51 | // buffer pools 52 | std::vector reply_buf_pool_; 53 | uint16_t reply_buf_slot_ = 0; 54 | 55 | std::vector >req_buf_pool_; 56 | std::vector req_buf_slots_; 57 | }; 58 | 59 | }; 60 | -------------------------------------------------------------------------------- /src/core/rrpc.cc: -------------------------------------------------------------------------------- 1 | #include "rrpc.h" 2 | 3 | #include "routine.h" 4 | #include "logging.h" 5 | 6 | #include "rdma_sched.h" 7 | 8 | namespace nocc { 9 | 10 | namespace oltp { 11 | 12 | RRpc::RRpc(int tid,int coroutines,int req_buf_num,int reply_buf_num) 13 | : worker_id_(tid), 14 | allocator_(req_buf_num,reply_buf_num, 15 | MAX_MSG_SIZE + rpc_padding(),1024, 16 | coroutines) 17 | { 18 | for(uint i = 0;i < MAX_RPC_SUPPORT;++i) 19 | register_[i] = false; 20 | 21 | reply_bufs_ = new char*[1 + coroutines]; 22 | reply_counts_ = new int[1 + coroutines]; 23 | 24 | std::fill_n(reply_counts_,1 + coroutines,0); 25 | std::fill_n(reply_bufs_,1 + coroutines,static_cast(NULL)); 26 | } 27 | 28 | bool RRpc::poll_comp_callback(char *msg,int from,int from_t) { 29 | 30 | rrpc_header *header = (rrpc_header *) msg; 31 | 32 | if(header->meta.type == REQ) { 33 | // normal rpcs 34 | try { 35 | callbacks_[header->meta.rpc_id](from,header->meta.cid,msg + sizeof(rrpc_header), 36 | (void *)(header->meta.payload)); 37 | } catch (...) { 38 | LOG(7) << "rpc called failed at " << worker_id_ << ";With rpc id " 39 | << header->meta.rpc_id; 40 | } 41 | } else if (header->meta.type == Y_REQ) { 42 | // copy the msg 43 | char *temp = (char *)malloc(header->meta.payload); 44 | memcpy(temp,msg + sizeof(rrpc_header),header->meta.payload); 45 | add_one_shot_routine(from,header->meta.cid,header->meta.rpc_id,temp); 46 | 47 | } else if (header->meta.type == REPLY) { 48 | // This is a reply 49 | ASSERT(header->meta.cid != 0); 50 | if(unlikely(reply_counts_[header->meta.cid] <= 0)) { 51 | LOG(7) << "receive a reply from cid" << header->meta.cid << " at worker " << worker_id_ << " @mac " << from 52 | << " which is not required."; 53 | } 54 | 55 | char *buf = reply_bufs_[header->meta.cid]; 56 | assert(header->meta.payload < 1024 - rpc_padding()); 57 | memcpy(buf,msg + sizeof(rrpc_header),header->meta.payload); 58 | 59 | reply_bufs_[header->meta.cid] += header->meta.payload; 60 | 61 | reply_counts_[header->meta.cid] -= 1; 62 | if(reply_counts_[header->meta.cid] == 0 63 | && RScheduler::pending_counts_[header->meta.cid] == 0) { // avoid the chain from being added twice 64 | 65 | reply_bufs_[header->meta.cid] = NULL; 66 | add_to_routine_list(header->meta.cid); 67 | } 68 | } else { 69 | assert(false); 70 | } 71 | return true; 72 | } 73 | 74 | __thread int *RRpc::reply_counts_ = NULL; 75 | 76 | } // namespace oltp 77 | 78 | } // namespace nocc 79 | -------------------------------------------------------------------------------- /src/core/rworker_yield_impl.hpp: -------------------------------------------------------------------------------- 1 | namespace nocc { 2 | namespace oltp { 3 | 4 | // class nocc_worker 5 | inline ALWAYS_INLINE 6 | void RWorker::indirect_yield(yield_func_t &yield) { 7 | // make a simple check to avoid yield, if possible 8 | if(unlikely(rdma_sched_->pending_counts_[cor_id_] == 0 9 | && !rpc_->has_pending_reqs(cor_id_))) { 10 | return; 11 | } 12 | indirect_must_yield(yield); 13 | } 14 | 15 | inline ALWAYS_INLINE 16 | void RWorker::indirect_must_yield(yield_func_t &yield) { 17 | 18 | int next = routine_meta_->next_->id_; 19 | cor_id_ = next; 20 | auto cur = routine_meta_; 21 | routine_meta_ = cur->next_; 22 | change_ctx(cor_id_); 23 | cur->yield_from_routine_list(yield); 24 | 25 | change_ctx(cor_id_); 26 | } 27 | 28 | 29 | inline ALWAYS_INLINE 30 | void RWorker::yield_next(yield_func_t &yield) { 31 | // yield to the next routine 32 | int next = routine_meta_->next_->id_; 33 | routine_meta_ = routine_meta_->next_; 34 | cor_id_ = next; 35 | change_ctx(cor_id_); 36 | 37 | routine_meta_->yield_to(yield); 38 | 39 | change_ctx(cor_id_); 40 | assert(cor_id_ == routine_meta_->id_); 41 | } 42 | 43 | // end class nocc_worker 44 | 45 | } // 46 | } // namespace nocc 47 | -------------------------------------------------------------------------------- /src/core/tcp_adapter.cc: -------------------------------------------------------------------------------- 1 | #include "tcp_adapter.hpp" 2 | 3 | // a dummy cc file for static member's allocations 4 | namespace nocc { 5 | 6 | /** 7 | * Global poller for TCP based communication 8 | */ 9 | zmq::context_t recv_context(1); 10 | zmq::context_t send_context(1); 11 | AdapterPoller *poller = NULL; 12 | std::vector local_comm_queues; 13 | 14 | std::vector Adapter::sockets; 15 | std::vector Adapter::locks; 16 | 17 | 18 | }; // namespace nocc 19 | -------------------------------------------------------------------------------- /src/core/utils/amd64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _AMD64_H_ 26 | #define _AMD64_H_ 27 | 28 | #include "macros.h" 29 | #include 30 | 31 | inline ALWAYS_INLINE void 32 | nop_pause() 33 | { 34 | __asm volatile("pause" : :); 35 | } 36 | 37 | #endif /* _AMD64_H_ */ 38 | -------------------------------------------------------------------------------- /src/core/utils/count_vector.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace nocc { 6 | 7 | // a vector class to store count data 8 | template 9 | class CountVector : public std::vector { 10 | public: 11 | CountVector() : 12 | std::vector() 13 | { 14 | } 15 | 16 | double average() { 17 | 18 | if(std::vector::size() == 0) 19 | return 0.0; 20 | 21 | T sum = 0; 22 | for(auto it = std::vector::begin();it != std::vector::end();++it) { 23 | sum += *it; 24 | } 25 | double res = static_cast(sum) / std::vector::size(); 26 | return res; 27 | } 28 | 29 | // erase the first ratio% and last ratio% items 30 | void erase(double ratio) { 31 | 32 | // If there are too less elements in the vector, not erase items 33 | if(std::vector::size() < 5) 34 | return; 35 | 36 | int temp_size = std::vector::size(); 37 | int idx = std::floor(std::vector::size() * ratio); 38 | std::vector::erase(std::vector::begin() + temp_size - idx,std::vector::end()); 39 | std::vector::erase(std::vector::begin(),std::vector::begin() + idx); 40 | } 41 | 42 | void sort() { 43 | std::sort(std::vector::begin(),std::vector::end()); 44 | } 45 | }; 46 | 47 | } // namespace nocc 48 | -------------------------------------------------------------------------------- /src/core/utils/latency_profier.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_LATENCY_PROFILER 2 | #define NOCC_LATENCY_PROFILER 3 | 4 | #include "core/logging.h" 5 | 6 | namespace nocc { 7 | 8 | #define NOCC_STATICS 1 9 | 10 | // usage: TODO 11 | 12 | #if NOCC_STATICS == 1 13 | // Performance counting stats 14 | // To be more self-contained 15 | inline __attribute__ ((always_inline)) 16 | uint64_t rdtsc(void) 17 | { 18 | uint32_t hi, lo; 19 | __asm volatile("rdtsc" : "=a"(lo), "=d"(hi)); 20 | return ((uint64_t)lo)|(((uint64_t)hi)<<32); 21 | } 22 | 23 | #define LAT_VARS(X) uint64_t _## X ##_lat_; uint64_t _pre_## X ##_lat_; \ 24 | uint64_t _## X ##count_; uint64_t _pre_## X ##count_; 25 | #define INIT_LAT_VARS(X) _## X ##_lat_= 0,_pre_## X ##_lat_ = 0,_## X ##count_ = 0,_pre_## X ##count_ = 0; 26 | 27 | /** 28 | * record the start time 29 | */ 30 | #define START(X) auto _## X ##start = rdtsc(); 31 | 32 | /** 33 | * record the end time, add to the report 34 | */ 35 | #define END(X) { _## X ##_lat_ += rdtsc() - _## X ##start; \ 36 | _## X ##count_ += 1; \ 37 | } 38 | #define END_C(C,X) { C-> _## X ##_lat_ += rdtsc() - _## X ##start; \ 39 | C-> _## X ## count_ += 1; \ 40 | } 41 | 42 | /** 43 | * Add user specific value to the report 44 | */ 45 | #define ADD(X,v) { _## X ##_lat_ += v; \ 46 | _## X ##count_ += 1; \ 47 | } 48 | 49 | #define REPORT_V(X,v) { auto counts = _## X ##count_ - _pre_## X ##count_; \ 50 | _pre_## X ##count_ = _## X ##count_; \ 51 | counts = ((counts == 0)?1:counts); \ 52 | auto temp = _## X ##_lat_; \ 53 | v = (temp - _pre_## X ##_lat_) / (double)counts; \ 54 | LOG(3) << #X << " lat: " << v << " ;counts " << counts; \ 55 | _pre_## X ##_lat_ = temp; \ 56 | } 57 | 58 | #else 59 | // clear the counting stats to reduce performance impact 60 | #define LAT_VARS(X) ; 61 | #define INIT_LAT_VARS(X) ; 62 | #define START(X) ; 63 | #define END(X) ; 64 | #define END_C(C,X); 65 | #define ADD(X,v); 66 | #define REPORT_V(X,v) ; 67 | 68 | #endif 69 | 70 | #define REPORT(X) { double res; \ 71 | REPORT_V(X,res); \ 72 | } 73 | 74 | 75 | } // namespace nocc 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /src/core/utils/ndb_type_traits.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _NDB_TYPE_TRAITS_H_ 26 | #define _NDB_TYPE_TRAITS_H_ 27 | 28 | #include 29 | 30 | namespace private_ { 31 | 32 | // std::is_trivially_destructible not supported in g++-4.7, so we 33 | // do some hacky [conservative] variant of it 34 | template 35 | struct is_trivially_destructible { 36 | static const bool value = std::is_scalar::value; 37 | }; 38 | 39 | template 40 | struct is_trivially_destructible> { 41 | static const bool value = 42 | is_trivially_destructible::value && 43 | is_trivially_destructible::value; 44 | }; 45 | 46 | // XXX: same for now 47 | template 48 | struct is_trivially_copyable : public is_trivially_destructible {}; 49 | 50 | // user types should add their own specializations 51 | 52 | template 53 | struct typeutil { typedef const T & func_param_type; }; 54 | 55 | template 56 | struct primitive_typeutil { typedef T func_param_type; }; 57 | 58 | // specialize typeutil for int types to use primitive_typeutil 59 | 60 | #define SPECIALIZE_PRIM_TYPEUTIL(tpe) \ 61 | template <> struct typeutil< tpe > : public primitive_typeutil< tpe > {}; 62 | 63 | SPECIALIZE_PRIM_TYPEUTIL(bool) 64 | SPECIALIZE_PRIM_TYPEUTIL(int8_t) 65 | SPECIALIZE_PRIM_TYPEUTIL(uint8_t) 66 | SPECIALIZE_PRIM_TYPEUTIL(int16_t) 67 | SPECIALIZE_PRIM_TYPEUTIL(uint16_t) 68 | SPECIALIZE_PRIM_TYPEUTIL(int32_t) 69 | SPECIALIZE_PRIM_TYPEUTIL(uint32_t) 70 | SPECIALIZE_PRIM_TYPEUTIL(int64_t) 71 | SPECIALIZE_PRIM_TYPEUTIL(uint64_t) 72 | } 73 | 74 | #endif /* _NDB_TYPE_TRAITS_H_ */ 75 | -------------------------------------------------------------------------------- /src/core/utils/spinbarrier.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _SPINBARRIER_H_ 26 | #define _SPINBARRIER_H_ 27 | 28 | #include "amd64.h" 29 | #include "macros.h" 30 | #include "util.h" 31 | 32 | /** 33 | * Barrier implemented by spinning 34 | */ 35 | 36 | class spin_barrier { 37 | public: 38 | spin_barrier(size_t n) 39 | : n(n) 40 | { 41 | ALWAYS_ASSERT(n > 0); 42 | } 43 | #if 0 44 | spin_barrier(const spin_barrier &) = delete; 45 | spin_barrier(spin_barrier &&) = delete; 46 | spin_barrier &operator=(const spin_barrier &) = delete; 47 | #endif 48 | ~spin_barrier() 49 | { 50 | //ALWAYS_ASSERT(n == 0); 51 | } 52 | 53 | void 54 | count_down() 55 | { 56 | // written like this (instead of using __sync_fetch_and_add()) 57 | // so we can have assertions 58 | for (;;) { 59 | size_t copy = n; 60 | ALWAYS_ASSERT(copy > 0); 61 | if (__sync_bool_compare_and_swap(&n, copy, copy - 1)) 62 | return; 63 | } 64 | } 65 | 66 | void 67 | wait_for() 68 | { 69 | while (n > 0) 70 | nop_pause(); 71 | } 72 | 73 | public: 74 | volatile size_t n; 75 | }; 76 | 77 | #endif /* _SPINBARRIER_H_ */ 78 | -------------------------------------------------------------------------------- /src/core/utils/varint.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #include 26 | 27 | #include "varint.h" 28 | #include "macros.h" 29 | #include "util.h" 30 | 31 | using namespace std; 32 | using namespace nocc::util; 33 | 34 | static void 35 | do_test(uint32_t v) 36 | { 37 | uint8_t buf[5]; 38 | uint8_t *p = &buf[0]; 39 | p = write_uvint32(p, v); 40 | ALWAYS_ASSERT(size_t(p - &buf[0]) == size_uvint32(v)); 41 | 42 | const uint8_t *p0 = &buf[0]; 43 | uint32_t v0 = 0; 44 | p0 = read_uvint32(p0, &v0); 45 | ALWAYS_ASSERT(v == v0); 46 | ALWAYS_ASSERT(p == p0); 47 | } 48 | 49 | void 50 | varint::Test() 51 | { 52 | fast_random r(2043859); 53 | for (int i = 0; i < 1000; i++) 54 | do_test(r.next_u32()); 55 | cerr << "varint tests passed" << endl; 56 | } 57 | -------------------------------------------------------------------------------- /src/framework/bench_listener.h: -------------------------------------------------------------------------------- 1 | #ifndef RTX_BENCH_LISTENER 2 | #define RTX_BENCH_LISTENER 3 | 4 | #include "bench_worker.h" 5 | #include "bench_reporter.h" 6 | 7 | namespace nocc { 8 | 9 | namespace oltp { 10 | 11 | class BenchLocalListener : public RWorker { 12 | public: 13 | BenchLocalListener(unsigned worker_id,const std::vector &workers, 14 | BenchReporter *reporter); 15 | 16 | /** 17 | * Main thread body of bench_listener. 18 | */ 19 | virtual void run(); 20 | 21 | /** 22 | * Main worker routine. 23 | */ 24 | virtual void worker_routine(yield_func_t &yield); 25 | 26 | /** 27 | * Print necessary information 28 | */ 29 | void exit_handler(); 30 | 31 | private: 32 | /** 33 | * Worker routine at the master. 34 | * It start slaves, collect results from others 35 | */ 36 | void worker_routine_master(yield_func_t &yield); 37 | 38 | 39 | /** 40 | * Worker routine at the slave. 41 | * It send results (throughput,etc) to the master 42 | */ 43 | void worker_routine_slave(yield_func_t &yield); 44 | 45 | /** 46 | * Re-set worker's running status to *running* 47 | */ 48 | void start_workers(); 49 | 50 | /** 51 | * Used to monitor user inputs. 52 | */ 53 | void sigint_handler(int); 54 | 55 | /** 56 | * handler slave's initilization done RPCs. 57 | */ 58 | void init_rpc_handler(int id,int cid,char *msg,void *arg); 59 | 60 | /** 61 | * handler the start requests 62 | */ 63 | void start_rpc_handler(int id,int cid,char *msg,void *arg); 64 | 65 | /** 66 | * handler slave's requests 67 | */ 68 | void get_result_rpc_handler(int id,int cid,char *msg,void *arg); 69 | 70 | /** 71 | * handler the exit request 72 | */ 73 | void exit_rpc_handler(int id,int cid,char *msg,void *arg); 74 | 75 | 76 | 77 | private: 78 | int epoch_ = 0; // execution epoch 79 | int nresult_returned_ = 0; // used to collect return results 80 | BenchReporter *reporter_ = NULL; // the reporter used to parse results 81 | std::vector workers_; // record current worker at this machine 82 | }; 83 | 84 | } // end namespace oltp 85 | } // end namespace nocc 86 | 87 | #endif 88 | -------------------------------------------------------------------------------- /src/framework/bench_reporter.h: -------------------------------------------------------------------------------- 1 | #ifndef RTX_BENCH_REPORTER 2 | #define RTX_BENCH_REPORTER 3 | 4 | #include "bench_worker.h" 5 | 6 | namespace nocc { 7 | 8 | namespace oltp { 9 | 10 | class BenchReporter { // not thread safe 11 | public: 12 | virtual void init(const std::vector *workers); 13 | virtual void merge_data(char *,int id); 14 | virtual void report_data(uint64_t epoch,std::ofstream &log_file); 15 | virtual void collect_data(char *data,struct timespec &start_t); // the data is stored in *data 16 | virtual size_t data_len(); 17 | virtual void end(); 18 | 19 | private: 20 | double throughput; 21 | double aborts; 22 | double abort_ratio; 23 | 24 | std::vector prev_commits_; 25 | std::vector prev_aborts_; 26 | std::vector prev_abort_ratio_; 27 | 28 | const std::vector *workers_; 29 | 30 | // helper functions 31 | uint64_t calculate_commits(std::vector &prevs); 32 | uint64_t calculate_aborts(std::vector &prevs); 33 | double calculate_abort_ratio(std::vector &prevs); 34 | double calculate_execute_ratio(); 35 | 36 | std::vector throughputs; 37 | }; 38 | 39 | } // end namespace oltp 40 | 41 | } // end namespace nocc 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/framework/bench_runner.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_BENCH_RUNNER 2 | #define NOCC_BENCH_RUNNER 3 | 4 | #include "bench_worker.h" 5 | 6 | namespace nocc { 7 | 8 | namespace oltp { 9 | 10 | extern std::string config_file_name; 11 | 12 | std::vector parse_network(int num, std::string &hosts); 13 | 14 | /* Bench runner is used to bootstrap the application */ 15 | class BenchRunner { 16 | public: 17 | BenchRunner(std::string &config_file); 18 | BenchRunner() : barrier_a_(1), barrier_b_(1) {} 19 | std::vector net_def_; 20 | 21 | void run(); 22 | protected: 23 | /* parse the arguments the application requires, if necessary*/ 24 | void parse_config(std::string &config_file); 25 | 26 | /* Inputs: 27 | partition: partition id of the database 28 | store: the local store handler 29 | */ 30 | virtual std::vector make_loaders(int partition, MemDB *store = NULL) { 31 | return std::vector(); 32 | } 33 | 34 | /* return a set of workers to execute application logic */ 35 | virtual std::vector make_workers() = 0; 36 | 37 | /* below 2 functions are used to init data structure 38 | The first is called before any RDMA connections are made, which is used ti init 39 | data structures resides on RDMA registered area. 40 | The second is called after RDMA connections are made, which is used to init global 41 | data structure that needs RDMA for communication 42 | */ 43 | /* warm up the rdma buffer, can be replaced by the application */ 44 | virtual void warmup_buffer(char *buffer) { 45 | 46 | } 47 | 48 | virtual void bootstrap_with_rdma(RdmaCtrl *r) { 49 | 50 | } 51 | 52 | // cache the remote addresses of the key-value store 53 | virtual void populate_cache() { 54 | 55 | } 56 | 57 | virtual void init_store(MemDB* &store) { 58 | 59 | } 60 | 61 | virtual void init_backup_store(MemDB* &store) { 62 | 63 | } 64 | 65 | virtual void init_put() { 66 | 67 | } 68 | 69 | spin_barrier barrier_a_; 70 | spin_barrier barrier_b_; 71 | MemDB *store_; 72 | private: 73 | SpinLock rdma_init_lock_; 74 | int8_t init_worker_count_; /*used for worker to notify qp creation done*/ 75 | }; 76 | 77 | } // namespace oltp 78 | }; // namespace nocc 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /src/framework/config.cc: -------------------------------------------------------------------------------- 1 | // This file contains the parsing of the framework 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include "core/logging.h" 15 | 16 | namespace nocc { 17 | 18 | namespace oltp { 19 | 20 | // return a network with @num@ servers given a host file 21 | std::vector parse_network(int num, std::string &hosts) { 22 | 23 | std::vector network; 24 | std::map black_list; 25 | network.clear(); 26 | black_list.clear(); 27 | 28 | using boost::property_tree::ptree; 29 | using namespace boost; 30 | using namespace property_tree; 31 | ptree pt; 32 | 33 | try { 34 | read_xml(hosts, pt); 35 | } catch (const ptree_error &e) { 36 | fprintf(stderr,"Failed to parse host file!"); 37 | assert(false); 38 | } 39 | 40 | try { 41 | // parse the black list of hosts 42 | BOOST_FOREACH(ptree::value_type &v, 43 | pt.get_child("hosts.black")) 44 | { 45 | std::string s = v.second.data(); 46 | boost::algorithm::trim_right(s); 47 | boost::algorithm::trim_left(s); 48 | 49 | if(black_list.find(s) == black_list.end()) 50 | black_list.insert(std::make_pair(s,true)); 51 | } 52 | } 53 | catch(const ptree_error &e) { 54 | // pass, no black list provided 55 | } 56 | // parse the hosts file 57 | try { 58 | 59 | // parse hosts 60 | BOOST_FOREACH(ptree::value_type &v, 61 | pt.get_child("hosts.macs")) 62 | { 63 | std::string s = v.second.data(); 64 | boost::algorithm::trim_right(s); 65 | boost::algorithm::trim_left(s); 66 | if(black_list.find(s) != black_list.end()) { 67 | 68 | } else { 69 | if(network.size() + 1 > num) 70 | break; 71 | network.push_back(s); 72 | } 73 | } // end iterating hosts 74 | 75 | } catch (const ptree_error &e) { 76 | assert(false); 77 | } 78 | 79 | assert(network.size() == num); 80 | return network; 81 | } 82 | 83 | }; // namespace oltp 84 | 85 | }; // namespace nocc 86 | -------------------------------------------------------------------------------- /src/framework/config.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_FRAMEWORK_CONFIG_H_ 2 | #define NOCC_FRAMEWORK_CONFIG_H_ 3 | 4 | #define MASTER_EPOCH 15 // runtime of test 5 | 6 | // rdma related stuffs 7 | #define HUGE_PAGE 1 8 | #define USE_UD_MSG 1 9 | #define USE_TCP_MSG 0 10 | #define SINGLE_MR 0 11 | #define BUF_SIZE 10480 // RDMA buffer size registered, in a small setting 12 | //#define BUF_SIZE 512 13 | 14 | // rpc related stuffs 15 | //#define RPC_TIMEOUT_FLAG 16 | #define RPC_TIMEOUT_TIME 10000000 17 | //#define RPC_CHECKSUM 18 | #define MAX_INFLIGHT_REPLY 256 19 | //#define MAX_INFLIGHT_REQS 768 // for TPC-C, smallbank usage 20 | #define MAX_INFLIGHT_REQS 128 21 | //#define MAX_INFLIGHT_REQS 16 22 | 23 | 24 | // print statements 25 | #define LOG_RESULTS // log results to a file 26 | #define LISTENER_PRINT_PERF 1 // print the results to the screen 27 | #define PER_THREAD_LOG 0 // per thread detailed log 28 | #define POLL_CYCLES 0 // print the polling cycle 29 | #define CALCULATE_LAT 1 30 | #define LATENCY 0 // 1: global latency report; 0: workload specific report 31 | 32 | #if USE_TCP_MSG == 1 33 | #undef USE_UD_MSG 34 | #define USE_UD_MSG 0 35 | #endif 36 | 37 | 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/framework/req_buf_allocator.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_DB_MEM_ALLOCATOR 2 | #define NOCC_DB_MEM_ALLOCATOR 3 | 4 | #include "all.h" 5 | #include "config.h" 6 | #include "./config.h" 7 | #include "ralloc/ralloc.h" 8 | #include "core/rrpc.h" 9 | 10 | #include 11 | #include 12 | 13 | namespace nocc { 14 | 15 | namespace oltp { 16 | 17 | // RPC memory allocator 18 | class RPCMemAllocator { 19 | public: 20 | 21 | RPCMemAllocator() { 22 | RThreadLocalInit(); 23 | for(int i = 0;i < MAX_INFLIGHT_REQS;++i) { 24 | #if 1 25 | // FIXME: this is legacy code, so we hard coded this. shall be removed later 26 | buf_pools_[i] = (char *)Rmalloc(MAX_MSG_SIZE) + sizeof(uint64_t) + sizeof(uint64_t); 27 | // check for alignment 28 | assert( ((uint64_t)(buf_pools_[i])) % 8 == 0); 29 | if(buf_pools_[i] != NULL) 30 | memset(buf_pools_[i],0,MAX_MSG_SIZE); 31 | else 32 | assert(false); 33 | #endif 34 | } 35 | current_buf_slot_ = 0; 36 | } 37 | 38 | inline char * operator[] (int id) const{ 39 | return buf_pools_[id]; 40 | } 41 | 42 | inline void rollback_buf() { 43 | current_buf_slot_ = current_buf_slot_ - 1; 44 | } 45 | 46 | inline char * get_req_buf() { 47 | 48 | uint16_t buf_idx = (current_buf_slot_++) % MAX_INFLIGHT_REQS; 49 | assert(buf_idx >= 0 && buf_idx < MAX_INFLIGHT_REQS); 50 | // fetch the buf 51 | char *res = buf_pools_[buf_idx]; 52 | return res; 53 | } 54 | 55 | inline char *post_buf(int num = 1) { 56 | current_freed_slot_ += num; 57 | } 58 | 59 | private: 60 | char *buf_pools_[MAX_INFLIGHT_REQS]; 61 | uint64_t current_buf_slot_; 62 | uint64_t current_freed_slot_; 63 | }; 64 | } // namespace db 65 | }; // namespace nocc 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /src/framework/tpc_msg.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SJTU-IPADS/drtmh/627e2e68706077357737461b7177c326f74a95d0/src/framework/tpc_msg.h -------------------------------------------------------------------------------- /src/framework/utils/amd64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _AMD64_H_ 26 | #define _AMD64_H_ 27 | 28 | #include "macros.h" 29 | #include 30 | 31 | inline ALWAYS_INLINE void 32 | nop_pause() 33 | { 34 | __asm volatile("pause" : :); 35 | } 36 | 37 | #endif /* _AMD64_H_ */ 38 | -------------------------------------------------------------------------------- /src/framework/utils/ndb_type_traits.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _NDB_TYPE_TRAITS_H_ 26 | #define _NDB_TYPE_TRAITS_H_ 27 | 28 | #include 29 | 30 | namespace private_ { 31 | 32 | // std::is_trivially_destructible not supported in g++-4.7, so we 33 | // do some hacky [conservative] variant of it 34 | template 35 | struct is_trivially_destructible { 36 | static const bool value = std::is_scalar::value; 37 | }; 38 | 39 | template 40 | struct is_trivially_destructible> { 41 | static const bool value = 42 | is_trivially_destructible::value && 43 | is_trivially_destructible::value; 44 | }; 45 | 46 | // XXX: same for now 47 | template 48 | struct is_trivially_copyable : public is_trivially_destructible {}; 49 | 50 | // user types should add their own specializations 51 | 52 | template 53 | struct typeutil { typedef const T & func_param_type; }; 54 | 55 | template 56 | struct primitive_typeutil { typedef T func_param_type; }; 57 | 58 | // specialize typeutil for int types to use primitive_typeutil 59 | 60 | #define SPECIALIZE_PRIM_TYPEUTIL(tpe) \ 61 | template <> struct typeutil< tpe > : public primitive_typeutil< tpe > {}; 62 | 63 | SPECIALIZE_PRIM_TYPEUTIL(bool) 64 | SPECIALIZE_PRIM_TYPEUTIL(int8_t) 65 | SPECIALIZE_PRIM_TYPEUTIL(uint8_t) 66 | SPECIALIZE_PRIM_TYPEUTIL(int16_t) 67 | SPECIALIZE_PRIM_TYPEUTIL(uint16_t) 68 | SPECIALIZE_PRIM_TYPEUTIL(int32_t) 69 | SPECIALIZE_PRIM_TYPEUTIL(uint32_t) 70 | SPECIALIZE_PRIM_TYPEUTIL(int64_t) 71 | SPECIALIZE_PRIM_TYPEUTIL(uint64_t) 72 | } 73 | 74 | #endif /* _NDB_TYPE_TRAITS_H_ */ 75 | -------------------------------------------------------------------------------- /src/framework/utils/spinbarrier.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #ifndef _SPINBARRIER_H_ 26 | #define _SPINBARRIER_H_ 27 | 28 | #include "amd64.h" 29 | #include "macros.h" 30 | #include "util.h" 31 | 32 | /** 33 | * Barrier implemented by spinning 34 | */ 35 | 36 | class spin_barrier { 37 | public: 38 | spin_barrier(size_t n) 39 | : n(n) 40 | { 41 | ALWAYS_ASSERT(n > 0); 42 | } 43 | #if 0 44 | spin_barrier(const spin_barrier &) = delete; 45 | spin_barrier(spin_barrier &&) = delete; 46 | spin_barrier &operator=(const spin_barrier &) = delete; 47 | #endif 48 | ~spin_barrier() 49 | { 50 | ALWAYS_ASSERT(n == 0); 51 | } 52 | 53 | void 54 | count_down() 55 | { 56 | // written like this (instead of using __sync_fetch_and_add()) 57 | // so we can have assertions 58 | for (;;) { 59 | size_t copy = n; 60 | ALWAYS_ASSERT(copy > 0); 61 | if (__sync_bool_compare_and_swap(&n, copy, copy - 1)) 62 | return; 63 | } 64 | } 65 | 66 | void 67 | wait_for() 68 | { 69 | while (n > 0) 70 | nop_pause(); 71 | } 72 | 73 | public: 74 | volatile size_t n; 75 | }; 76 | 77 | #endif /* _SPINBARRIER_H_ */ 78 | -------------------------------------------------------------------------------- /src/framework/utils/varint.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #include 26 | 27 | #include "varint.h" 28 | #include "macros.h" 29 | #include "util.h" 30 | 31 | using namespace std; 32 | using namespace nocc::util; 33 | 34 | static void 35 | do_test(uint32_t v) 36 | { 37 | uint8_t buf[5]; 38 | uint8_t *p = &buf[0]; 39 | p = write_uvint32(p, v); 40 | ALWAYS_ASSERT(size_t(p - &buf[0]) == size_uvint32(v)); 41 | 42 | const uint8_t *p0 = &buf[0]; 43 | uint32_t v0 = 0; 44 | p0 = read_uvint32(p0, &v0); 45 | ALWAYS_ASSERT(v == v0); 46 | ALWAYS_ASSERT(p == p0); 47 | } 48 | 49 | void 50 | varint::Test() 51 | { 52 | fast_random r(2043859); 53 | for (int i = 0; i < 1000; i++) 54 | do_test(r.next_u32()); 55 | cerr << "varint tests passed" << endl; 56 | } 57 | -------------------------------------------------------------------------------- /src/memstore/cluster_chaining_remote_op.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "framework/bench_worker.h" 4 | #include "cluster_chaining.hpp" 5 | 6 | namespace nocc { 7 | 8 | extern __thread oltp::BenchWorker* worker; 9 | 10 | namespace drtm { 11 | 12 | template 13 | void ClusterHash::fetch_node(RCQP *qp,uint64_t off,char *buf,int size) { 14 | 15 | qp->post_send(IBV_WR_RDMA_READ,buf,size,off,IBV_SEND_SIGNALED); 16 | ibv_wc wc = {}; 17 | auto res = qp->poll_till_completion(wc,::rdmaio::no_timeout); 18 | ASSERT(res == SUCC) << "res " << (int)res; 19 | } 20 | 21 | template 22 | void ClusterHash::fetch_node(RCQP *qp,uint64_t off,char *buf,int size, 23 | nocc::oltp::RScheduler *sched,yield_func_t &yield) { 24 | int flag = IBV_SEND_SIGNALED; 25 | //if(size < 64) flag |= IBV_SEND_INLINE; 26 | 27 | sched->post_send(qp,worker->cor_id(), 28 | IBV_WR_RDMA_READ,buf,size,off,flag); 29 | worker->indirect_yield(yield); 30 | } 31 | 32 | }; // namespace drtm 33 | 34 | }; // namespace nocc 35 | -------------------------------------------------------------------------------- /src/memstore/memdb.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file contains a db class built from memstore 3 | */ 4 | 5 | 6 | #ifndef MEM_DB 7 | #define MEM_DB 8 | 9 | #include 10 | 11 | #include "memstore.h" 12 | 13 | #include "rdma_ctrl.hpp" 14 | 15 | /* For main table */ 16 | #include "memstore_bplustree.h" 17 | /* For string index */ 18 | #include "memstore_uint64bplustree.h" 19 | #include "memstore_hash.h" 20 | 21 | #define MAX_TABLE_SUPPORTED 32 22 | 23 | enum TABLE_CLASS { 24 | TAB_BTREE, 25 | TAB_BTREE1, 26 | // String b+ tree 27 | TAB_SBTREE, 28 | TAB_HASH 29 | }; 30 | 31 | class MemDB { 32 | 33 | public: 34 | 35 | struct TableSchema { 36 | 37 | bool versioned; 38 | int klen; 39 | 40 | // We didn't support varible length of value 41 | int vlen; 42 | 43 | // The size of meta data 44 | int meta_len; 45 | 46 | // Total legnth, = round( meta_len + vlen ) up to cache_line size 47 | int total_len; 48 | 49 | // Which class of underlying store is used 50 | TABLE_CLASS c; 51 | }; 52 | 53 | TableSchema _schemas[MAX_TABLE_SUPPORTED]; // table's meta data infor 54 | Memstore * stores_[MAX_TABLE_SUPPORTED]; // primary tables 55 | MemstoreUint64BPlusTree *_indexs[MAX_TABLE_SUPPORTED]; // secondary indexes 56 | char *store_buffer_; // RDMA buffer used to store the memory store 57 | 58 | 59 | /* 60 | Do not give the same store_buffer to different MemDB instances! 61 | */ 62 | MemDB(char *s_buffer = NULL): store_buffer_(s_buffer) { } 63 | 64 | // expected_num: the number of records in table 65 | void AddSchema(int tableid, TABLE_CLASS c, int klen,int vlen,int meta_len,int expected_num = 1024,bool need_cache = true); 66 | 67 | /** 68 | Important! 69 | If the remote accesses are enabled, then each node shall ensure the order 70 | of addschema is the same. 71 | For example, 72 | if we have 2 tables, A,B. 73 | Node 0 shall call AddSchema(A),AddSchema(B). 74 | Node 1 shall call AddSchema(A),AddSchema(B), the same order. 75 | 76 | Further, the store_buffer_ offset shall be the same. 77 | We do this for convenience. 78 | Otherwise nodes need sync about the offsets of the RDMA region in the table. 79 | */ 80 | void EnableRemoteAccess(int tableid,char *s_ptr); 81 | 82 | void AddSecondIndex(int index_id,TABLE_CLASS c, int klen); 83 | uint64_t *Get(int tableid,uint64_t key); 84 | uint64_t *GetIndex(int tableid,uint64_t key); 85 | MemNode *Put(int tableid,uint64_t key,uint64_t *value,int len = 0); 86 | void PutIndex(int indexid,uint64_t key,uint64_t *value); 87 | 88 | uint64_t store_size_ = 0; // store size alloced on the RDMA area 89 | }; 90 | 91 | #endif 92 | -------------------------------------------------------------------------------- /src/memstore/memstore.cc: -------------------------------------------------------------------------------- 1 | #include "memstore.h" 2 | 3 | #if RECORD_STALE 4 | std_time_t_ MemNode::init_time; 5 | #endif 6 | -------------------------------------------------------------------------------- /src/memstore/memstore_hash.cc: -------------------------------------------------------------------------------- 1 | #include "memstore/memstore_hash.h" 2 | 3 | namespace leveldb { 4 | __thread MemstoreHashTable::HashNode *MemstoreHashTable::dummynode_ = NULL; 5 | } 6 | -------------------------------------------------------------------------------- /src/memstore/rdma_hash.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "tx_config.h" 4 | 5 | #include "cluster_chaining.hpp" 6 | #include "cluster_chaining_remote_op.hpp" 7 | #include "memstore.h" 8 | 9 | #include "core/logging.h" 10 | #include "util/util.h" 11 | 12 | #include "simple_cache.hpp" 13 | 14 | #include 15 | 16 | extern size_t total_partition; 17 | 18 | namespace nocc { 19 | #define DRTM_CLUSTER_NUM 4 20 | #define CACHE_BUCKET_NUM 16 21 | 22 | // a wrapper over cluster_chaining which implements MemStore 23 | typedef drtm::ClusterHash loc_cache_t; 24 | class RHash : public Memstore, public drtm::ClusterHash { 25 | public: 26 | RHash(int expected_data, char *ptr,bool need_cache) 27 | : drtm::ClusterHash (expected_data, ptr), 28 | /** 29 | * FIXME! use a simple cache 30 | */ 31 | cache(need_cache?(300 * 1024 * 1024) : 0) 32 | { 33 | #if RDMA_CACHE 34 | if(need_cache) { 35 | //uint64_t expected_cached_num = ceil((double)(expected_data) / CACHE_BUCKET_NUM); 36 | //loc_cache_ 37 | //= new drtm::ClusterHash(0.8 * expected_cached_num * total_partition); 38 | //LOG(2) << "Cache size: " << get_memory_size_g(loc_cache_->size()) << "G"; 39 | } 40 | #endif 41 | } 42 | 43 | MemNode *_GetWithInsert(uint64_t key,char *val) { 44 | MemNode *node = get_with_insert(key); 45 | node->off = base_off_ + ((char *)node - data_ptr_); 46 | if(unlikely(node->value == NULL)) 47 | node->value = (uint64_t *)val; 48 | return node; 49 | } 50 | 51 | MemNode *Get(uint64_t key) { 52 | return get(key); 53 | } 54 | 55 | MemNode *Put(uint64_t key,uint64_t *val) { 56 | return _GetWithInsert(key,(char *)val); 57 | } 58 | 59 | uint64_t RemoteTraverse(uint64_t key,rdmaio::RCQP *qp, 60 | nocc::oltp::RScheduler *sched, yield_func_t &yield,char *val) { 61 | #if RDMA_CACHE 62 | //auto res = *(loc_cache_->get(key)); 63 | auto res = cache.get(key); 64 | return res; 65 | #else 66 | return remote_get(key,qp,sched,yield,val); 67 | #endif 68 | } 69 | 70 | uint64_t RemoteTraverse(uint64_t key,rdmaio::RCQP *qp, 71 | char *val) { 72 | auto res = remote_get(key,qp,val); 73 | #if RDMA_CACHE 74 | //auto ptr = loc_cache_->get_with_insert(key); 75 | MemNode *node = (MemNode *)val; 76 | //*ptr = node->off; // cache the real data offset 77 | cache.put(key,node->off); 78 | return res; 79 | #endif 80 | } 81 | private: 82 | SimpleCache cache; 83 | }; 84 | }; // namespace nocc 85 | -------------------------------------------------------------------------------- /src/memstore/simple_cache.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "util/util.h" 4 | 5 | namespace nocc { 6 | 7 | class SimpleCache { 8 | public: 9 | SimpleCache(int size) : size_(size) 10 | { 11 | if(size != 0) { 12 | content_ = (uint64_t *)malloc_huge_pages(size,HUGE_PAGE_SZ,true); 13 | assert(content_ != NULL); 14 | } 15 | } 16 | 17 | void put(uint64_t key,uint64_t off) { 18 | assert(key < size_ / sizeof(uint64_t)); 19 | content_[key] = off; 20 | } 21 | 22 | inline uint64_t get(uint64_t key) { 23 | return content_[key]; 24 | } 25 | private: 26 | uint64_t *content_ = NULL; 27 | int size_; 28 | }; 29 | 30 | }; // namespace nocc 31 | -------------------------------------------------------------------------------- /src/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_posix.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /src/port/atomic.h: -------------------------------------------------------------------------------- 1 | #ifndef LEVELDB_ATOMIC_H 2 | #define LEVELDB_ATOMIC_H 3 | 4 | #include 5 | 6 | #define xglue(a, b) a ## b 7 | // If a/b is macro, it will expand first, then pass to xglue 8 | #define glue(a, b) xglue(a, b) 9 | 10 | #define xstr(s) # s 11 | #define sstr(s) xstr(s) 12 | 13 | #define __inline__ inline __attribute__((always_inline)) 14 | 15 | /* Compile read-write barrier */ 16 | #define barrier() asm volatile("": : :"memory") 17 | 18 | #define rmb() asm volatile("lfence":::"memory") 19 | #define wmb() asm volatile("sfence":::"memory") 20 | #define mb() asm volatile("mfence":::"memory") 21 | 22 | 23 | 24 | /* Pause instruction to prevent excess processor bus usage */ 25 | #define cpu_relax() asm volatile("pause\n": : :"memory") 26 | 27 | #define CACHE_LINE_SIZE 64 28 | inline void prefetch(const void *ptr) { 29 | typedef struct { char x[CACHE_LINE_SIZE]; } cacheline_t; 30 | asm volatile("prefetcht0 %0" : : "m" (*(const cacheline_t *)ptr)); 31 | } 32 | 33 | 34 | #define LOCK_PREFIX "lock; " 35 | 36 | // Is this the correct way to detect 64 system? 37 | #if (__LP64__== 1) 38 | /* Compares the value in rdx:rax to value in memp, if equal load rcx:rbx into 39 | * memp, else load value into rdx:rax. 40 | * Return 1 on success, 0 otherwise. */ 41 | static __inline__ uint8_t atomic_cmpxchg16b(uint64_t *memp, 42 | uint64_t old0, uint64_t old1, 43 | uint64_t new0, uint64_t new1) { 44 | uint8_t z; 45 | asm volatile ( 46 | "lock; cmpxchg16b %3\n\t" 47 | "setz %2\n\t" 48 | : "+a" (old0), "+d" (old1), "=r" (z), "+m" (*memp) 49 | : "b" (new0), "c" (new1) 50 | : "memory", "cc" ); 51 | return z; 52 | } 53 | #endif 54 | 55 | static __inline__ uint8_t atomic_cmpxchg8b(uint32_t *memp, 56 | uint32_t old0, uint32_t old1, 57 | uint32_t new0, uint32_t new1) { 58 | uint8_t z; 59 | asm volatile ( 60 | "lock; cmpxchg8b %3\n\t" 61 | "setz %2\n\t" 62 | : "+a" (old0), "+d" (old1), "=r" (z), "+m" (*memp) 63 | : "b" (new0), "c" (new1) 64 | : "memory", "cc" ); 65 | return z; 66 | } 67 | 68 | #define DATA_BITS 8 69 | #include "port/atomic-template.h" 70 | 71 | #define DATA_BITS 16 72 | #include "port/atomic-template.h" 73 | 74 | #define DATA_BITS 32 75 | #include "port/atomic-template.h" 76 | 77 | #define DATA_BITS 64 78 | #include "port/atomic-template.h" 79 | 80 | 81 | #endif /* _ATOMIC_H */ 82 | 83 | -------------------------------------------------------------------------------- /src/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_posix.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /src/port/port_posix.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "port/port_posix.h" 6 | 7 | #include 8 | #include 9 | #include 10 | //#include "util/logging.h" 11 | 12 | namespace leveldb { 13 | namespace port { 14 | 15 | static void PthreadCall(const char* label, int result) { 16 | if (result != 0) { 17 | fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); 18 | abort(); 19 | } 20 | } 21 | 22 | Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); } 23 | 24 | Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); } 25 | 26 | void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); } 27 | 28 | void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } 29 | 30 | SpinLock::SpinLock() { PthreadCall("init spinlock", pthread_spin_init(&lock_, PTHREAD_PROCESS_PRIVATE)); } 31 | 32 | SpinLock::~SpinLock() { PthreadCall("destroy spinlock", pthread_spin_destroy(&lock_)); } 33 | 34 | void SpinLock::Lock() { PthreadCall("lock", pthread_spin_lock(&lock_)); } 35 | 36 | void SpinLock::Unlock() { PthreadCall("unlock", pthread_spin_unlock(&lock_)); } 37 | 38 | 39 | 40 | CondVar::CondVar(Mutex* mu) 41 | : mu_(mu) { 42 | PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); 43 | } 44 | 45 | CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); } 46 | 47 | void CondVar::Wait() { 48 | PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); 49 | } 50 | 51 | void CondVar::Signal() { 52 | PthreadCall("signal", pthread_cond_signal(&cv_)); 53 | } 54 | 55 | void CondVar::SignalAll() { 56 | PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); 57 | } 58 | 59 | void InitOnce(OnceType* once, void (*initializer)()) { 60 | PthreadCall("once", pthread_once(once, initializer)); 61 | } 62 | 63 | } // namespace port 64 | } // namespace leveldb 65 | -------------------------------------------------------------------------------- /src/port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H 6 | 7 | // Some environments provide custom macros to aid in static thread-safety 8 | // analysis. Provide empty definitions of such macros unless they are already 9 | // defined. 10 | 11 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 12 | #define EXCLUSIVE_LOCKS_REQUIRED(...) 13 | #endif 14 | 15 | #ifndef SHARED_LOCKS_REQUIRED 16 | #define SHARED_LOCKS_REQUIRED(...) 17 | #endif 18 | 19 | #ifndef LOCKS_EXCLUDED 20 | #define LOCKS_EXCLUDED(...) 21 | #endif 22 | 23 | #ifndef LOCK_RETURNED 24 | #define LOCK_RETURNED(x) 25 | #endif 26 | 27 | #ifndef LOCKABLE 28 | #define LOCKABLE 29 | #endif 30 | 31 | #ifndef SCOPED_LOCKABLE 32 | #define SCOPED_LOCKABLE 33 | #endif 34 | 35 | #ifndef EXCLUSIVE_LOCK_FUNCTION 36 | #define EXCLUSIVE_LOCK_FUNCTION(...) 37 | #endif 38 | 39 | #ifndef SHARED_LOCK_FUNCTION 40 | #define SHARED_LOCK_FUNCTION(...) 41 | #endif 42 | 43 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 44 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) 45 | #endif 46 | 47 | #ifndef SHARED_TRYLOCK_FUNCTION 48 | #define SHARED_TRYLOCK_FUNCTION(...) 49 | #endif 50 | 51 | #ifndef UNLOCK_FUNCTION 52 | #define UNLOCK_FUNCTION(...) 53 | #endif 54 | 55 | #ifndef NO_THREAD_SAFETY_ANALYSIS 56 | #define NO_THREAD_SAFETY_ANALYSIS 57 | #endif 58 | 59 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H 60 | -------------------------------------------------------------------------------- /src/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /src/rocc_config.h.in: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // whether to use RDMA as a communication primitive 4 | #cmakedefine USE_RDMA @USE_RDMA@ 5 | #ifndef USE_RDMA 6 | #define USE_RDMA 1 // by default use RDMA 7 | #endif 8 | 9 | #cmakedefine ROCC_RBUF_SIZE_M @ROCC_RBUF_SIZE_M@ // rdma registered buf size 10 | #ifndef ROCC_RBUF_SIZE_M 11 | #define ROCC_RBUF_SIZE_M 10240 // default size: 10G 12 | #endif 13 | 14 | static_assert( 15 | ROCC_RBUF_SIZE_M <= 20240 && ROCC_RBUF_SIZE_M > 0, 16 | "RDMA registered buf size too large or too small!"); 17 | -------------------------------------------------------------------------------- /src/rtx/global_vars.cc: -------------------------------------------------------------------------------- 1 | #include "global_vars.h" 2 | 3 | namespace nocc { 4 | namespace rtx { 5 | 6 | SymmetricView *global_view = NULL; 7 | 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/rtx/global_vars.h: -------------------------------------------------------------------------------- 1 | #ifndef GLOBAL_VARS_H_ 2 | #define GLOBAL_VARS_H_ 3 | 4 | #include "view.h" 5 | 6 | namespace nocc { 7 | 8 | namespace rtx { 9 | 10 | /** 11 | * New meta data for each record 12 | */ 13 | struct RdmaValHeader { 14 | uint64_t lock; 15 | uint64_t seq; 16 | }; 17 | 18 | 19 | extern SymmetricView *global_view; 20 | 21 | } // namespace rtx 22 | } // namespace nocc 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/rtx/log_cleaner.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "core/rrpc.h" 4 | #include "log_store_manager.hpp" 5 | 6 | namespace nocc { 7 | 8 | namespace rtx { 9 | 10 | using namespace oltp; 11 | 12 | // The cleaner leverages log_store_manager to access backup stores 13 | class LogCleaner : public LogStoreManager { 14 | public: 15 | explicit LogCleaner(int num) : LogStoreManager(num) { 16 | 17 | } 18 | 19 | // the register_callback is used to register *clean_log* 20 | virtual void register_callback(int id,RRpc *rpc) = 0; 21 | 22 | virtual void clean_log(int nid,int cid,char *log,void *) = 0; 23 | }; 24 | 25 | }; // namespace rtx 26 | }; // namespace nocc 27 | -------------------------------------------------------------------------------- /src/rtx/log_store_manager.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "memstore/cluster_chaining.hpp" 4 | #include "memstore/memdb.h" 5 | 6 | namespace nocc { 7 | namespace rtx { 8 | 9 | #define RTX_BACKUP_MAX 4 // maxinum backups stored at this server 10 | 11 | class LogStoreManager { 12 | typedef drtm::ClusterHash store_map_t; 13 | public: 14 | explicit LogStoreManager(int expected_backup_num = RTX_BACKUP_MAX) 15 | :backup_stores_(expected_backup_num) 16 | { 17 | } 18 | 19 | // add a particular store to a specific DB 20 | void add_backup_store(int id,MemDB *db) { 21 | assert(backup_stores_.get(id) == NULL); 22 | MemDB **new_val_p = backup_stores_.get_with_insert(id); 23 | *new_val_p = db; 24 | } 25 | 26 | MemDB *get_backed_store(int id) { 27 | MemDB **ptr = backup_stores_.get(id); 28 | if(likely(ptr != nullptr)) 29 | return *ptr; 30 | else { 31 | return nullptr; 32 | } 33 | } 34 | 35 | protected: 36 | store_map_t backup_stores_; 37 | }; 38 | 39 | }; // namespace rtx 40 | 41 | }; 42 | 43 | -------------------------------------------------------------------------------- /src/rtx/logger.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "core/rrpc.h" 5 | 6 | #include "tx_config.h" 7 | 8 | #include "tx_operator.hpp" 9 | #include "log_mem_manager.hpp" 10 | #include "default_log_cleaner_impl.hpp" 11 | 12 | namespace nocc { 13 | 14 | #define MAX_BACKUP_NUM 2 15 | 16 | namespace rtx { 17 | 18 | class RdmaChecker; 19 | class OCC; 20 | 21 | // Abstract logger class 22 | class Logger { 23 | public: 24 | /* 25 | * rpc: rpc_handler 26 | * expected_store_num: backup stores at this node 27 | * local_p: log memory buffer pointer 28 | * ms: total mac log expected 29 | * ts: total thread at each mac 30 | * size: total log area of the mac 31 | * entry_size: the max size of each log 32 | */ 33 | Logger(RRpc *rpc,int ack_rpc_id,uint64_t base_off, 34 | int expected_store_num,char *local_p,int ms,int ts,int size,int entry_size = RTX_LOG_ENTRY_SIZE): 35 | mem_(local_p,ms,ts,size,entry_size,base_off), 36 | cleaner_(expected_store_num,rpc), 37 | rpc_handler_(rpc), 38 | ack_rpc_id_(ack_rpc_id), 39 | reply_buf_((char *)malloc(ms * sizeof(uint64_t))) 40 | 41 | { 42 | cleaner_.register_callback(ack_rpc_id_,rpc_handler_); 43 | } 44 | 45 | ~Logger() { 46 | free(reply_buf_); 47 | } 48 | 49 | // log remote remote server defined in mac_set 50 | virtual void log_remote(BatchOpCtrlBlock &ctrl,int cor_id) = 0; 51 | 52 | virtual void check_remote(BatchOpCtrlBlock &ctrl,int cor_id,yield_func_t &yield) { 53 | } 54 | 55 | // ack log at remote servers 56 | // a helper function to broadcast this message to others 57 | virtual void log_ack(BatchOpCtrlBlock &ctrl,int cor_id) { 58 | ctrl.send_batch_op(rpc_handler_,cor_id,ack_rpc_id_,PA); 59 | // a yield call is necessary after this 60 | } 61 | 62 | inline void add_backup_store(int id,MemDB *backup_store) { 63 | cleaner_.add_backup_store(id,backup_store); 64 | } 65 | 66 | public: 67 | LogMemManager mem_; 68 | DefaultLogCleaner cleaner_; 69 | RRpc *rpc_handler_ = nullptr; 70 | char *reply_buf_ = nullptr; 71 | 72 | private: 73 | const int ack_rpc_id_; 74 | 75 | friend RdmaChecker; 76 | friend OCC; 77 | 78 | DISABLE_COPY_AND_ASSIGN(Logger); 79 | }; // logger 80 | 81 | }; // namespace rtx 82 | 83 | }; // namespace nocc 84 | 85 | // RTX provides two logger implementation 86 | #include "rdma_logger_impl.hpp" 87 | #include "rpc_logger_impl.hpp" 88 | -------------------------------------------------------------------------------- /src/rtx/logger_test.cxx: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "gtest/gtest.h" 4 | #include "log_mem_manager.hpp" 5 | #include "log_store_manager.hpp" 6 | 7 | #include "core/utils/util.h" 8 | 9 | using namespace nocc::rtx; 10 | using namespace nocc::util; 11 | 12 | int total_mac = 12; 13 | int total_thread = 12; 14 | 15 | // First is test hierachy, second is the specific test namespace 16 | TEST(logger_test, log_mem_offset_case) 17 | { 18 | 19 | char *test_ptr = (char *)malloc(1024 * 1024 * 12); 20 | auto test_manager = LogMemManager(test_ptr,total_mac,total_thread,4096); 21 | LogStoreManager manager(2); 22 | // test thread id 23 | for(uint tid = 0;tid < total_thread;++tid) { 24 | for(uint fmid = 0;fmid < total_mac;++fmid) { 25 | for(uint ttid = 0;ttid < total_mac;++ttid) { 26 | EXPECT_EQ(test_manager.get_remote_log_offset(fmid,tid,ttid,0) + test_ptr, 27 | test_manager.get_local_log_ptr(fmid,tid)); 28 | } 29 | } 30 | } 31 | // 32 | free(test_ptr); 33 | } 34 | 35 | TEST(logger_test,send_recv_case) { 36 | char *test_buffer = (char *)malloc(1024 * 1024 * 12); 37 | 38 | fast_random rand(73); 39 | 40 | // 1024 test case 41 | for(uint tid = 0;tid < total_thread;++tid) { 42 | 43 | auto test_manager = LogMemManager(test_buffer,total_mac,total_thread,4096); 44 | 45 | for(uint i = 0;i < 1024;++i) { 46 | int send_mac = rand.next() % total_mac; 47 | int to_mac = rand.next() % total_mac; 48 | 49 | int send_tid = tid; 50 | int send_value = rand.next(); 51 | 52 | int log_size = rand.next() % 128; 53 | log_size = log_size + 64 - log_size % 64; // aligment 54 | 55 | char *send_ptr = test_manager.get_remote_log_offset(send_mac,tid,send_mac,log_size) + test_buffer; 56 | *(uint64_t *)send_ptr = send_value; 57 | 58 | const char *recv_ptr = test_manager.get_next_log(send_mac,tid,log_size); 59 | EXPECT_EQ(recv_ptr,send_ptr); 60 | ASSERT_EQ((*(uint64_t *)recv_ptr),send_value); 61 | } 62 | } 63 | delete test_buffer; 64 | } 65 | -------------------------------------------------------------------------------- /src/rtx/msg_format.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // This header contains common msg definiation used in RTX 4 | 5 | namespace nocc { 6 | 7 | namespace rtx { 8 | 9 | enum req_type_t { 10 | RTX_REQ_READ = 0, 11 | RTX_REQ_READ_LOCK, 12 | RTX_REQ_INSERT 13 | }; 14 | 15 | // header of the requests message 16 | struct RTXRequestHeader { 17 | uint8_t cor_id; 18 | int8_t num; // num TX items in the message 19 | uint64_t padding; // a padding is left for user-defined entries 20 | } __attribute__ ((aligned (8))); 21 | 22 | struct RTXReadItem { 23 | req_type_t type; 24 | uint8_t pid; 25 | uint64_t key; 26 | uint8_t tableid; 27 | uint16_t idx; 28 | uint16_t len; 29 | inline RTXReadItem(req_type_t type,uint8_t pid,uint64_t key,uint8_t tableid,uint16_t len,uint8_t idx) 30 | :type(type),pid(pid),key(key),tableid(tableid),len(len),idx(idx) 31 | { 32 | } 33 | } __attribute__ ((aligned (8))); 34 | 35 | // entries of the request message 36 | struct RTXRemoteWriteItem { 37 | uint8_t pid; 38 | union { 39 | MemNode *node; 40 | uint64_t key; 41 | }; 42 | uint16_t payload; 43 | uint8_t tableid; 44 | } __attribute__ ((aligned (8))); 45 | 46 | struct RtxWriteItem { 47 | uint8_t pid; 48 | uint8_t tableid; 49 | uint64_t key; 50 | uint16_t len; 51 | RtxWriteItem(int pid,int tableid,uint64_t key,int len) : 52 | pid(pid),tableid(tableid),key(key),len(len) { 53 | 54 | } 55 | } __attribute__ ((aligned (8))); 56 | 57 | 58 | }; // namespace rtx 59 | }; // namespace nocc 60 | -------------------------------------------------------------------------------- /src/rtx/occ_internal_structure.h: -------------------------------------------------------------------------------- 1 | // class RtxOCC 2 | protected: 3 | // a simple ReadSetItem for buffering read/write records 4 | struct ReadSetItem { 5 | uint8_t tableid; 6 | uint8_t len; 7 | uint64_t key; 8 | union { 9 | MemNode *node; 10 | uint64_t off; 11 | }; 12 | char *data_ptr; 13 | uint64_t seq; // buffered seq 14 | uint8_t pid; 15 | 16 | inline ReadSetItem(int tableid,uint64_t key,MemNode *node,char *data_ptr,uint64_t seq,int len,int pid): 17 | tableid(tableid), 18 | key(key), 19 | node(node), 20 | data_ptr(data_ptr), 21 | seq(seq), 22 | len(len), 23 | pid(pid) 24 | { 25 | } 26 | 27 | inline ReadSetItem(const ReadSetItem &item) : 28 | tableid(item.tableid), 29 | key(item.key), 30 | node(item.node), 31 | data_ptr(item.data_ptr), 32 | seq(item.seq), 33 | len(item.len), 34 | pid(item.pid) 35 | { 36 | } 37 | 38 | } __attribute__ ((aligned (8))); 39 | 40 | struct RtxLockItem { 41 | uint8_t pid; 42 | uint8_t tableid; 43 | uint64_t key; 44 | uint64_t seq; 45 | 46 | RtxLockItem(uint8_t pid,uint8_t tableid,uint64_t key,uint64_t seq) 47 | :pid(pid),tableid(tableid),key(key),seq(seq) 48 | { 49 | } 50 | } __attribute__ ((aligned (8))); 51 | 52 | struct CommitItem { 53 | uint32_t len; 54 | uint32_t tableid; 55 | uint64_t key; 56 | } __attribute__ ((aligned (8))); 57 | 58 | struct ReadItem { 59 | uint32_t pid; 60 | uint32_t tableid; 61 | uint64_t key; 62 | } __attribute__ ((aligned (8))); 63 | 64 | 65 | struct ReplyHeader { 66 | uint16_t num; 67 | }; 68 | 69 | struct OCCResponse { 70 | uint16_t payload; 71 | uint16_t idx; 72 | uint64_t seq; 73 | }; 74 | 75 | 76 | // 77 | /* 16 bit mac | 6 bit thread | 10 bit cor_id */ 78 | #define ENCODE_LOCK_CONTENT(mac,tid,cor_id) ( ((mac) << 16) | ((tid) << 10) | (cor_id) ) 79 | #define DECODE_LOCK_MAC(lock) ( (lock) >> 16) 80 | #define DECODE_LOCK_TID(lock) (((lock) & 0xffff ) >> 10) 81 | #define DECODE_LOCK_CID(lock) ( (lock) & 0x3f) 82 | 83 | 84 | #define LOCK_SUCCESS_MAGIC 73 85 | #define LOCK_FAIL_MAGIC 12 86 | -------------------------------------------------------------------------------- /src/rtx/occ_iterator.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace nocc { 4 | 5 | namespace rtx { 6 | 7 | class RTXIterator { 8 | public: 9 | RTXIterator(OCC *tx,int tableid,bool sec = false): 10 | tx_(tx){ 11 | if(sec) { 12 | iter_ = (tx_->db_->_indexs[tableid])->GetIterator(); 13 | } else { 14 | iter_ = (tx_->db_->stores_[tableid])->GetIterator(); 15 | } 16 | } 17 | 18 | bool valid() { 19 | return cur_ != NULL && cur_->value != NULL; 20 | } 21 | 22 | uint64_t key() { 23 | return iter_->Key(); 24 | } 25 | 26 | char *value() { 27 | if(valid()) 28 | return (char *)val_; 29 | return NULL; 30 | } 31 | 32 | MemNode *node() { 33 | return (MemNode *)cur_; 34 | } 35 | 36 | void next() { 37 | bool r = iter_->Next(); 38 | 39 | while(iter_->Valid()) { 40 | cur_ = iter_->CurNode(); 41 | { // RTM scope 42 | RTMScope rtm(NULL); 43 | val_ = cur_->value; 44 | 45 | if(prev_link_ != iter_->GetLink()) { 46 | prev_link_ = iter_->GetLink(); 47 | } 48 | 49 | if(val_ != NULL) { 50 | return; 51 | } 52 | } 53 | iter_->Next(); 54 | } 55 | cur_ = NULL; 56 | } 57 | 58 | void prev() { 59 | assert(false); 60 | } 61 | 62 | void seek(uint64_t key) { 63 | 64 | iter_->Seek(key); 65 | cur_ = iter_->CurNode(); 66 | 67 | //No keys is equal or larger than key 68 | if(!iter_->Valid()){ 69 | assert(cur_ == NULL); 70 | return; 71 | } 72 | 73 | //Second, find the first key which value is not NULL 74 | while(iter_->Valid()) { 75 | { 76 | RTMScope rtm(NULL); 77 | val_ = cur_->value; 78 | 79 | if(val_ != NULL) { 80 | return; 81 | } 82 | } 83 | 84 | iter_->Next(); 85 | cur_ = iter_->CurNode(); 86 | } 87 | cur_ = NULL; 88 | } 89 | 90 | private: 91 | OCC *tx_; 92 | Memstore::Iterator *iter_; 93 | 94 | MemNode* cur_ = NULL; 95 | uint64_t *val_ = NULL; 96 | uint64_t *prev_link_ = NULL; 97 | }; 98 | 99 | } // namespace rtx 100 | 101 | } // namespace nocc 102 | -------------------------------------------------------------------------------- /src/rtx/occ_statistics.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This file should be included in occ.h 3 | */ 4 | 5 | LAT_VARS(lock); 6 | LAT_VARS(validate); 7 | LAT_VARS(commit); 8 | LAT_VARS(log); 9 | LAT_VARS(temp); 10 | 11 | #if NOCC_STATICS 12 | CountVector lock_; 13 | CountVector validate_; 14 | CountVector commit_; 15 | CountVector log_; 16 | CountVector temp_; 17 | 18 | void report_statics(uint64_t one_second) { 19 | 20 | lock_.erase(0.1); 21 | //validate_.erase(0.1); 22 | commit_.erase(0.1); 23 | log_.erase(0.1); 24 | temp_.erase(0.1); 25 | 26 | LOG(4) << "lock time: " << util::BreakdownTimer::rdtsc_to_ms(lock_.average(),one_second) << "ms"; 27 | LOG(4) << "logging time: " << util::BreakdownTimer::rdtsc_to_ms(log_.average(),one_second) << "ms"; 28 | LOG(4) << "commit time: " << util::BreakdownTimer::rdtsc_to_ms(commit_.average(),one_second) << "ms"; 29 | LOG(4) << "temp time: " << temp_.average() << "cycle"; 30 | } 31 | 32 | void record() { 33 | double res; 34 | REPORT_V(lock,res); 35 | lock_.push_back(res); 36 | 37 | REPORT_V(log,res); 38 | log_.push_back(res); 39 | 40 | REPORT_V(commit,res); 41 | commit_.push_back(res); 42 | 43 | REPORT_V(temp,res); 44 | temp_.push_back(res); 45 | } 46 | 47 | #else 48 | // no counting, return null 49 | void report_statics(uint64_t) { 50 | 51 | } 52 | 53 | void record() { 54 | 55 | } 56 | #endif 57 | -------------------------------------------------------------------------------- /src/rtx/op_test.cxx: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | 3 | #include "tx_operator.hpp" 4 | 5 | 6 | using namespace nocc::rtx; 7 | 8 | 9 | TEST(tx_test,get_op_test) { 10 | 11 | TXOpBase op_set; 12 | char buf[1024]; 13 | 14 | struct TestStruct { 15 | int x; 16 | int y; 17 | inline TestStruct(int x,int y): x(x),y(y) {} 18 | }; 19 | 20 | op_set.remote_rpc_op(0,buf,NULL,12,14); 21 | TestStruct *s = (TestStruct *)buf; 22 | ASSERT_EQ(s->x,12); 23 | ASSERT_EQ(s->y,14); 24 | 25 | op_set.remote_rpc_op(0,buf + sizeof(TestStruct),NULL,50,64); 26 | ASSERT_EQ(s[1].x,50); 27 | ASSERT_EQ(s[1].y,64); 28 | } 29 | -------------------------------------------------------------------------------- /src/rtx/qp_selection_helper.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is included in the *class definiation file*. 3 | * It provides utilities to help select RC qp for the remote servers. 4 | * We use this file to help using multiple QPs to connection to the same server. 5 | * 6 | * Pre-assumptions: 7 | * The class should contains a qp_vec_, which is type std::vector to store RC qps. 8 | */ 9 | // QP vector to store all QPs 10 | std::vector qp_vec_; 11 | // FIXME: now only use 16 machines 12 | uint qp_idx_[16]; 13 | 14 | #if LARGE_CONNECTION 15 | 16 | // Use a large number of connections 17 | inline __attribute__((always_inline)) 18 | rdmaio::RCQP* get_qp(int pid){ 19 | int idx = (qp_idx_[pid]++) % QP_NUMS; 20 | QP *qp = qp_vec_[pid * QP_NUMS + idx]; 21 | return qp; 22 | } 23 | 24 | #else 25 | 26 | inline __attribute__((always_inline)) 27 | rdmaio::RCQP* get_qp(int pid) { 28 | return qp_vec_[pid]; 29 | } 30 | 31 | #endif 32 | 33 | 34 | void fill_qp_vec(RdmaCtrl *cm,int num_nodes,int wid) { 35 | 36 | assert(qp_vec_.size() == 0); 37 | // get rc qps 38 | for(int i = 0;i < num_nodes;++i) { 39 | for(int j = 0;j < QP_NUMS; j++){ 40 | rdmaio::RCQP *qp = cm->get_rc_qp(QPIdx {.node_id = i,.worker_id = wid,.index = j}); 41 | ASSERT(qp != nullptr) << "qp id : " << i << " " << wid << " " << j; 42 | qp_vec_.push_back(qp); 43 | } 44 | } 45 | #if LARGE_CONNECTION 46 | // init the first QP idx to 0 47 | memset(qp_idx_,0,sizeof(qp_idx_)); 48 | #endif 49 | } 50 | 51 | // snaity checks 52 | static_assert(QP_NUMS >= 1,"Each mac requires at least one QP!"); 53 | #if !LARGE_CONNECTION 54 | static_assert(QP_NUMS == 1,"If not use a large connection, we should use exactly one QP to link to another node."); 55 | #endif 56 | -------------------------------------------------------------------------------- /src/rtx/rdma_logger_impl.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "core/rdma_sched.h" 4 | 5 | namespace nocc { 6 | namespace rtx { 7 | 8 | using namespace rdmaio; 9 | using namespace oltp; 10 | 11 | class RDMALogger : public Logger { 12 | public: 13 | RDMALogger(RdmaCtrl *cm, RScheduler* rdma_sched,int nid,int tid,uint64_t base_off, 14 | RRpc *rpc,int ack_rpc_id, 15 | int expected_store,char *local_p,int ms,int ts,int size,int entry_size = RTX_LOG_ENTRY_SIZE) 16 | :Logger(rpc,ack_rpc_id,base_off,expected_store,local_p,ms,ts,size,entry_size), 17 | node_id_(nid),worker_id_(tid), 18 | scheduler_(rdma_sched) 19 | { 20 | // init local QP vector 21 | fill_qp_vec(cm,ms,worker_id_); 22 | } 23 | 24 | inline void log_remote(BatchOpCtrlBlock &clk, int cor_id) { 25 | 26 | int size = clk.batch_msg_size(); // log size 27 | assert(clk.mac_set_.size() > 0); 28 | // post the log to remote servers 29 | for(auto it = clk.mac_set_.begin();it != clk.mac_set_.end();++it) { 30 | int mac_id = *it; 31 | auto qp = get_qp(mac_id); 32 | auto off = mem_.get_remote_log_offset(node_id_,worker_id_,mac_id,size); 33 | assert(off != 0); 34 | scheduler_->post_send(qp,cor_id, 35 | IBV_WR_RDMA_WRITE,clk.req_buf_,size,off, 36 | IBV_SEND_SIGNALED | ((size < 64)?IBV_SEND_INLINE:0)); 37 | } 38 | // requires yield call after this! 39 | } 40 | 41 | private: 42 | RScheduler *scheduler_; 43 | int node_id_; 44 | int worker_id_; 45 | 46 | #include "qp_selection_helper.h" 47 | }; 48 | 49 | }; // namespace rtx 50 | }; // namespace nocc 51 | -------------------------------------------------------------------------------- /src/rtx/rdma_op_impl.hpp: -------------------------------------------------------------------------------- 1 | #include "tx_config.h" 2 | 3 | namespace nocc { 4 | namespace rtx { 5 | 6 | inline __attribute__((always_inline)) 7 | uint64_t TXOpBase::rdma_lookup_op(int pid,int tableid,uint64_t key,char *val, 8 | yield_func_t &yield,int meta_len) { 9 | RCQP *qp = get_qp(pid); 10 | assert(qp != NULL); 11 | // MemNode will be stored in val, if necessary 12 | auto off = db_->stores_[tableid]->RemoteTraverse(key,qp,scheduler_,yield,val); 13 | return off; 14 | } 15 | 16 | inline __attribute__ ((always_inline)) 17 | uint64_t TXOpBase::rdma_read_val(int pid,int tableid,uint64_t key,int len,char *val,yield_func_t &yield,int meta_len) { 18 | auto data_off = pending_rdma_read_val(pid,tableid,key,len,val,yield,meta_len); 19 | worker_->indirect_yield(yield); // yield for waiting for NIC's completion 20 | return data_off; 21 | } 22 | 23 | inline __attribute__ ((always_inline)) 24 | uint64_t TXOpBase::pending_rdma_read_val(int pid,int tableid,uint64_t key,int len,char *val,yield_func_t &yield,int meta_len) { 25 | // store the memnode in val 26 | auto off = rdma_lookup_op(pid,tableid,key,val,yield,meta_len); 27 | MemNode *node = (MemNode *)val; 28 | 29 | auto data_off = off; 30 | #if !RDMA_CACHE 31 | data_off = node->off; // fetch the offset from the content 32 | #endif 33 | 34 | // fetch the content 35 | RCQP *qp = get_qp(pid); 36 | scheduler_->post_send(qp,worker_->cor_id(), 37 | IBV_WR_RDMA_READ,val,len + meta_len,data_off, IBV_SEND_SIGNALED); 38 | 39 | if(unlikely(qp->need_poll())) { 40 | worker_->indirect_yield(yield); 41 | } 42 | 43 | return data_off; 44 | } 45 | 46 | } // namespace rtx 47 | 48 | } // namespace nocc 49 | -------------------------------------------------------------------------------- /src/rtx/rpc_logger_impl.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "core/rdma_sched.h" 4 | #include "core/rrpc.h" 5 | 6 | namespace nocc { 7 | 8 | namespace rtx { 9 | 10 | using namespace rdmaio; 11 | using namespace oltp; 12 | 13 | class RpcLogger : public Logger { 14 | public: 15 | RpcLogger(RRpc *rpc,int log_rpc_id,int ack_rpc_id,uint64_t base_off, 16 | int expected_store,char *local_p,int ms,int ts,int size,int entry_size = RTX_LOG_ENTRY_SIZE) 17 | :Logger(rpc,ack_rpc_id,base_off,expected_store,local_p,ms,ts,size,entry_size), 18 | log_rpc_id_(log_rpc_id) 19 | { 20 | // register RPC if necessary 21 | rpc->register_callback(std::bind(&RpcLogger::log_remote_handler,this, 22 | std::placeholders::_1, 23 | std::placeholders::_2, 24 | std::placeholders::_3, 25 | std::placeholders::_4), 26 | log_rpc_id_,true); 27 | } 28 | 29 | inline void log_remote(BatchOpCtrlBlock &clk,int cor_id) { 30 | assert(clk.batch_size_ > 0); 31 | clk.send_batch_op(rpc_handler_,cor_id,log_rpc_id_,false); 32 | } 33 | 34 | void log_remote_handler(int id,int cid,char *msg,void *arg) { 35 | int size = (uint64_t)arg; 36 | //char *local_ptr = mem_.get_next_log(id,rpc_handler_->worker_id_,size); 37 | assert(size < RTX_LOG_ENTRY_SIZE && size > 0); 38 | //memcpy(local_ptr,msg,size); 39 | memcpy(local_buffer + cid * MAX_MSG_SIZE,msg,size); 40 | 41 | char* reply_msg = rpc_handler_->get_reply_buf(); 42 | rpc_handler_->send_reply(reply_msg,0,id,cid); // a dummy reply 43 | } 44 | 45 | private: 46 | const int log_rpc_id_; 47 | char local_buffer[MAX_MSG_SIZE * 32]; 48 | }; 49 | 50 | 51 | } // namespace rtx 52 | } // namespace nocc 53 | -------------------------------------------------------------------------------- /src/rtx/view.cc: -------------------------------------------------------------------------------- 1 | #include "core/logging.h" 2 | #include "view.h" 3 | 4 | namespace nocc { 5 | 6 | namespace rtx { 7 | 8 | void SymmetricView::assign_backups(int total) { 9 | ASSERT(rep_factor_ <= RTX_MAX_BACKUP) << "Too many backups supported."; 10 | LOG(2) << "total " << total <<" backups to assign"; 11 | for(uint i = 0;i < total;++i) { 12 | mapping_.emplace_back(rep_factor_); 13 | assign_one(i,total); 14 | } 15 | } 16 | 17 | void SymmetricView::assign_one(int idx,int total) { 18 | 19 | ASSERT(total > rep_factor_) << "Cannot assign backups properly. Total mac " << total 20 | << "; rep factor " << rep_factor_; 21 | for(uint i = 0;i < rep_factor_;++i) { 22 | // uses static mapping. 23 | // maybe we can use a random strategy 24 | mapping_[idx][i] = (idx + i + 1) % (total); 25 | } 26 | } 27 | 28 | void SymmetricView::print() { 29 | for(uint i = 0;i < mapping_.size();++i) { 30 | LOG(2) << "Mac [" << i << "] backed by " << mapping_[i]; 31 | } 32 | } 33 | 34 | } // namespace rtx 35 | 36 | } // namespace nocc 37 | -------------------------------------------------------------------------------- /src/rtx/view.h: -------------------------------------------------------------------------------- 1 | #ifndef RTX_VIEW_H_ 2 | #define RTX_VIEW_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "core/logging.h" 10 | 11 | namespace nocc { 12 | 13 | namespace rtx { 14 | 15 | #define RTX_MAX_BACKUP 2 16 | 17 | struct BackupInfo { 18 | int mapping[RTX_MAX_BACKUP]; 19 | const int rep_factor; 20 | 21 | explicit BackupInfo(int rep) 22 | :rep_factor(rep) 23 | { 24 | std::fill_n(mapping,RTX_MAX_BACKUP,-1); 25 | } 26 | 27 | inline int& operator[] (std::size_t idx) { 28 | return mapping[idx]; 29 | } 30 | 31 | friend std::ostream& operator<<(std::ostream& ss, const BackupInfo& b) { 32 | for(uint i = 0;i < b.rep_factor;++i) 33 | ss << b.mapping[i] << ","; 34 | ss << "."; 35 | } 36 | }; 37 | 38 | class SymmetricView { 39 | public: 40 | SymmetricView(int rep_factor,int total_mac) : 41 | rep_factor_(rep_factor >= total_mac?0:rep_factor) 42 | { 43 | if(rep_factor_ >= total_mac) { 44 | LOG(3) << "Disable backups!" 45 | << "Rep factor requires at least " << rep_factor_ + 1 <<" macs," 46 | << "yet total " << total_mac << "in the setting."; 47 | } 48 | LOG(3) << "Start with " << rep_factor_ << " backups."; 49 | assign_backups(total_mac); 50 | } 51 | 52 | /** 53 | * add pid's backup list to the mac_set. 54 | */ 55 | inline void add_backup(int pid,std::set &mac_set) { 56 | for(uint i = 0;i < rep_factor_;++i) 57 | mac_set.insert(mapping_[pid][i]); 58 | } 59 | 60 | /** 61 | * If bid is one of pid's backup, return true. 62 | * Otherwise, return false. 63 | */ 64 | inline bool is_backup(int bid,int pid) { 65 | return is_backup(bid,pid,rep_factor_); 66 | } 67 | 68 | inline size_t response_for(int bid,std::set &set) { 69 | for(uint i = 0;i < mapping_.size();++i) 70 | if(is_backup(bid,i)) 71 | set.insert(i); 72 | return set.size(); 73 | } 74 | 75 | /** 76 | * If bid is one of pid's backup in the first iterating_num mac, return true. 77 | * Otherwise, return false. 78 | */ 79 | inline bool is_backup(int bid,int pid,int iterating_num) { 80 | for(uint i = 0;i < iterating_num;++i) 81 | if(mapping_[pid][i] == bid) 82 | return true; 83 | return false; 84 | } 85 | 86 | void print(); 87 | 88 | const int rep_factor_; 89 | 90 | private: 91 | std::vector mapping_; 92 | 93 | void assign_backups(int total); 94 | void assign_one(int idx,int total); 95 | 96 | 97 | DISABLE_COPY_AND_ASSIGN(SymmetricView); 98 | }; // SymmetricView 99 | 100 | } // namespace rtx 101 | 102 | } // namespace nocc 103 | 104 | 105 | #endif 106 | -------------------------------------------------------------------------------- /src/util/Makefile: -------------------------------------------------------------------------------- 1 | name = rdtsc 2 | 3 | rdtsc: 4 | g++ -o $(name) rdtsc_converter.cxx -std=c++0x 5 | 6 | clean: 7 | rm $(name) 8 | -------------------------------------------------------------------------------- /src/util/mapped_log.h: -------------------------------------------------------------------------------- 1 | #ifndef _MAPPED_LOG_H 2 | #define _MAPPED_LOG_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | 11 | // Expose this structure so we can use inline function. 12 | // Call next_log_entry to obtain next avaiable buffer. 13 | typedef struct { 14 | char *start; 15 | char *buf; // buf can be changed 16 | char *end; 17 | int fd; 18 | 19 | int inc_size; 20 | int log_size; 21 | } MappedLog; 22 | 23 | // Create a new log. inc_size specifies how much space should enlarge_mapped_log 24 | // increase upon each call, it should be a MULTIPLE OF PAGE SIZE (4096). 25 | // Return 0 on success, -1 on error. 26 | int new_mapped_log(const char *path, MappedLog *log, int inc_size); 27 | 28 | // Open an existing log 29 | // Return 0 on success, -1 on error. 30 | int open_mapped_log(const char *path, MappedLog *log); 31 | 32 | int unmap_log(MappedLog *log); 33 | 34 | // Usually you should just call next_log_entry, no need to call this directly. 35 | // Return 0 on success, -1 on error. 36 | int enlarge_mapped_log(MappedLog *log); 37 | 38 | // Use this function to obtain next avaiable buffer. 39 | // On error, return NULL; 40 | static inline char *next_log_entry(MappedLog *log, int entry_size) { 41 | 42 | if ((log->buf + entry_size) > log->end) { 43 | if (enlarge_mapped_log(log) != 0) { 44 | exit(1); 45 | return NULL; 46 | } 47 | } 48 | if(log->log_size + entry_size > 1024 * 1024 * 12) { 49 | // avoids log to be too large 50 | //return NULL; 51 | } 52 | char *start = log->buf; 53 | log->buf += entry_size; 54 | log->log_size += entry_size; 55 | return start; 56 | } 57 | 58 | static inline char *read_log_entry(MappedLog *log, int entry_size) { 59 | if ((log->buf + entry_size) > log->end) { 60 | return NULL; 61 | } 62 | char *start = log->buf; 63 | log->buf += entry_size; 64 | return start; 65 | } 66 | 67 | #ifdef __cplusplus 68 | } 69 | #endif 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /src/util/msg_buf_allocator.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_UTIL_BUF_ALLOCATOR 2 | #define NOCC_UTIL_BUF_ALLOCATOR 3 | 4 | 5 | namespace nocc { 6 | 7 | namespace util { 8 | 9 | class MsgBufAllocator { 10 | 11 | }; // end class 12 | 13 | }; 14 | 15 | }; 16 | 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/util/mutexlock.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 6 | #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 7 | 8 | #include "port/port.h" 9 | #include "port/thread_annotations.h" 10 | #include "util/spinlock.h" 11 | 12 | 13 | namespace leveldb { 14 | 15 | // Helper class that locks a mutex on construction and unlocks the mutex when 16 | // the destructor of the MutexLock object is invoked. 17 | // 18 | // Typical usage: 19 | // 20 | // void MyClass::MyMethod() { 21 | // MutexLock l(&mu_); // mu_ is an instance variable 22 | // ... some complex code, possibly with multiple return paths ... 23 | // } 24 | 25 | class SCOPED_LOCKABLE MutexLock { 26 | public: 27 | explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) 28 | : mu_(mu) { 29 | this->mu_->Lock(); 30 | } 31 | ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 32 | 33 | private: 34 | port::Mutex *const mu_; 35 | // No copying allowed 36 | MutexLock(const MutexLock&); 37 | void operator=(const MutexLock&); 38 | }; 39 | 40 | 41 | class SCOPED_LOCKABLE MutexSpinLock { 42 | public: 43 | explicit MutexSpinLock(port::SpinLock *mu) EXCLUSIVE_LOCK_FUNCTION(mu) 44 | : mu_(mu) { 45 | this->mu_->Lock(); 46 | } 47 | ~MutexSpinLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } 48 | 49 | private: 50 | port::SpinLock *const mu_; 51 | // No copying allowed 52 | MutexSpinLock(const MutexSpinLock&); 53 | void operator=(const MutexSpinLock&); 54 | }; 55 | 56 | class SCOPED_LOCKABLE SpinLockScope { 57 | public: 58 | explicit SpinLockScope(SpinLock *slock) 59 | : slock_(slock) { 60 | slock_->Lock(); 61 | } 62 | ~SpinLockScope() { slock_->Unlock(); } 63 | 64 | private: 65 | SpinLock *slock_; 66 | // No copying allowed 67 | SpinLockScope(const SpinLockScope&); 68 | void operator=(const SpinLockScope&); 69 | }; 70 | 71 | 72 | } // namespace leveldb 73 | 74 | 75 | #endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ 76 | -------------------------------------------------------------------------------- /src/util/random.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_ 6 | #define STORAGE_LEVELDB_UTIL_RANDOM_H_ 7 | 8 | #include 9 | 10 | namespace leveldb { 11 | 12 | // A very simple random number generator. Not especially good at 13 | // generating truly random bits, but good enough for our needs in this 14 | // package. 15 | class Random { 16 | private: 17 | uint32_t seed_; 18 | public: 19 | explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) { } 20 | uint32_t Next() { 21 | static const uint32_t M = 2147483647L; // 2^31-1 22 | static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0 23 | // We are computing 24 | // seed_ = (seed_ * A) % M, where M = 2^31-1 25 | // 26 | // seed_ must not be zero or M, or else all subsequent computed values 27 | // will be zero or M respectively. For all other values, seed_ will end 28 | // up cycling through every number in [1,M-1] 29 | uint64_t product = seed_ * A; 30 | 31 | // Compute (product % M) using the fact that ((x << 31) % M) == x. 32 | seed_ = static_cast((product >> 31) + (product & M)); 33 | // The first reduction may overflow by 1 bit, so we may need to 34 | // repeat. mod == M is not possible; using > allows the faster 35 | // sign-bit-based test. 36 | if (seed_ > M) { 37 | seed_ -= M; 38 | } 39 | return seed_; 40 | } 41 | // Returns a uniformly distributed value in the range [0..n-1] 42 | // REQUIRES: n > 0 43 | uint32_t Uniform(int n) { return Next() % n; } 44 | 45 | // Randomly returns true ~"1/n" of the time, and false otherwise. 46 | // REQUIRES: n > 0 47 | bool OneIn(int n) { return (Next() % n) == 0; } 48 | 49 | // Skewed: pick "base" uniformly from range [0,max_log] and then 50 | // return "base" random bits. The effect is to pick a number in the 51 | // range [0,2^max_log-1] with exponential bias towards smaller numbers. 52 | uint32_t Skewed(int max_log) { 53 | return Uniform(1 << Uniform(max_log + 1)); 54 | } 55 | }; 56 | 57 | } // namespace leveldb 58 | 59 | #endif // STORAGE_LEVELDB_UTIL_RANDOM_H_ 60 | -------------------------------------------------------------------------------- /src/util/rdtsc_converter.cxx: -------------------------------------------------------------------------------- 1 | #include "util.h" 2 | 3 | #include 4 | #include 5 | 6 | 7 | int main(int argc,char **argv) { 8 | 9 | assert(argc > 1); 10 | 11 | double cycles = atof(argv[1]); 12 | auto one_second_cycle = Breakdown_Timer::get_one_second_cycle(); 13 | 14 | double converted = cycles / one_second_cycle; 15 | 16 | fprintf(stdout,"one second cycles %lu\n",one_second_cycle); 17 | fprintf(stdout,"Takes %f us.\n",converted * 1000000); 18 | 19 | return 0; 20 | } 21 | -------------------------------------------------------------------------------- /src/util/rtm.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * The code is part of our project called DrTM, which leverages HTM and RDMA for speedy distributed 3 | * in-memory transactions. 4 | * 5 | * 6 | * Copyright (C) 2015 Institute of Parallel and Distributed Systems (IPADS), Shanghai Jiao Tong University 7 | * All rights reserved. 8 | * 9 | * Licensed under the Apache License, Version 2.0 (the "License"); 10 | * you may not use this file except in compliance with the License. 11 | * You may obtain a copy of the License at 12 | * 13 | * http://www.apache.org/licenses/LICENSE-2.0 14 | * 15 | * Unless required by applicable law or agreed to in writing, software 16 | * distributed under the License is distributed on an "AS IS" BASIS, 17 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | * See the License for the specific language governing permissions and 19 | * limitations under the License. 20 | * 21 | * For more about this software, visit: http://ipads.se.sjtu.edu.cn/drtm.html 22 | * 23 | */ 24 | 25 | #include "rtm.h" 26 | 27 | //define the static member here 28 | SpinLock RTMScope::fblock; 29 | RTMProfile RTMTX::localprofile; 30 | __thread bool RTMTX::owner = false; 31 | -------------------------------------------------------------------------------- /src/util/simple_logger.h: -------------------------------------------------------------------------------- 1 | #ifndef SIMPLE_LOG 2 | #define SIMPLE_LOG 3 | 4 | 5 | // This is a simple wrapper to help manage debug infors 6 | 7 | #include "mapped_log.h" 8 | 9 | 10 | namespace simple_log { 11 | 12 | class SimpleLogger { 13 | 14 | // assert the results, if the assertion fails, print the message and exit 15 | void inline debug_assert(bool expression, xxx) { 16 | 17 | } 18 | }; 19 | }; 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /src/util/spinlock.h: -------------------------------------------------------------------------------- 1 | #ifndef LEVELDB_SPINLOCK_H 2 | #define LEVELDB_SPINLOCK_H 3 | 4 | #include 5 | #include "port/atomic.h" 6 | 7 | /* The counter should be initialized to be 0. */ 8 | class SpinLock { 9 | 10 | public: 11 | //0: free, 1: busy 12 | //occupy an exclusive cache line 13 | volatile uint8_t padding1[32]; 14 | volatile uint16_t lock; 15 | volatile uint8_t padding2[32]; 16 | public: 17 | 18 | SpinLock(){ lock = 0;} 19 | 20 | inline void Lock() { 21 | while (1) { 22 | if (!xchg16((uint16_t *)&lock, 1)) return; 23 | 24 | while (lock) cpu_relax(); 25 | } 26 | } 27 | 28 | inline void Unlock() 29 | { 30 | barrier(); 31 | lock = 0; 32 | } 33 | 34 | 35 | inline uint16_t Trylock() 36 | { 37 | return xchg16((uint16_t *)&lock, 1); 38 | } 39 | 40 | inline uint16_t IsLocked(){return lock;} 41 | 42 | 43 | }; 44 | 45 | #endif /* _RWLOCK_H */ 46 | -------------------------------------------------------------------------------- /src/util/timer.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_UTIL_TIMER 2 | #define NOCC_UTIL_TIMER 3 | 4 | #include 5 | #include 6 | 7 | 8 | namespace nocc { 9 | namespace util { 10 | 11 | class BreakdownTimer { 12 | const uint64_t max_elems = 1000000; 13 | public: 14 | uint64_t sum; 15 | uint64_t count; 16 | uint64_t temp; 17 | std::vector buffer; 18 | BreakdownTimer(): sum(0), count(0) {} 19 | void start() { temp = rdtsc(); } 20 | void end() { auto res = (rdtsc() - temp);sum += res; count += 1; 21 | emplace(res); 22 | } 23 | void emplace(uint64_t res) { 24 | if(buffer.size() >= max_elems) return; 25 | buffer.push_back(res); 26 | } 27 | double report() { 28 | if(count == 0) return 0.0; // avoids divided by zero 29 | double ret = (double) sum / (double)count; 30 | // clear the info 31 | //sum = 0; 32 | //count = 0; 33 | return ret; 34 | } 35 | 36 | void calculate_detailed() { 37 | 38 | if(buffer.size() == 0) { 39 | fprintf(stderr,"no timer!\n"); 40 | return; 41 | } 42 | // first erase some items 43 | int temp_size = buffer.size(); 44 | int idx = std::floor(buffer.size() * 0.1 / 100.0); 45 | buffer.erase(buffer.begin() + temp_size - idx, buffer.end()); 46 | buffer.erase(buffer.begin(),buffer.begin() + idx ); 47 | 48 | // then sort 49 | std::sort(buffer.begin(),buffer.end()); 50 | } 51 | 52 | double report_medium() { 53 | if(buffer.size() == 0) return 0; 54 | return buffer[buffer.size() / 2]; 55 | } 56 | 57 | double report_90(){ 58 | if(buffer.size() == 0) return 0; 59 | int idx = std::floor( buffer.size() * 90 / 100.0); 60 | return buffer[idx]; 61 | } 62 | 63 | double report_99() { 64 | if(buffer.size() == 0) return 0; 65 | int idx = std::floor(buffer.size() * 99 / 100.0); 66 | return buffer[idx]; 67 | } 68 | 69 | double report_avg() { 70 | if(buffer.size() == 0) return 0; 71 | double average = 0; 72 | uint64_t count = 0; 73 | for(uint i = 0;i < buffer.size();++i) { 74 | average += (buffer[i] - average) / (++count); 75 | } 76 | return average; 77 | } 78 | 79 | static uint64_t get_one_second_cycle() { 80 | uint64_t begin = rdtsc(); 81 | sleep(1); 82 | return rdtsc() - begin; 83 | } 84 | 85 | static double rdtsc_to_ms(uint64_t rdts, uint64_t one_second_cycle) { 86 | return ((double)rdts / (double)one_second_cycle) * 1000; 87 | } 88 | }; 89 | } // namespace util 90 | } // namespace nocc 91 | #endif 92 | -------------------------------------------------------------------------------- /src/util/util.h: -------------------------------------------------------------------------------- 1 | #ifndef NOCC_UTIL_H 2 | #define NOCC_UTIL_H 3 | 4 | /* This util file is imported from DrTM+ */ 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include /* sort */ 14 | #include 15 | 16 | #include /* floor */ 17 | 18 | #include 19 | 20 | #include 21 | 22 | #define ALWAYS_INLINE __attribute__((always_inline)) 23 | inline ALWAYS_INLINE uint64_t rdtsc(void) { uint32_t hi, lo; __asm volatile("rdtsc" : "=a"(lo), "=d"(hi)); return ((uint64_t)lo)|(((uint64_t)hi)<<32); } 24 | #define unlikely(x) __builtin_expect(!!(x), 0) 25 | #define MAX(x,y) ( (x) > (y) ? (x) : (y)) 26 | 27 | namespace nocc { 28 | 29 | namespace util { 30 | 31 | template 32 | constexpr R BitMask(unsigned int const onecount) 33 | { 34 | return static_cast(-(onecount != 0)) 35 | & (static_cast(-1) >> ((sizeof(R) * CHAR_BIT) - onecount)); 36 | } 37 | 38 | 39 | int BindToCore (int thread_id); 40 | int CorePerSocket(); 41 | template inline ALWAYS_INLINE // Round "a" according to "b" 42 | Num Round (Num a, Num b) { 43 | if(a < b) return b; 44 | Num r = a % b; 45 | return r?(a + (b - r)): a ; 46 | } 47 | int DiffTimespec(const struct timespec &end, const struct timespec &start); 48 | 49 | // wrappers for parsing configure file 50 | // !! notice that even a failed parsing will results for cursor advancing 51 | bool NextDouble(std::istream &ist,double &res); 52 | bool NextInt(std::istream &ist,int &res); 53 | bool NextLine(std::istream &ist,std::string &res); 54 | bool NextString(std::istream &ist,std::string &res); 55 | void BypassLine(std::istream &ist); 56 | 57 | inline uint64_t TimeToMs(struct timespec &t) { return t.tv_sec * 1000 + t.tv_nsec / 1000000;} 58 | 59 | std::pair get_system_memory_info(); // instruction's memory comsuption 60 | 61 | void *malloc_huge_pages(size_t size,uint64_t huge_page_sz,bool flag = true); 62 | 63 | inline double get_memory_size_g(uint64_t bytes) { static double G = 1024.0 * 1024 * 1024; return bytes / G; } 64 | 65 | void print_stacktrace(FILE *out = stderr, unsigned int max_frames = 63); 66 | 67 | } // namespace util 68 | } // namespace nocc 69 | 70 | // other related utils 71 | #include "timer.h" // timer helper 72 | #endif 73 | -------------------------------------------------------------------------------- /third_party/micautil/citycrc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include "city.h" 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /third_party/micautil/citycrc_mod.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include "city.h" 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /third_party/micautil/hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_HASH_H_ 3 | #define MICA_UTIL_HASH_H_ 4 | 5 | #include "citycrc_mod.h" 6 | 7 | namespace mica { 8 | namespace util { 9 | template 10 | static uint64_t hash(const T* key, size_t len) { 11 | return CityHash64(reinterpret_cast(key), len); 12 | } 13 | } 14 | } 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/del.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_DEL_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_DEL_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result FixedTable::del(uint32_t caller_id, uint64_t key_hash, 9 | ft_key_t key) { 10 | // Can be called at both primary and backup datastores 11 | uint32_t bucket_index = calc_bucket_index(key_hash); 12 | Bucket* bucket = get_bucket(bucket_index); 13 | 14 | // We must be holding the lock on this bucket at primaries. 15 | if(is_primary) { 16 | assert(is_locked(bucket->timestamp)); 17 | assert(bucket->locker_id == caller_id); 18 | } 19 | 20 | Bucket* located_bucket; 21 | size_t item_index = find_item_index(bucket, key, &located_bucket); 22 | 23 | // The key must exist - we checked this when we acquired the bucket lock 24 | // at the primary 25 | assert(item_index < StaticConfig::kBucketCap); 26 | 27 | // If we are here, the key exists 28 | located_bucket->key_arr[item_index] = kFtInvalidKey; 29 | stat_dec(&Stats::count); 30 | 31 | fill_hole(located_bucket, item_index); 32 | 33 | // Coordinators acquire bucket locks at primary. No need to unlock at backups. 34 | if(is_primary) { 35 | // unlock_bucket_ptr() will only release the lock that was previously 36 | // acquired at the primary for this key's deletion. Other locks acquired by 37 | // @caller_id on this bucket are still held. 38 | unlock_bucket_ptr(caller_id, bucket); 39 | } 40 | 41 | stat_inc(&Stats::delete_found); 42 | return Result::kSuccess; 43 | } 44 | } 45 | } 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/info.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_DEBUG_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_DEBUG_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void FixedTable::print_bucket(const Bucket* bucket) const { 9 | printf(": ", (size_t)bucket); 10 | for (size_t item_index = 0; item_index < StaticConfig::kBucketCap; 11 | item_index++) { 12 | printf("key %zx, ", static_cast(bucket->item_vec[item_index].key)); 13 | } 14 | 15 | printf("\n"); 16 | } 17 | 18 | template 19 | void FixedTable::print_buckets() const { 20 | for (size_t bucket_index = 0; bucket_index < num_buckets_; bucket_index++) { 21 | Bucket* bucket = get_bucket(bucket_index); 22 | print_bucket(bucket); 23 | } 24 | printf("\n"); 25 | } 26 | 27 | template 28 | void FixedTable::print_stats() const { 29 | if (StaticConfig::kCollectStats) { 30 | printf("count: %10zu\n", stats_.count); 31 | printf("set_new: %10zu | ", stats_.set_new); 32 | printf("get_found: %10zu | ", stats_.get_found); 33 | printf("get_notfound: %10zu\n", stats_.get_notfound); 34 | printf("test_found: %10zu | ", stats_.test_found); 35 | printf("test_notfound: %10zu\n", stats_.test_notfound); 36 | printf("delete_found: %10zu | ", stats_.delete_found); 37 | printf("delete_notfound: %10zu\n", stats_.delete_notfound); 38 | } 39 | } 40 | 41 | template 42 | void FixedTable::reset_stats(bool reset_count) { 43 | if (StaticConfig::kCollectStats) { 44 | size_t count = stats_.count; 45 | ::mica::util::memset(&stats_, 0, sizeof(stats_)); 46 | if (!reset_count) stats_.count = count; 47 | } 48 | } 49 | 50 | template 51 | void FixedTable::stat_inc(size_t Stats::*counter) const { 52 | if (StaticConfig::kCollectStats) __sync_add_and_fetch(&(stats_.*counter), 1); 53 | } 54 | 55 | template 56 | void FixedTable::stat_dec(size_t Stats::*counter) const { 57 | if (StaticConfig::kCollectStats) __sync_sub_and_fetch(&(stats_.*counter), 1); 58 | } 59 | } 60 | } 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/item.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_ITEM_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_ITEM_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | 8 | template 9 | void FixedTable::set_item(Bucket *located_bucket, 10 | size_t item_index, ft_key_t key, 11 | const char* value) { 12 | located_bucket->key_arr[item_index] = key; 13 | uint8_t *_val = get_value(located_bucket, item_index); 14 | ::mica::util::memcpy(_val, value, val_size); 15 | } 16 | } 17 | } 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/lock_bkt.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_H_ 4 | 5 | // This is a datapath API function exposed to users. The internal locking 6 | // function @lock_bucket_ptr() is not exposed to users. 7 | namespace mica { 8 | namespace table { 9 | template 10 | Result FixedTable::lock_bucket_hash(uint32_t caller_id, 11 | uint64_t key_hash) { 12 | assert(is_primary); 13 | 14 | uint32_t bucket_index = calc_bucket_index(key_hash); 15 | Bucket* bucket = get_bucket(bucket_index); 16 | 17 | bool res = lock_bucket_ptr(caller_id, bucket); 18 | 19 | return res ? Result::kSuccess : Result::kLocked; 20 | } 21 | } 22 | } 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/lock_bkt_and_get.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_AND_GET_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_AND_GET_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | 8 | template 9 | Result FixedTable::lock_bkt_and_get(uint32_t caller_id, 10 | uint64_t key_hash, ft_key_t key, uint64_t *out_timestamp, char *value) { 11 | assert(is_primary); 12 | 13 | uint32_t bucket_index = calc_bucket_index(key_hash); 14 | Bucket* bucket = get_bucket(bucket_index); 15 | 16 | bool lock_success = lock_bucket_ptr(caller_id, bucket); 17 | if(lock_success) { 18 | // We acquired the lock, or we were already holding it for @caller_id 19 | Bucket* located_bucket; 20 | size_t item_index = find_item_index(bucket, key, &located_bucket); 21 | 22 | if (item_index < StaticConfig::kBucketCap) { 23 | // The key exists 24 | *out_timestamp = bucket->timestamp; 25 | uint64_t *_val = (uint64_t *) get_value(located_bucket, item_index); 26 | ::mica::util::memcpy(value, _val, val_size); 27 | return Result::kSuccess; 28 | } 29 | 30 | // The key does not exist, i.e., we failed. unlock_bucket_ptr() will only 31 | // release the lock that this function acquired. Other locks acquired 32 | // by @caller_id on this bucket are still held. 33 | unlock_bucket_ptr(caller_id, bucket); 34 | return Result::kNotFound; 35 | } else { 36 | return Result::kLocked; 37 | } 38 | 39 | } 40 | } 41 | } 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/lock_bkt_for_ins.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_FOR_INS_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_LOCK_BKT_FOR_INS_ 4 | 5 | namespace mica { 6 | namespace table { 7 | 8 | template 9 | Result FixedTable::lock_bkt_for_ins(uint32_t caller_id, 10 | uint64_t key_hash, ft_key_t key, uint64_t *out_timestamp) { 11 | assert(is_primary); 12 | 13 | uint32_t bucket_index = calc_bucket_index(key_hash); 14 | Bucket* bucket = get_bucket(bucket_index); 15 | 16 | bool lock_success = lock_bucket_ptr(caller_id, bucket); 17 | if(lock_success) { 18 | // We acquired the lock, or we were already holding it for @caller_id 19 | Bucket* located_bucket; 20 | size_t item_index = find_item_index(bucket, key, &located_bucket); 21 | 22 | if (item_index == StaticConfig::kBucketCap) { 23 | // The key does not exist 24 | *out_timestamp = bucket->timestamp; 25 | return Result::kSuccess; 26 | } 27 | 28 | // The key exists, i.e., we failed. unlock_bucket_ptr() will only release 29 | // the lock that was previously acquired by this function. Other locks 30 | // acquired by @caller_id on this bucket are still held. 31 | unlock_bucket_ptr(caller_id, bucket); 32 | return Result::kExists; 33 | } else { 34 | return Result::kLocked; 35 | } 36 | 37 | } 38 | } 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/prefetch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_PREFETCH_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_PREFETCH_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void FixedTable::prefetch_table(uint64_t key_hash) const { 9 | uint32_t bucket_index = calc_bucket_index(key_hash); 10 | const Bucket* bucket = get_bucket(bucket_index); 11 | 12 | // bucket address is already 64-byte aligned 13 | 14 | // When value size is 16B, we need to prefetch 3 cache lines. For larger 15 | // values, we may need to prefetch more cache lines, but it does not seem to 16 | // improve performance. 17 | __builtin_prefetch(bucket, 0, 0); 18 | __builtin_prefetch(reinterpret_cast(bucket) + 64, 0, 0); 19 | __builtin_prefetch(reinterpret_cast(bucket) + 128, 0, 0); 20 | 21 | } 22 | 23 | } 24 | } 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/set.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_SET_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_SET_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result FixedTable::set(uint32_t caller_id, uint64_t key_hash, 9 | ft_key_t key, const char* value) { 10 | // Can be called at both primary and backup datastores 11 | uint32_t bucket_index = calc_bucket_index(key_hash); 12 | Bucket* bucket = get_bucket(bucket_index); 13 | 14 | // We must be holding the lock on this bucket at primaries. 15 | if(is_primary) { 16 | assert(is_locked(bucket->timestamp)); 17 | assert(bucket->locker_id == caller_id); 18 | } 19 | 20 | Bucket* located_bucket; 21 | size_t item_index = find_item_index(bucket, key, &located_bucket); 22 | 23 | if (item_index == StaticConfig::kBucketCap) { 24 | // The key does not exist in the table 25 | item_index = get_empty(bucket, &located_bucket); 26 | if (item_index == StaticConfig::kBucketCap) { 27 | // No more space. This should be fatal. 28 | unlock_bucket_ptr(caller_id, bucket); 29 | return Result::kInsufficientSpaceIndex; // This should be fatal 30 | } 31 | 32 | stat_inc(&Stats::set_new); 33 | } 34 | 35 | // Here, @located_bucket either contains @key at index @item_index, or is 36 | // empty at this slot. 37 | set_item(located_bucket, item_index, key, value); 38 | 39 | // Coordinators acquire bucket locks at primary. No need to unlock at backups. 40 | if(is_primary) { 41 | // unlock_bucket_ptr() will only release the lock that was previously 42 | // acquired at the primary for this set(). Other locks acquired by 43 | // @caller_id on this bucket are still held. 44 | unlock_bucket_ptr(caller_id, bucket); 45 | } 46 | 47 | return Result::kSuccess; 48 | } 49 | } 50 | } 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/set_spinlock.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_SET_LOCKED_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_SET_LOCKED_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result FixedTable::set_spinlock(uint32_t caller_id, 9 | uint64_t key_hash, ft_key_t key, const char* value) { 10 | // Can be called *locally* at both primary and backup datastores 11 | uint32_t bucket_index = calc_bucket_index(key_hash); 12 | Bucket* bucket = get_bucket(bucket_index); 13 | 14 | while(!lock_bucket_ptr(caller_id, bucket)) { 15 | // Spin on the bucket lock 16 | } 17 | 18 | Bucket* located_bucket; 19 | size_t item_index = find_item_index(bucket, key, &located_bucket); 20 | 21 | if (item_index == StaticConfig::kBucketCap) { 22 | // The key does not exist in the table 23 | item_index = get_empty(bucket, &located_bucket); 24 | if (item_index == StaticConfig::kBucketCap) { 25 | // no more space 26 | unlock_bucket_ptr(caller_id, bucket); 27 | return Result::kInsufficientSpaceIndex; // This should be fatal 28 | } 29 | 30 | stat_inc(&Stats::set_new); 31 | } 32 | 33 | // Here, @located_bucket either contains @key at index @item_index, or is 34 | // empty at this slot. 35 | set_item(located_bucket, item_index, key, value); 36 | unlock_bucket_ptr(caller_id, bucket); 37 | return Result::kSuccess; 38 | } 39 | } 40 | } 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /third_party/micautil/table/fixedtable_impl/unlock_bkt.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_FIXED_TABLE_IMPL_UNLOCK_BKT_H_ 3 | #define MICA_TABLE_FIXED_TABLE_IMPL_UNLOCK_BKT_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | // This is a datapath API function exposed to users. The internal locking 9 | // function @lock_bucket_ptr() is not exposed to users. 10 | Result FixedTable::unlock_bucket_hash(uint32_t caller_id, 11 | uint64_t key_hash) { 12 | assert(is_primary); 13 | 14 | uint32_t bucket_index = calc_bucket_index(key_hash); 15 | Bucket* bucket = get_bucket(bucket_index); 16 | 17 | // unlock_bucket_ptr() will only release one lock if @caller_id holds multiple 18 | // locks on this bucket. It will also do sanity checks. 19 | unlock_bucket_ptr(caller_id, bucket); 20 | return Result::kSuccess; 21 | } 22 | } 23 | } 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /third_party/micautil/table/ltable_impl/del.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_DEL_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_DEL_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::del(uint64_t key_hash, const char* key, 9 | size_t key_length) { 10 | assert(key_length <= kMaxKeyLength); 11 | 12 | uint32_t bucket_index = calc_bucket_index(key_hash); 13 | uint16_t tag = calc_tag(key_hash); 14 | 15 | Bucket* bucket = buckets_ + bucket_index; 16 | 17 | lock_bucket(bucket); 18 | 19 | Bucket* located_bucket; 20 | size_t item_index = 21 | find_item_index(bucket, key_hash, tag, key, key_length, &located_bucket); 22 | if (item_index == StaticConfig::kBucketSize) { 23 | unlock_bucket(bucket); 24 | stat_inc(&Stats::delete_notfound); 25 | return Result::kNotFound; 26 | } 27 | 28 | pool_->release(get_item_offset(located_bucket->item_vec[item_index])); 29 | 30 | located_bucket->item_vec[item_index] = 0; 31 | stat_dec(&Stats::count); 32 | 33 | fill_hole(located_bucket, item_index); 34 | 35 | unlock_bucket(bucket); 36 | 37 | stat_inc(&Stats::delete_found); 38 | return Result::kSuccess; 39 | } 40 | } 41 | } 42 | 43 | #endif -------------------------------------------------------------------------------- /third_party/micautil/table/ltable_impl/item.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_ITEM_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_ITEM_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | uint32_t LTable::get_key_length(uint32_t kv_length_vec) { 9 | return kv_length_vec >> 24; 10 | } 11 | 12 | template 13 | uint32_t LTable::get_value_length(uint32_t kv_length_vec) { 14 | return kv_length_vec & Item::kValueMask; 15 | } 16 | 17 | template 18 | uint32_t LTable::make_kv_length_vec(uint32_t key_length, 19 | uint32_t value_length) { 20 | return (key_length << 24) | value_length; 21 | } 22 | 23 | template 24 | uint16_t LTable::calc_tag(uint64_t key_hash) { 25 | uint16_t tag = (uint16_t)(key_hash & Bucket::kTagMask); 26 | if (tag == 0) 27 | return 1; 28 | else 29 | return tag; 30 | } 31 | 32 | template 33 | void LTable::set_item(Item* item, uint64_t key_hash, 34 | const char* key, uint32_t key_length, 35 | const char* value, uint32_t value_length) { 36 | assert(key_length <= Item::kKeyMask); 37 | assert(value_length <= Item::kValueMask); 38 | 39 | item->kv_length_vec = make_kv_length_vec(key_length, value_length); 40 | item->key_hash = key_hash; 41 | ::mica::util::memcpy<8>(item->data, key, key_length); 42 | ::mica::util::memcpy<8>(item->data + ::mica::util::roundup<8>(key_length), 43 | value, value_length); 44 | } 45 | 46 | template 47 | void LTable::set_item_value(Item* item, const char* value, 48 | uint32_t value_length) { 49 | assert(value_length <= Item::kValueMask); 50 | 51 | uint32_t key_length = get_key_length(item->kv_length_vec); 52 | item->kv_length_vec = make_kv_length_vec(key_length, value_length); 53 | ::mica::util::memcpy<8>(item->data + ::mica::util::roundup<8>(key_length), 54 | value, value_length); 55 | } 56 | 57 | template 58 | bool LTable::compare_keys(const char* key1, size_t key1_len, 59 | const char* key2, size_t key2_len) { 60 | return key1_len == key2_len && 61 | ::mica::util::memcmp_equal<8>(key1, key2, 62 | ::mica::util::roundup<8>(key1_len)); 63 | } 64 | } 65 | } 66 | 67 | #endif -------------------------------------------------------------------------------- /third_party/micautil/table/ltable_impl/prefetch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_PREFETCH_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_PREFETCH_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void LTable::prefetch_table(uint64_t key_hash) const { 9 | uint32_t bucket_index = calc_bucket_index(key_hash); 10 | const Bucket* bucket = buckets_ + bucket_index; 11 | 12 | // bucket address is already 64-byte aligned 13 | __builtin_prefetch(bucket, 0, 0); 14 | 15 | if (StaticConfig::kBucketSize > 7) 16 | __builtin_prefetch(reinterpret_cast(bucket) + 64, 0, 0); 17 | 18 | // XXX: prefetch extra buckets, too? 19 | } 20 | 21 | template 22 | void LTable::prefetch_pool(uint64_t key_hash) const { 23 | uint32_t bucket_index = calc_bucket_index(key_hash); 24 | const Bucket* bucket = buckets_ + bucket_index; 25 | 26 | uint16_t tag = calc_tag(key_hash); 27 | 28 | size_t item_index; 29 | for (item_index = 0; item_index < StaticConfig::kBucketSize; item_index++) { 30 | uint64_t item_vec = bucket->item_vec[item_index]; 31 | if (get_tag(item_vec) != tag) continue; 32 | 33 | pool_->prefetch_item(get_item_offset(item_vec)); 34 | } 35 | } 36 | } 37 | } 38 | 39 | #endif -------------------------------------------------------------------------------- /third_party/micautil/table/ltable_impl/specialization.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_SPECIALIZATION_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_SPECIALIZATION_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | class LTablePoolSpecialization { 9 | public: 10 | template 11 | static uint64_t get_tail(const Pool* pool); 12 | 13 | template 14 | static uint64_t get_mask(const Pool* pool); 15 | 16 | template 17 | static uint64_t get_size(const Pool* pool); 18 | 19 | template 20 | static bool is_valid(const Pool* pool, typename Pool::Offset offset); 21 | }; 22 | 23 | template 24 | template 25 | uint64_t LTablePoolSpecialization::get_tail(const Pool* pool) { 26 | (void)pool; 27 | return 0; 28 | } 29 | 30 | template <> 31 | template 32 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_tail( 33 | const Pool* pool) { 34 | return pool->get_tail(); 35 | } 36 | 37 | template 38 | template 39 | uint64_t LTablePoolSpecialization::get_mask(const Pool* pool) { 40 | (void)pool; 41 | return 0; 42 | } 43 | 44 | template <> 45 | template 46 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_mask( 47 | const Pool* pool) { 48 | return pool->get_mask(); 49 | } 50 | 51 | template 52 | template 53 | uint64_t LTablePoolSpecialization::get_size(const Pool* pool) { 54 | (void)pool; 55 | return 0; 56 | } 57 | 58 | template <> 59 | template 60 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_size( 61 | const Pool* pool) { 62 | return pool->get_size(); 63 | } 64 | 65 | template 66 | template 67 | bool LTablePoolSpecialization::is_valid(const Pool* pool, 68 | typename Pool::Offset offset) { 69 | (void)pool; 70 | (void)offset; 71 | return true; 72 | } 73 | 74 | template <> 75 | template 76 | bool LTablePoolSpecialization<::mica::pool::CircularLogTag>::is_valid( 77 | const Pool* pool, typename Pool::Offset offset) { 78 | return pool->is_valid(offset); 79 | } 80 | } 81 | } 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /third_party/micautil/table/ltable_impl/test.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_TEST_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_TEST_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::test(uint64_t key_hash, const char* key, 9 | size_t key_length) const { 10 | assert(key_length <= kMaxKeyLength); 11 | 12 | uint32_t bucket_index = calc_bucket_index(key_hash); 13 | uint16_t tag = calc_tag(key_hash); 14 | 15 | const Bucket* bucket = buckets_ + bucket_index; 16 | 17 | while (true) { 18 | uint32_t version_start = read_version_begin(bucket); 19 | 20 | const Bucket* located_bucket; 21 | size_t item_index = find_item_index(bucket, key_hash, tag, key, key_length, 22 | &located_bucket); 23 | 24 | if (version_start != read_version_end(bucket)) continue; 25 | 26 | if (item_index != StaticConfig::kBucketSize) { 27 | stat_inc(&Stats::test_found); 28 | return Result::kSuccess; 29 | } else { 30 | stat_inc(&Stats::test_notfound); 31 | return Result::kNotFound; 32 | } 33 | } 34 | // Not reachable. 35 | } 36 | } 37 | } 38 | 39 | #endif -------------------------------------------------------------------------------- /third_party/micautil/table/table.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_TABLE_H_ 3 | #define MICA_TABLE_TABLE_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/table/types.h" 7 | 8 | namespace mica { 9 | namespace table { 10 | class TableInterface { 11 | public: 12 | void reset(); 13 | 14 | Result del(uint64_t key_hash, const char* key, size_t key_length); 15 | 16 | Result get(uint64_t key_hash, const char* key, size_t key_length, 17 | char* out_value, size_t in_value_length, size_t* out_value_length, 18 | bool allow_mutation) const; 19 | 20 | Result increment(uint64_t key_hash, const char* key, size_t key_length, 21 | uint64_t increment, uint64_t* out_value); 22 | 23 | Result set(uint64_t key_hash, const char* key, size_t key_length, 24 | const char* value, size_t value_length, bool overwrite); 25 | 26 | Result test(uint64_t key_hash, const char* key, size_t key_length) const; 27 | 28 | void prefetch_table(uint64_t key_hash) const; 29 | void prefetch_pool(uint64_t key_hash) const; 30 | 31 | void print_buckets() const; 32 | void print_stats() const; 33 | void reset_stats(bool reset_count); 34 | }; 35 | } 36 | } 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /third_party/micautil/table/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_TYPES_H_ 3 | #define MICA_TABLE_TYPES_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | 8 | namespace mica { 9 | namespace table { 10 | enum class Result { 11 | kSuccess = 0, 12 | kLocked, 13 | kError, 14 | kInsufficientSpacePool, 15 | kInsufficientSpaceIndex, 16 | kExists, 17 | kNotEven, 18 | kNotFound, 19 | kPartialValue, 20 | kNotProcessed, 21 | kNotSupported, 22 | kTimedOut, 23 | kRejected, 24 | }; 25 | 26 | static std::string ResultString(enum Result r) 27 | { 28 | switch(r) { 29 | case Result::kSuccess: 30 | return std::string("Success"); 31 | case Result::kLocked: 32 | return std::string("Locked"); 33 | case Result::kError: 34 | return std::string("Error"); 35 | case Result::kInsufficientSpacePool: 36 | return std::string("Insufficient space in pool"); 37 | case Result::kInsufficientSpaceIndex: 38 | return std::string("Insufficient space in index"); 39 | case Result::kExists: 40 | return std::string("Exists"); 41 | case Result::kNotEven: 42 | return std::string("Value not even (F&A)"); 43 | case Result::kNotFound: 44 | return std::string("Not found"); 45 | case Result::kPartialValue: 46 | return std::string("Partial value"); 47 | case Result::kNotProcessed: 48 | return std::string("Not processed"); 49 | case Result::kNotSupported: 50 | return std::string("Not supported"); 51 | case Result::kTimedOut: 52 | return std::string("Timed out"); 53 | case Result::kRejected: 54 | return std::string("Rejected"); 55 | default: 56 | return std::string("Invalid ::mica::table Result type"); 57 | }; 58 | } 59 | 60 | } 61 | } 62 | 63 | #endif 64 | --------------------------------------------------------------------------------