├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cluster ├── README.md ├── TXDashboard.json ├── clusterstate.py ├── configs │ ├── cluster.cfg │ ├── common.cfg │ ├── cpo.cfg │ ├── nodepool.cfg │ ├── persist.cfg │ ├── tpcc_client.cfg │ ├── tpcc_load.cfg │ ├── tso.cfg │ ├── txnbench.cfg │ └── txnload.cfg ├── inspect │ ├── Inspect.py │ └── README.md ├── locals.py ├── parse.py └── run.py ├── docker ├── Dockerfile └── README.md ├── docs ├── CODING_STYLE.md ├── METRICS.md ├── MILESTONES.md ├── NOTES.md ├── PERFORMANCE.md ├── PERSISTENCE.md ├── README.md ├── RFC │ ├── DROP_COLLECTION.md │ ├── DROP_SCHEMA.md │ ├── K23SI_READ_COMMITTED.md │ ├── K23SI_TESTING.md │ ├── K2_NODE_OVERVIEW.md │ ├── KEY_ENCODING.md │ ├── PARTITION_TESTING.md │ ├── QUERY_TESTING.md │ ├── README.md │ ├── RESEARCH_AREAS.md │ └── SCHEMA_VERSIONING.md ├── design │ ├── FILTER_EXPRESSIONS.md │ ├── K23SI_CLIENT.md │ ├── K23SI_TXN.md │ ├── K2PROCESS.md │ ├── K2_BASIC_CONCEPTS.md │ ├── SKV.md │ ├── TRANSPORT.md │ └── TSO.md ├── images │ ├── Chogori-node-overview-pic1.pptx │ ├── Chogori-node-overview.png │ ├── Data_Deployment.gliffy │ ├── Data_Deployment.png │ ├── HighLevelTxn.gliffy │ ├── HighLevelTxn.png │ ├── JepsenConsistency.jpg │ ├── K23SI-Testing Scenario-04.png │ ├── K2Collection.gliffy │ ├── K2Collection.png │ ├── K2NodePool.gliffy │ ├── K2NodePool.png │ ├── K2process.gliffy │ ├── K2process.png │ ├── TSO-High.gliffy │ ├── TSO-High.png │ ├── TSO-Low-multi.gliffy │ ├── TSO-Low-multi.png │ ├── TSO-Low.gliffy │ ├── TSO-Low.png │ ├── TSO.gliffy │ ├── TSO.png │ ├── TWIM_States.gliffy │ ├── TWIM_States.png │ ├── TWIM_abandoned.gliffy │ ├── TWIM_abandoned.png │ ├── TWIM_abort.gliffy │ ├── TWIM_abort.png │ ├── TWIM_commit.gliffy │ ├── TWIM_commit.png │ ├── TWIM_iprpip_abort.gliffy │ ├── TWIM_iprpip_abort.png │ ├── TimestampRefresh.gliffy │ ├── TimestampRefresh.png │ ├── TxnArchitecture.gliffy │ ├── TxnArchitecture.png │ ├── TxnBegin.gliffy │ ├── TxnBegin.png │ ├── TxnCommit.gliffy │ ├── TxnCommit.png │ ├── TxnRead.gliffy │ ├── TxnRead.png │ ├── TxnReadConflict.gliffy │ ├── TxnReadConflict.png │ ├── TxnStates.gliffy │ ├── TxnStates.png │ ├── TxnStates_Abandoned.gliffy │ ├── TxnStates_Abandoned.png │ ├── TxnStates_Abort.gliffy │ ├── TxnStates_Abort.png │ ├── TxnStates_Commit.gliffy │ ├── TxnStates_Commit.png │ ├── TxnStates_RetentionWindow.gliffy │ ├── TxnStates_RetentionWindow.png │ ├── TxnWrite.gliffy │ ├── TxnWrite.png │ ├── TxnWriteConflict.gliffy │ ├── TxnWriteConflict.png │ ├── time_sync.png │ └── time_sync.puml ├── papers │ ├── SerializableSnapshotIsolation-fekete-sigmod2008.pdf │ ├── YCSB+T.pdf │ ├── harding.pdf │ ├── sinfonia transactions.pdf │ └── warp.pdf ├── perf │ ├── README.md │ ├── TPCCLatency.txt │ ├── TPCCThroughput.txt │ ├── YCSB-A-Latency.txt │ ├── YCSB-A-Throughput.txt │ ├── YCSB-B-Latency.txt │ └── YCSB-B-Throughput.txt └── presentations │ ├── Chogori Riselab2020Aug.pptx │ ├── Chogori-SAP-2020Aug.pptx │ └── DFV- Chogori SIGMOD20.pptx ├── install_deps.sh ├── src ├── CMakeLists.txt ├── k2 │ ├── CMakeLists.txt │ ├── README.md │ ├── appbase │ │ ├── AppEssentials.h │ │ ├── Appbase.cpp │ │ ├── Appbase.h │ │ └── CMakeLists.txt │ ├── assignmentManager │ │ ├── AssignmentManager.cpp │ │ ├── AssignmentManager.h │ │ └── CMakeLists.txt │ ├── cmd │ │ ├── CMakeLists.txt │ │ ├── controlPlaneOracle │ │ │ ├── CMakeLists.txt │ │ │ └── cpo_main.cpp │ │ ├── demo │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── rpc_echo.cpp │ │ │ ├── rpc_service.cpp │ │ │ ├── service.cpp │ │ │ └── tso_sample_app.cpp │ │ ├── httpproxy │ │ │ ├── CMakeLists.txt │ │ │ └── http_proxy_main.cpp │ │ ├── nodepool │ │ │ ├── CMakeLists.txt │ │ │ └── nodepool_main.cpp │ │ ├── persistence │ │ │ ├── CMakeLists.txt │ │ │ └── persistence_main.cpp │ │ ├── plog │ │ │ ├── CMakeLists.txt │ │ │ └── plog_main.cpp │ │ ├── tpcc │ │ │ ├── CMakeLists.txt │ │ │ ├── Log.h │ │ │ ├── README │ │ │ ├── datagen.h │ │ │ ├── dataload.h │ │ │ ├── schema.h │ │ │ ├── tpcc_client.cpp │ │ │ ├── tpcc_rand.h │ │ │ ├── transactions.h │ │ │ ├── verify.cpp │ │ │ └── verify.h │ │ ├── tso │ │ │ ├── CMakeLists.txt │ │ │ └── tso_main.cpp │ │ ├── txbench │ │ │ ├── CMakeLists.txt │ │ │ ├── Log.h │ │ │ ├── k23sibench_client.cpp │ │ │ ├── rpcbench_client.cpp │ │ │ ├── rpcbench_common.h │ │ │ ├── rpcbench_server.cpp │ │ │ ├── txbench_client.cpp │ │ │ ├── txbench_combine.cpp │ │ │ ├── txbench_common.h │ │ │ └── txbench_server.cpp │ │ └── ycsb │ │ │ ├── CMakeLists.txt │ │ │ ├── Log.h │ │ │ ├── README │ │ │ ├── data.h │ │ │ ├── dataload.h │ │ │ ├── transactions.h │ │ │ ├── ycsb_client.cpp │ │ │ └── ycsb_rand.h │ ├── common │ │ ├── CMakeLists.txt │ │ ├── Common.h │ │ ├── Defer.h │ │ ├── ExpiryList.h │ │ ├── Log.h │ │ ├── MapUtil.h │ │ ├── SpinMutex.h │ │ ├── Timer.h │ │ ├── TypeMap.h │ │ └── VecUtil.h │ ├── config │ │ ├── CMakeLists.txt │ │ └── Config.h │ ├── cpo │ │ ├── CMakeLists.txt │ │ ├── client │ │ │ ├── CMakeLists.txt │ │ │ ├── Client.cpp │ │ │ ├── Client.h │ │ │ ├── Heartbeat.cpp │ │ │ └── Heartbeat.h │ │ └── service │ │ │ ├── CMakeLists.txt │ │ │ ├── HealthMonitor.cpp │ │ │ ├── HealthMonitor.h │ │ │ ├── Log.h │ │ │ ├── Service.cpp │ │ │ └── Service.h │ ├── dto │ │ ├── AssignmentManager.h │ │ ├── CMakeLists.txt │ │ ├── Collection.cpp │ │ ├── Collection.h │ │ ├── ControlPlaneOracle.cpp │ │ ├── ControlPlaneOracle.h │ │ ├── Expression.cpp │ │ ├── Expression.h │ │ ├── FieldTypes.cpp │ │ ├── FieldTypes.h │ │ ├── K23SI.cpp │ │ ├── K23SI.h │ │ ├── K23SIInspect.h │ │ ├── Log.h │ │ ├── LogStream.h │ │ ├── MessageVerbs.h │ │ ├── Persistence.h │ │ ├── PersistenceCluster.h │ │ ├── README.md │ │ ├── SKVRecord.cpp │ │ ├── SKVRecord.h │ │ ├── TSO.h │ │ ├── Timestamp.cpp │ │ └── Timestamp.h │ ├── httpproxy │ │ ├── CMakeLists.txt │ │ ├── HTTPProxy.cpp │ │ └── HTTPProxy.h │ ├── indexer │ │ ├── CMakeLists.txt │ │ ├── HOTIndexer.h │ │ ├── IndexerInterface.h │ │ ├── MapIndexer.h │ │ └── UnorderedMapIndexer.h │ ├── infrastructure │ │ ├── APIServer.cpp │ │ ├── APIServer.h │ │ └── CMakeLists.txt │ ├── module │ │ ├── CMakeLists.txt │ │ └── k23si │ │ │ ├── CMakeLists.txt │ │ │ ├── Config.h │ │ │ ├── Indexer.cpp │ │ │ ├── Indexer.h │ │ │ ├── Log.h │ │ │ ├── Module.cpp │ │ │ ├── Module.h │ │ │ ├── Persistence.cpp │ │ │ ├── Persistence.h │ │ │ ├── TxnManager.cpp │ │ │ ├── TxnManager.h │ │ │ ├── TxnWIMetaManager.cpp │ │ │ ├── TxnWIMetaManager.h │ │ │ └── client │ │ │ ├── CMakeLists.txt │ │ │ ├── Log.h │ │ │ ├── k23si_client.cpp │ │ │ ├── k23si_client.h │ │ │ ├── query.cpp │ │ │ └── query.h │ ├── persistence │ │ ├── .DS_Store │ │ ├── CMakeLists.txt │ │ ├── logStream │ │ │ ├── CMakeLists.txt │ │ │ ├── LogStream.cpp │ │ │ └── LogStream.h │ │ ├── plog_client │ │ │ ├── CMakeLists.txt │ │ │ ├── PlogClient.cpp │ │ │ └── PlogClient.h │ │ ├── plog_service │ │ │ ├── CMakeLists.txt │ │ │ ├── PlogServer.cpp │ │ │ └── PlogServer.h │ │ └── service │ │ │ ├── CMakeLists.txt │ │ │ ├── PersistenceService.cpp │ │ │ └── PersistenceService.h │ ├── transport │ │ ├── AutoRRDMARPCProtocol.cpp │ │ ├── AutoRRDMARPCProtocol.h │ │ ├── BaseTypes.h │ │ ├── CMakeLists.txt │ │ ├── Discovery.cpp │ │ ├── Discovery.h │ │ ├── DiscoveryDTO.h │ │ ├── IRPCProtocol.cpp │ │ ├── IRPCProtocol.h │ │ ├── Log.h │ │ ├── Payload.cpp │ │ ├── Payload.h │ │ ├── PayloadFileUtil.cpp │ │ ├── PayloadFileUtil.h │ │ ├── PayloadSerialization.h │ │ ├── Prometheus.cpp │ │ ├── Prometheus.h │ │ ├── README.md │ │ ├── RPCDispatcher.cpp │ │ ├── RPCDispatcher.h │ │ ├── RPCHeader.cpp │ │ ├── RPCHeader.h │ │ ├── RPCParser.cpp │ │ ├── RPCParser.h │ │ ├── RPCProtocolFactory.cpp │ │ ├── RPCProtocolFactory.h │ │ ├── RPCTypes.h │ │ ├── RRDMARPCChannel.cpp │ │ ├── RRDMARPCChannel.h │ │ ├── RRDMARPCProtocol.cpp │ │ ├── RRDMARPCProtocol.h │ │ ├── Request.cpp │ │ ├── Request.h │ │ ├── RetryStrategy.cpp │ │ ├── RetryStrategy.h │ │ ├── Status.cpp │ │ ├── Status.h │ │ ├── TCPRPCChannel.cpp │ │ ├── TCPRPCChannel.h │ │ ├── TCPRPCProtocol.cpp │ │ ├── TCPRPCProtocol.h │ │ ├── TXEndpoint.cpp │ │ ├── TXEndpoint.h │ │ ├── Util.h │ │ ├── VirtualNetworkStack.cpp │ │ └── VirtualNetworkStack.h │ └── tso │ │ ├── CMakeLists.txt │ │ ├── client │ │ ├── CMakeLists.txt │ │ ├── Client.cpp │ │ └── Client.h │ │ └── service │ │ ├── CMakeLists.txt │ │ ├── Clock.cpp │ │ ├── Clock.h │ │ ├── Exceptions.h │ │ ├── Log.h │ │ ├── Service.cpp │ │ └── Service.h ├── logging │ ├── CMakeLists.txt │ └── src │ │ ├── CMakeLists.txt │ │ └── k2 │ │ ├── CMakeLists.txt │ │ └── logging │ │ ├── AutoGenFormattingUtils.h │ │ ├── CMakeLists.txt │ │ ├── Chrono.h │ │ ├── Decimal.h │ │ ├── FormattingUtils.h │ │ ├── Log.h │ │ ├── MacroUtils.h │ │ └── gen_macros.py └── skvhttpclient │ ├── CMakeLists.txt │ ├── src │ ├── CMakeLists.txt │ └── skvhttp │ │ ├── CMakeLists.txt │ │ ├── client │ │ ├── CMakeLists.txt │ │ ├── SKVClient.cpp │ │ └── SKVClient.h │ │ ├── common │ │ ├── Binary.h │ │ ├── CMakeLists.txt │ │ ├── Common.h │ │ ├── Serialization.h │ │ ├── Status.cpp │ │ ├── Status.h │ │ └── VecUtil.h │ │ ├── dto │ │ ├── CMakeLists.txt │ │ ├── Collection.cpp │ │ ├── Collection.h │ │ ├── ControlPlaneOracle.cpp │ │ ├── ControlPlaneOracle.h │ │ ├── Expression.cpp │ │ ├── Expression.h │ │ ├── FieldTypes.cpp │ │ ├── FieldTypes.h │ │ ├── K23SI.h │ │ ├── Log.h │ │ ├── SKVRecord.cpp │ │ ├── SKVRecord.h │ │ ├── Timestamp.cpp │ │ └── Timestamp.h │ │ ├── httplib │ │ ├── CMakeLists.txt │ │ ├── httplib.cpp │ │ └── httplib.h │ │ └── mpack │ │ ├── CMakeLists.txt │ │ ├── MPackSerialization.h │ │ ├── mpack.cpp │ │ └── mpack.h │ └── test │ ├── CMakeLists.txt │ ├── ClientIntegrationTest.cpp │ ├── ExpressionTest.cpp │ ├── SKVRecordTest.cpp │ ├── include │ └── catch.hpp │ └── mpacktest.cpp └── test ├── CMakeLists.txt ├── common ├── CMakeLists.txt └── CommonTest.cpp ├── cpo ├── CMakeLists.txt ├── CPOClientTest.cpp ├── CPOTest.cpp ├── CPOTest.h └── Main.cpp ├── dto ├── CMakeLists.txt └── PartitionTest.cpp ├── include ├── TestUtil.h └── catch2 │ ├── catch.hpp │ ├── catch_reporter_automake.hpp │ ├── catch_reporter_tap.hpp │ └── catch_reporter_teamcity.hpp ├── integration ├── CMakeLists.txt ├── common_defs.sh ├── failing_test_logstream.sh ├── heartbeat_monitor.py ├── run.sh ├── run_k2_cluster.sh ├── skvclient.py ├── test_3si_txn.sh ├── test_collection.sh ├── test_cpo_tso_assign.py ├── test_heartbeat_monitor.sh ├── test_http.py ├── test_http_client.sh ├── test_http_cpp_client.sh ├── test_k23si.sh ├── test_k23si_tpcc.sh ├── test_k23si_ycsb.sh ├── test_partition.sh ├── test_partition_assigned.sh ├── test_plog.sh ├── test_query.sh ├── test_schema_create.sh ├── test_skv_client.sh ├── test_tso_assignment.sh ├── test_tso_errorbound.sh └── test_tso_errorbound_fail.py ├── k23si ├── 3SITxnTest.cpp ├── CMakeLists.txt ├── ExpressionTest.cpp ├── IndexerTest.cpp ├── K23SITest.cpp ├── KeyEncodingTest.cpp ├── Log.h ├── QueryTest.cpp ├── SKVClientTest.cpp ├── SKVRecordTest.cpp ├── SKVSerTest.cpp └── SchemaCreationTest.cpp ├── plog ├── CMakeLists.txt ├── LogStreamTest.cpp └── PlogClientTest.cpp └── transport ├── CMakeLists.txt ├── EndpointTest.cpp └── PayloadTest.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .editorconfig 3 | .vscode/ 4 | build/ 5 | deps/ 6 | *.rpm 7 | *.deb 8 | *.changes 9 | *.iml 10 | k2b 11 | cluster/__pycache__/ 12 | test/integration/__pycache__/ 13 | *.log 14 | *.swp 15 | *.swa 16 | *.swo 17 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/.gitmodules -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.13) 2 | 3 | project (k2_platform VERSION 0.0.1 LANGUAGES CXX) 4 | set(CMAKE_CXX_STANDARD 17) 5 | add_compile_options(-O3 -g -Werror -Wall -Wextra -march=skylake -mtune=skylake-avx512 -mavx -mavx2 -mxsave -mbmi -mbmi2 -mlzcnt -maes -mpopcnt) 6 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 7 | 8 | # K2_VERBOSE_LOGGING enables some verbose logging for debug purposes. By default it is disabled and compiled out 9 | if(DEFINED ENV{K2_VERBOSE_LOGGING}) 10 | set(K2_VERBOSE_LOGGING $ENV{K2_VERBOSE_LOGGING}) 11 | 12 | else() 13 | set(K2_VERBOSE_LOGGING 0) 14 | endif() 15 | 16 | add_compile_definitions(K2_VERBOSE_LOGGING=${K2_VERBOSE_LOGGING}) 17 | 18 | # K2_MODULE_POOL_ALLOCATOR enables pool allocator for the module indexer. By default it is disabled 19 | if(DEFINED ENV{K2_MODULE_POOL_ALLOCATOR}) 20 | message("Found variable K2_MODULE_POOL_ALLOCATOR = '$ENV{K2_MODULE_POOL_ALLOCATOR}'") 21 | set(K2_MODULE_POOL_ALLOCATOR $ENV{K2_MODULE_POOL_ALLOCATOR}) 22 | else() 23 | set(K2_MODULE_POOL_ALLOCATOR 0) 24 | endif() 25 | 26 | 27 | add_compile_definitions(K2_MODULE_POOL_ALLOCATOR=${K2_MODULE_POOL_ALLOCATOR}) 28 | 29 | include_directories(src) 30 | 31 | find_package(Seastar REQUIRED) 32 | find_package(fmt REQUIRED) 33 | find_package(skvhttp REQUIRED) 34 | find_package(yule REQUIRED) 35 | 36 | add_subdirectory (src) 37 | add_subdirectory (test) 38 | 39 | # install the cmake config for the entire project so that it can be used by upstream 40 | # via find_package(k2 REQUIRED) 41 | # and pulling individual dependencies like so: 42 | # DEPENDS ..... k2::appbase k2::transport 43 | install( 44 | EXPORT k2Targets 45 | FILE k2-config.cmake 46 | DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/k2" 47 | NAMESPACE k2:: 48 | ) 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Futurewei Cloud 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | # chogori-platform 6 | K2 Project is a platform for building low-latency (μs) in-memory distributed persistent OLTP databases. 7 | 8 | This repository contains implementations for K2 core services and subsystems (transport, persistence, etc.). 9 | 10 | For more interactive discussions, news, planning and other questions, please visit our discussion board here: 11 | https://groups.google.com/forum/#!forum/chogori-dev 12 | 13 | ## Build instructions 14 | 15 | ### Install instructions 16 | * Install chogori-seastar-rd (see instructions in the repo https://github.com/futurewei-cloud/chogori-seastar-rd) 17 | * run `./install_deps.sh` to install other dependency libraries 18 | * build and install the cmake subprojects under src/logging and src/skvhttpclient 19 | * generate cmake and build `mkdir build && cd build && cmake .. && make -j` 20 | * run tests `cd build/test && ctest` 21 | * run integration tests `./test/integration/run.sh` 22 | -------------------------------------------------------------------------------- /cluster/README.md: -------------------------------------------------------------------------------- 1 | This directory contains basic python scripts and configuration files to help run Chogori 2 | performance benchmarks on a cluster of machines. 3 | 4 | 5 | Requirements: 6 | - passwordless ssh between machine running script and target machines 7 | - passwordless sudo on target machine 8 | - python 3 9 | - python fabric library, e.g. "sudo apt install python3-pip && pip3 install fabric" 10 | - Docker repo with images that contain the Chogori executables 11 | 12 | 13 | run.py is the top-level script for running the benchmarks. It has commands to start, stop, remove, and 14 | get logs for the cluster components. For example the full sequence of commands to run the TPC-C benchmark 15 | could look like this: 16 | 1. ./run.py --config\_file configs/cluster.cfg --start cpo tso persist nodepool 17 | 2. ./run.py --config\_file configs/cluster.cfg --start load 18 | 3. ./run.py --config\_file configs/cluster.cfg --log load 19 | 4. ./run.py --config\_file configs/cluster.cfg --stop load 20 | 5. ./run.py --config\_file configs/cluster.cfg --remove load 21 | 6. ./run.py --config\_file configs/cluster.cfg --start client 22 | 7. ./run.py --config\_file configs/cluster.cfg --log client 23 | 8. ./run.py --config\_file configs/cluster.cfg --stop cpo tso persist nodepool client 24 | 9. ./run.py --config\_file configs/cluster.cfg --remove cpo tso persist nodepool client 25 | 26 | 27 | Each item listed after --start will start a docker container on the cluster. --stop will stop containers 28 | but not remove them (so the logs can still be examined), and --remove removes the container. The file 29 | specified with --config\_file is a top-level configuration file that describes each cluster component 30 | and where and how to run. The locals.py file has local configuration information that 31 | must be modified for every cluster environment; it contains hostname URLs and docker image URLs. 32 | 33 | 34 | Individual configuration files (e.g. configs/cpo.cfg, etc.) contain docker and seastar specific options, 35 | such as the amount of memory to use. They also contain placeholders for options that are automatically 36 | configured by the script, for example the TCP endpoints. These placeholder options are prefaced with "$$". 37 | 38 | 39 | Some guidelines for performance benchmarking: 40 | - Use poll-mode seastar option 41 | - Configure 2MB hugepages (via hugeadm Linux command) and enable the seastar option hugepages 42 | - Use RDMA if available with seastar rmda option 43 | - Use seastar cpuset option to pin to cores, ideally without using hyperthreading 44 | - Turn off swap with Linux "swapoff -a" 45 | -------------------------------------------------------------------------------- /cluster/configs/cluster.cfg: -------------------------------------------------------------------------------- 1 | [cpo] 2 | hosts = 1 3 | host_config = A 4 | cpus = 1 5 | configs = configs/cpo.cfg configs/common.cfg 6 | 7 | [tso] 8 | hosts = 1 9 | host_config = A 10 | cpus = 10 11 | configs = configs/tso.cfg configs/common.cfg 12 | 13 | [persist] 14 | hosts = 1 15 | host_config = A 16 | cpus = 8 17 | configs = configs/persist.cfg configs/common.cfg 18 | 19 | [nodepool] 20 | hosts = 2 21 | host_config = A 22 | cpus = 4 23 | configs = configs/nodepool.cfg configs/common.cfg 24 | 25 | [txnload] 26 | hosts = 1 27 | host_config = A 28 | cpus = 1 29 | configs = configs/txnload.cfg configs/common.cfg 30 | 31 | [txnbench] 32 | hosts = 1 33 | host_config = A 34 | cpus = 10 35 | configs = configs/txnbench.cfg configs/common.cfg 36 | 37 | [client] 38 | hosts = 1 39 | host_config = A 40 | cpus = 1 41 | configs = configs/tpcc_client.cfg configs/common.cfg 42 | 43 | [load] 44 | hosts = 1 45 | host_config = A 46 | cpus = 1 47 | configs = configs/tpcc_load.cfg configs/common.cfg 48 | 49 | [netserver] 50 | hosts = 1 51 | host_config = B 52 | cpus = 5 53 | configs = configs/netserver.cfg configs/common.cfg 54 | 55 | [netclient] 56 | hosts = 1 57 | host_config = B 58 | cpus = 5 59 | configs = configs/netclient.cfg configs/common.cfg 60 | 61 | [rpcserver] 62 | hosts = 1 63 | host_config = A 64 | cpus = 10 65 | configs = configs/rpcserver.cfg configs/common.cfg 66 | 67 | [rpcclient] 68 | hosts = 1 69 | host_config = A 70 | cpus = 10 71 | configs = configs/rpcclient.cfg configs/common.cfg 72 | -------------------------------------------------------------------------------- /cluster/configs/common.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | docker_args = --privileged --network=host -v "/dev/:/dev" -v "/sys/:/sys/" -w /usr/local/bin/ -d -e RDMAV_HUGEPAGES_SAFE=1 3 | 4 | [program_args] 5 | hugepages= 6 | poll-mode= 7 | rdma = $$rdma 8 | cpuset = $$cpus_expand 9 | smp = $$cpus 10 | reactor-backend = epoll 11 | -------------------------------------------------------------------------------- /cluster/configs/cpo.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./cpo_main 4 | 5 | [program_args] 6 | tcp_endpoints = $$my_endpoints 7 | data_dir = ./ 8 | memory = 1G 9 | txn_heartbeat_deadline = 1s 10 | assignment_timeout = 10s 11 | -------------------------------------------------------------------------------- /cluster/configs/nodepool.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./nodepool 4 | 5 | [program_args] 6 | k23si_cpo_endpoint = $$cpo_endpoints 7 | k23si_persistence_endpoints = $$persist_endpoints 8 | tcp_endpoints = $$my_endpoints 9 | tso_endpoint = $$tso_endpoints 10 | memory = 30G 11 | partition_request_timeout = 10s 12 | cpo_request_timeout = 5s 13 | cpo_request_backoff = 1s 14 | -------------------------------------------------------------------------------- /cluster/configs/persist.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./persistence 4 | 5 | [program_args] 6 | tcp_endpoints = $$my_endpoints 7 | memory = 20G 8 | -------------------------------------------------------------------------------- /cluster/configs/tpcc_client.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./tpcc_client 4 | 5 | [program_args] 6 | test_duration_s = 150 7 | tcp_remotes = $$nodepool_endpoints 8 | cpo = $$cpo_endpoints 9 | tso_endpoint = $$tso_endpoints 10 | memory = 30G 11 | partition_request_timeout = 1s 12 | cpo_request_timeout = 5s 13 | cpo_request_backoff = 1s 14 | data_load = false 15 | num_warehouses = 12 16 | num_concurrent_txns = 1 17 | do_verification = false 18 | txn_weights = {43,4,4,45,4} 19 | -------------------------------------------------------------------------------- /cluster/configs/tpcc_load.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./tpcc_client 4 | 5 | [program_args] 6 | test_duration_s = 30 7 | tcp_remotes = $$nodepool_endpoints 8 | cpo = $$cpo_endpoints 9 | tso_endpoint = $$tso_endpoints 10 | memory = 30G 11 | partition_request_timeout = 20s 12 | cpo_request_timeout = 5s 13 | cpo_request_backoff = 1s 14 | data_load = true 15 | num_warehouses = 12 16 | num_concurrent_txns = 1 17 | do_verification = false 18 | txn_weights = {43,4,4,45,4} 19 | -------------------------------------------------------------------------------- /cluster/configs/tso.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./tso 4 | 5 | [program_args] 6 | tcp_endpoints = $$my_endpoints 7 | memory = 2G 8 | -------------------------------------------------------------------------------- /cluster/configs/txnbench.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./k23sibench_client 4 | 5 | [program_args] 6 | test_duration = 120s 7 | tcp_remotes = $$nodepool_endpoints 8 | cpo = $$cpo_endpoints 9 | tso_endpoint = $$tso_endpoints 10 | memory = 30G 11 | partition_request_timeout = 1s 12 | cpo_request_timeout = 5s 13 | cpo_request_backoff = 1s 14 | pipeline_depth = 1 15 | txn_timeout = 1s 16 | reads = 1 17 | writes = 2 18 | sync_finalize = false 19 | -------------------------------------------------------------------------------- /cluster/configs/txnload.cfg: -------------------------------------------------------------------------------- 1 | [deployment] 2 | image = docker.mydomain.com/tpcc:latest 3 | binary = ./k23sibench_client 4 | 5 | [program_args] 6 | test_duration = 120s 7 | tcp_remotes = $$nodepool_endpoints 8 | cpo = $$cpo_endpoints 9 | tso_endpoint = $$tso_endpoints 10 | memory = 30G 11 | partition_request_timeout = 1s 12 | cpo_request_timeout = 5s 13 | cpo_request_backoff = 1s 14 | pipeline_depth = 1 15 | txn_timeout = 1s 16 | reads = 0 17 | writes = 4 18 | sync_finalize = true 19 | -------------------------------------------------------------------------------- /cluster/inspect/Inspect.py: -------------------------------------------------------------------------------- 1 | ''' 2 | MIT License 3 | 4 | Copyright (c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | ''' 24 | 25 | import requests, json 26 | from urllib.parse import urlparse 27 | import argparse 28 | 29 | parser = argparse.ArgumentParser(description="Utility for accessing K2 API functions") 30 | parser.add_argument("--cpo", help="API URL for CPO") 31 | args = parser.parse_args() 32 | 33 | class CPO: 34 | def __init__(self, cpo_url): 35 | cpo = urlparse(cpo_url) 36 | self.cpo = cpo.geturl() 37 | 38 | def getCollection(self, name): 39 | url = self.cpo + "/api/CollectionGet" 40 | data = {"name": name} 41 | r = requests.post(url, data=json.dumps(data)) 42 | return r.json() 43 | 44 | def getSchemas(self, name): 45 | url = self.cpo + "/api/GetSchemas" 46 | data = {"collectionName": name} 47 | r = requests.post(url, data=json.dumps(data)) 48 | return r.json() 49 | 50 | -------------------------------------------------------------------------------- /cluster/inspect/README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | These scripts provide easy access to K2's JSON API server over HTTP. The API server provides access 3 | to functions such as getting collections or schemas or getting all keys on a K2 server. 4 | 5 | The library is designed to be used through the REPL. Command line arguments can be passed like this: 6 | "python3 - --cpo http://localhost:19000" 7 | 8 | ## Requirements 9 | python3 and requests library: "python3 -m pip install requests" 10 | -------------------------------------------------------------------------------- /cluster/locals.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict 2 | 3 | 4 | # The smallest unit of deployment target. It is comprised of a single 5 | # network port and its fair share of cores. 6 | class HostNode: 7 | def __init__(self, dns: str, fastip: str, cores: List[int], rdma: str, config: str): 8 | """ 9 | :param dns: DNS name of host 10 | :param fastip: IP address of the network port to listen on 11 | :param cores: List of core IDs to use 12 | :param rdma: Name of RDMA port to use, e.g. "mlx5_0" 13 | :param config: Hardware config of the server 14 | """ 15 | self.dns = dns 16 | self.fastip = fastip 17 | self.cores = cores 18 | self.rdma = rdma 19 | self.config = config 20 | 21 | 22 | host_nodes: List[HostNode] = [ 23 | HostNode("server-1.mydomain.com", "192.168.1.2", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 24 | HostNode("server-2.mydomain.com", "192.168.1.4", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 25 | HostNode("server-3.mydomain.com", "192.168.1.6", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 26 | HostNode("server-4.mydomain.com", "192.168.1.8", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 27 | HostNode("server-5.mydomain.com", "192.168.1.10", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 28 | HostNode("server-6.mydomain.com", "192.168.1.12", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 29 | HostNode("server-7.mydomain.com", "192.168.1.14", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 30 | HostNode("server-8.mydomain.com", "192.168.1.16", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 31 | HostNode("server-9.mydomain.com", "192.168.1.18", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A"), 32 | HostNode("server-10.mydomain.com", "192.168.1.20", [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], "mlx5_0", "A")] 33 | 34 | port_bases: Dict[str, int] = {"cpo": 7000, "tso": 8000, "persist": 4000, "nodepool": 10000} 35 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | RUN apt update && apt upgrade -y 5 | RUN apt install -y build-essential cmake vim gdb strace psmisc pkg-config python3 python3-pip git sed 6 | RUN pip3 install ipython requests msgpack 7 | 8 | CMD ["/bin/bash"] 9 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | Before we can build the platform, we need an environment with all dependencies pre-installed. These instructions help you create this environment using Docker. 2 | # Preparation 3 | 4 | ## Building the image 5 | assuming you checked-out chogori-platform under some folder, say `workspace` 6 | ``` sh 7 | cd workspace/chogori-platform/docker 8 | docker build -t "chogori-builder" . 9 | ``` 10 | 11 | ## Building and installing Chogori-Seastar-RD 12 | This is a fundamental dependency for the project and has to be installed in order to build and run the platform. 13 | ### Clone the repo 14 | ``` sh 15 | cd workspace && git clone https://github.com/futurewei-cloud/chogori-seastar-rd.git 16 | ``` 17 | ### Run the container from the image we built above 18 | This mounts your host current directory `PWD` into the `/host` directory of the container. Any changes you do in the container inside the `/host` directory are reflected into the host (much like a symlink) 19 | ``` sh 20 | cd workspace/chogori-seastar-rd 21 | docker run -it --rm --init -v ${PWD}:/host chogori-builder 22 | ``` 23 | 24 | ### Run the chogori-seastar-rd installation steps inside this container 25 | ``` sh 26 | cd /host 27 | ./install-dependencies.sh 28 | ./configure.py --mode=release 29 | ninja -C build/release install 30 | ``` 31 | 32 | # Building and installing Chogori-Platform 33 | We're now ready to build the platform 34 | 35 | 36 | ## And now run the platform build and installation steps inside this container 37 | ``` sh 38 | cd /host 39 | ./install_deps.sh 40 | ``` 41 | ## Persisting the container-local changes 42 | at this time, we have a running container with the seastar-rd dependency installed. We should now save the container state so that we can restart it if we want with the dependency already pre-installed. 43 | Do not exit the container from above just yet. Instead, go to another shell and run the command: 44 | ``` sh 45 | $ docker container ls 46 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 47 | b34f8a36a25b chogori-builder "/bin/bash" 6 minutes ago Up 6 minutes determined_wilson 48 | $ docker commit b34f8a36a25b chogori-builder 49 | ``` 50 | This will overwrite the container with the latest state. The actual ID would be different for you as it is randomly generated. 51 | Now you can exit the container in your original session if you'd like 52 | 53 | ## Building the platform code 54 | ``` sh 55 | mkdir build && cd build && cmake ../ && make -j && cd - 56 | ``` 57 | 58 | To run the tests and install: 59 | ``` sh 60 | cd test 61 | ctest 62 | cd ../ 63 | make install 64 | ``` 65 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Various documentation for the Chogori-Platform (a.k.a. K2) storage project. 2 | 3 | - Design docs are found in the `design` folder 4 | - Presentations which the team has done are in the `presentations` folder 5 | - The `papers` folder contains interesting and relevant papers published from other projects 6 | - The `RFC` folder is the landing area for new ideas which we may consider adopting. Once adopted, we would move them to the `design` folder 7 | -------------------------------------------------------------------------------- /docs/RFC/PARTITION_TESTING.md: -------------------------------------------------------------------------------- 1 | This document describes test scenarios for the Partition requests. 2 | 3 | # Background 4 | 5 | The test cases currently are constructed in a form of an integration test. The tests are in the test runner `chogori-platform/test/dto/PartitionTest.cpp`, which is a stand-alone application that can send/receive partition related requests against a K2 cluster. The tests are currently executed via the integration tester `chogori-platform/test/integration/test_partition.sh` 6 | 7 | New tests can be added to the existing test runner in the same manner as the existing tests 8 | 9 | Common notes: 10 | 11 | - Generate keys with scenario prefix for range/partition, e.g. keys generated in scenario 05 are all prefixed with SC05_ 12 | - Use the Inspect verbs to validate state of transactions and data 13 | - We may have to re-initialize a cluster in some cases. It is probably best for now to just start a new cluster with a new collection, creating collections with say a timestamp prefix so that we don't get collection name collisions 14 | - There are global configurations which multiply the test cases. These are listed here 15 | - Hash/Range partitioning: we should repeat all tests and make sure they pass for both hash and range-based collections 16 | 17 | # Test scenarios 18 | 19 | ## Scenario 01 - get partition for key through range scheme 20 | 21 | ### Test setup 22 | 23 | - start a cluster with three k2 nodes. 24 | - assign collection with range partition. Set the range of each partition as follows: ["", c), [c, e), [e, "") 25 | 26 | ### Test cases 27 | 28 | | test case | Expected result | Possible fix needed | 29 | | ------------------------------------------------------------ | ----------------------------------- | ------------------- | 30 | | using a null-empty key to get Partition with default reverse and exclusiveKey flag | partition that key belongs | | 31 | | using an empty key to get Partition with default reverse and exclusiveKey flag | First partition | | 32 | | using a null-empty key to get Partition with reverse flag set to be TRUE and default exclusiveKey flag | partition that key belongs | | 33 | | using an empty key to get Partition with reverse flag set to be TRUE and default exclusiveKey flag | last partition | | 34 | | using a null-empty key to get Partition with reverse and exclusiveKey flag set to be TRUE. the key is NOT the Start key of any partitions. | partition that key belongs | | 35 | | using a null-empty key to get Partition with reverse and exclusiveKey flag set to be TRUE. the key is the Start key of a partition. | previous partition that key belongs | | 36 | | using an empty key to get Partition with reverse and exclusiveKey flag set to be TRUE. | last partition | | 37 | -------------------------------------------------------------------------------- /docs/RFC/README.md: -------------------------------------------------------------------------------- 1 | This folder contains RFC documents which describe proposals for new additions or changes to the project. 2 | -------------------------------------------------------------------------------- /docs/design/K23SI_CLIENT.md: -------------------------------------------------------------------------------- 1 | Describe K23SI client design 2 | -------------------------------------------------------------------------------- /docs/design/K2PROCESS.md: -------------------------------------------------------------------------------- 1 | This document describes the architecture for a K2 process. 2 | 3 | # General process architecture 4 | ![architecture](../images/K2process.png) 5 | 6 | We use Seastar to manage our processes. When our process is started, we create N reactor instances - each instance pinned to a particular core. The number of instances and their core assignment is driven via command line configuration. 7 | 8 | We have a class used to help create K2 applications, called `K2::App`. It creates all of the necessary subsystems, and allows the user to add their custom code. 9 | The user code is packaged in `Applets`. An `applet` is simply any class which has a start and stop methods (see many examples under k2/src/cmd/demo). The `K2::App` class allows the `applet` to be registered, and the `K2::App` will ensure that there is exactly one instance of the `applet` created and running on each core onto which the process is configured to run. 10 | 11 | Underlying services are initialized by the K2 Appbase, to provide access to infrastructure such as: 12 | - RPC/Messaging 13 | - Configuration 14 | - Logging 15 | - Metrics 16 | 17 | # K2 Nodepool 18 | In our architecture, a K2 cluster is a cluster of `K2Node`s. Each `K2Node` essentially maps 1-1 to a CPU core, (a reactor in the K2 process) and is externally addressable - it can send and receive messages addressed to it. Our partition maps will contain endpoints, naming particular `K2Node`s which would be responsible for particular data ranges. 19 | 20 | When we start `K2Node`s, we don't start each one as a separate process. Instead we start each one as a thread. The process which spins these threads is called the `K2Nodepool`. The main thread in this process is reused as the reactor thread for the first core so that a process which we want to run on 3 cores will have 3 threads. 21 | 22 | A `K2Nodepool` is started with pre-determined number of K2Nodes - we cannot add/remove K2Nodes at runtime from a node pool - we can only start/stop nodepools in order to manage capacity. 23 | 24 | ![architecture](../images/K2NodePool.png) 25 | 26 | The K2Node is a set of Applets which we start in each reactor in order to compose the functionality of a node in the K2 cluster. This includes: 27 | - Assignment Manager: This applet receives external partition assign/offload commands from our ControlPlaneOracle and is responsible for creating new partitions or removing existing ones 28 | - Monitor: This applet is responsible for collecting usage statistics and sending them to our ControlPlaneOracle allowing the CPO to monitor the health of the cluster and take scaling/failover actions if needed. 29 | - TSOClient: a client library for the TSO service, which allows the node to obtain K2 timestamps 30 | - APIServer: Exposes a json-over-http port over which some RPCs can be invoked. Mostly used for testing/debugging 31 | -------------------------------------------------------------------------------- /docs/design/TRANSPORT.md: -------------------------------------------------------------------------------- 1 | Describe transport design 2 | -------------------------------------------------------------------------------- /docs/images/Chogori-node-overview-pic1.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/Chogori-node-overview-pic1.pptx -------------------------------------------------------------------------------- /docs/images/Chogori-node-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/Chogori-node-overview.png -------------------------------------------------------------------------------- /docs/images/Data_Deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/Data_Deployment.png -------------------------------------------------------------------------------- /docs/images/HighLevelTxn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/HighLevelTxn.png -------------------------------------------------------------------------------- /docs/images/JepsenConsistency.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/JepsenConsistency.jpg -------------------------------------------------------------------------------- /docs/images/K23SI-Testing Scenario-04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/K23SI-Testing Scenario-04.png -------------------------------------------------------------------------------- /docs/images/K2Collection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/K2Collection.png -------------------------------------------------------------------------------- /docs/images/K2NodePool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/K2NodePool.png -------------------------------------------------------------------------------- /docs/images/K2process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/K2process.png -------------------------------------------------------------------------------- /docs/images/TSO-High.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TSO-High.png -------------------------------------------------------------------------------- /docs/images/TSO-Low-multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TSO-Low-multi.png -------------------------------------------------------------------------------- /docs/images/TSO-Low.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TSO-Low.png -------------------------------------------------------------------------------- /docs/images/TSO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TSO.png -------------------------------------------------------------------------------- /docs/images/TWIM_States.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TWIM_States.png -------------------------------------------------------------------------------- /docs/images/TWIM_abandoned.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TWIM_abandoned.png -------------------------------------------------------------------------------- /docs/images/TWIM_abort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TWIM_abort.png -------------------------------------------------------------------------------- /docs/images/TWIM_commit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TWIM_commit.png -------------------------------------------------------------------------------- /docs/images/TWIM_iprpip_abort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TWIM_iprpip_abort.png -------------------------------------------------------------------------------- /docs/images/TimestampRefresh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TimestampRefresh.png -------------------------------------------------------------------------------- /docs/images/TxnArchitecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnArchitecture.png -------------------------------------------------------------------------------- /docs/images/TxnBegin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnBegin.png -------------------------------------------------------------------------------- /docs/images/TxnCommit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnCommit.png -------------------------------------------------------------------------------- /docs/images/TxnRead.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnRead.png -------------------------------------------------------------------------------- /docs/images/TxnReadConflict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnReadConflict.png -------------------------------------------------------------------------------- /docs/images/TxnStates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnStates.png -------------------------------------------------------------------------------- /docs/images/TxnStates_Abandoned.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnStates_Abandoned.png -------------------------------------------------------------------------------- /docs/images/TxnStates_Abort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnStates_Abort.png -------------------------------------------------------------------------------- /docs/images/TxnStates_Commit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnStates_Commit.png -------------------------------------------------------------------------------- /docs/images/TxnStates_RetentionWindow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnStates_RetentionWindow.png -------------------------------------------------------------------------------- /docs/images/TxnWrite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnWrite.png -------------------------------------------------------------------------------- /docs/images/TxnWriteConflict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/TxnWriteConflict.png -------------------------------------------------------------------------------- /docs/images/time_sync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/images/time_sync.png -------------------------------------------------------------------------------- /docs/images/time_sync.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | !pragma teoz true 3 | 4 | title Fused-time Synchronization 5 | 6 | autonumber 13 "Time Local Send TLs: " 7 | {start} Alice -> Chronos: - Request 8 | activate Alice 9 | 10 | note over Alice 11 | ts = 2 12 | tr = 5 13 | end note 14 | 15 | autonumber 12 "Remote Time TR: " 16 | Chronos -> Chronos: 17 | 18 | autonumber 20 "Time Local Receive TLr: " 19 | {end} Chronos --> Alice: - Response 20 | {start} <-> {end} : duration: ts + tr = 7 21 | deactivate Alice 22 | 23 | @enduml 24 | -------------------------------------------------------------------------------- /docs/papers/SerializableSnapshotIsolation-fekete-sigmod2008.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/papers/SerializableSnapshotIsolation-fekete-sigmod2008.pdf -------------------------------------------------------------------------------- /docs/papers/YCSB+T.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/papers/YCSB+T.pdf -------------------------------------------------------------------------------- /docs/papers/harding.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/papers/harding.pdf -------------------------------------------------------------------------------- /docs/papers/sinfonia transactions.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/papers/sinfonia transactions.pdf -------------------------------------------------------------------------------- /docs/papers/warp.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/papers/warp.pdf -------------------------------------------------------------------------------- /docs/perf/TPCCLatency.txt: -------------------------------------------------------------------------------- 1 | Total clients cores 2 | New Order Latency (ms) 3 | Cores vs Latency 4 | TPCCLatency.pdf 5 | 4 6 | 80 72 64 56 48 40 32 24 16 8 7 | 4.14 3.47 3.08 2.51 1.91 1.35 0.92 0.64 0.58 0.57 8 | p50 9 | 80 72 64 56 48 40 32 24 16 8 10 | 6.35 5 3.98 3.41 2.77 2.08 1.52 1 0.85 0.83 11 | p90 12 | 80 72 64 56 48 40 32 24 16 8 13 | 10.1 7.15 5.26 4.47 3.84 3.35 2.65 2.2 2.3 2.3 14 | p99 15 | 80 72 64 56 48 40 32 24 16 8 16 | 23.1 9.93 7.1 6.29 5.09 4.32 3.61 3 3.2 3.4 17 | p99.9 18 | -------------------------------------------------------------------------------- /docs/perf/TPCCThroughput.txt: -------------------------------------------------------------------------------- 1 | Total client cores 2 | Throughput (1K Txns/sec) 3 | Cores vs Throughput 4 | TPCCThroughput.pdf 5 | 1 6 | 80 72 64 56 48 40 32 24 16 8 7 | 27.2 28.7 29.8 30.2 30.6 30.8 31.4 27.3 19 9.98 8 | -------------------------------------------------------------------------------- /docs/perf/YCSB-A-Latency.txt: -------------------------------------------------------------------------------- 1 | Total clients cores 2 | Read Latency (us) 3 | YCSB A Read Latency 4 | YCSB-A-Latency.pdf 5 | 4 6 | 80 72 64 56 48 40 32 24 16 8 7 | 29 29 29 28 27 27 26 26 25 25 8 | p50 9 | 80 72 64 56 48 40 32 24 16 8 10 | 37 37 37 35 34 34 32 31 31 30 11 | p90 12 | 80 72 64 56 48 40 32 24 16 8 13 | 600 750 630 251 247 233 63 45 40 37 14 | p99 15 | 80 72 64 56 48 40 32 24 16 8 16 | 4000 4000 3470 2460 2110 2110 1420 1290 1000 50 17 | p99.9 18 | -------------------------------------------------------------------------------- /docs/perf/YCSB-A-Throughput.txt: -------------------------------------------------------------------------------- 1 | Total client cores 2 | Throughput (1K Txns/sec) 3 | Cores vs Throughput 4 | YCSB-A-Throughput.pdf 5 | 1 6 | 80 72 64 56 48 40 32 24 16 8 7 | 432 393 414 422 409 423 391 389 304 165 8 | -------------------------------------------------------------------------------- /docs/perf/YCSB-B-Latency.txt: -------------------------------------------------------------------------------- 1 | Total clients cores 2 | Read Latency (us) 3 | YCSB B Write Latency 4 | YCSB-B-Latency.pdf 5 | 4 6 | 80 72 64 56 48 40 32 24 16 8 7 | 75 75 75 74 73 74 74 74 74 73 8 | p50 9 | 80 72 64 56 48 40 32 24 16 8 10 | 93 93 93 92 90 91 91 91 91 87 11 | p90 12 | 80 72 64 56 48 40 32 24 16 8 13 | 2570 1930 1770 990 818 1080 733 444 234 146 14 | p99 15 | 80 72 64 56 48 40 32 24 16 8 16 | 3330 2510 2100 1720 1410 1460 1140 827 1120 334 17 | p99.9 18 | -------------------------------------------------------------------------------- /docs/perf/YCSB-B-Throughput.txt: -------------------------------------------------------------------------------- 1 | Total client cores 2 | Throughput (1K Txns/sec) 3 | Cores vs Throughput 4 | YCSB-B-Throughput.pdf 5 | 1 6 | 80 72 64 56 48 40 32 24 16 8 7 | 104 105 104 109 110 104 82.6 81.7 88 87.1 8 | -------------------------------------------------------------------------------- /docs/presentations/Chogori Riselab2020Aug.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/presentations/Chogori Riselab2020Aug.pptx -------------------------------------------------------------------------------- /docs/presentations/Chogori-SAP-2020Aug.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/presentations/Chogori-SAP-2020Aug.pptx -------------------------------------------------------------------------------- /docs/presentations/DFV- Chogori SIGMOD20.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/docs/presentations/DFV- Chogori SIGMOD20.pptx -------------------------------------------------------------------------------- /install_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | mkdir -p deps 5 | cd deps 6 | if [ ! -d "crc32c" ]; then git clone --branch 1.1.0 https://github.com/google/crc32c.git; fi 7 | cd crc32c 8 | git submodule update --init --recursive 9 | mkdir -p build 10 | cd build 11 | cmake -DCRC32C_BUILD_TESTS=0 -DCRC32C_BUILD_BENCHMARKS=0 .. && make -j all install 12 | cd ../../../ 13 | 14 | mkdir -p src/logging/build && cd src/logging/build && cmake ../ && make -j install && cd - 15 | mkdir -p src/skvhttpclient/build && cd src/skvhttpclient/build && cmake ../ && make -j install && cd - 16 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(k2) 2 | -------------------------------------------------------------------------------- /src/k2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (appbase) 2 | add_subdirectory (assignmentManager) 3 | add_subdirectory (cmd) 4 | add_subdirectory (common) 5 | add_subdirectory (config) 6 | add_subdirectory (cpo) 7 | add_subdirectory (dto) 8 | add_subdirectory (httpproxy) 9 | add_subdirectory (indexer) 10 | add_subdirectory (infrastructure) 11 | add_subdirectory (module) 12 | add_subdirectory (persistence) 13 | add_subdirectory (transport) 14 | add_subdirectory (tso) 15 | -------------------------------------------------------------------------------- /src/k2/README.md: -------------------------------------------------------------------------------- 1 | 26 | # K2 source code 27 | This folder contains source code of the main K2 platform components. 28 | 29 | # Structure 30 | * node: code related to Application (Node) Service 31 | * manager: Partition Manager service 32 | * client: K2 client 33 | * common: code shared between components 34 | * transport: K2 transport library 35 | * benchmarker: benchmarking related code 36 | * config: K2 configuration 37 | -------------------------------------------------------------------------------- /src/k2/appbase/AppEssentials.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | // This is a convenience header, which helps get access to various concerns for k2 applications 25 | // k2 stuff 26 | #include // for to common data types 27 | #include // for access to initialized Config 28 | #include // for access to clock 29 | #include // for access to initialized logging 30 | #include // for access to initialized etrics 31 | #include // for access to initialized RPC 32 | 33 | // seastar stuff 34 | #include // for distributed<> 35 | #include // for future stuff 36 | #include 37 | #include 38 | #include // metrics 39 | #include // periodic timer 40 | #include // for passing references around 41 | 42 | using namespace std::chrono_literals; // so that we can type "1ms" 43 | namespace bpo = boost::program_options; 44 | namespace sm = seastar::metrics; 45 | -------------------------------------------------------------------------------- /src/k2/appbase/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(appbase OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(appbase PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (appbase PRIVATE config common transport Seastar::seastar ) 12 | 13 | # export the library in the common k2Targets 14 | install(TARGETS appbase EXPORT k2Targets DESTINATION lib/k2) 15 | install(FILES ${HEADERS} DESTINATION include/k2/appbase) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS appbase NAMESPACE k2:: FILE appbase-config.cmake) 18 | -------------------------------------------------------------------------------- /src/k2/assignmentManager/AssignmentManager.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | 26 | // third-party 27 | #include 28 | #include // for future stuff 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | namespace k2 { 36 | namespace log { 37 | inline thread_local k2::logging::Logger amgr("k2::assignment_manager"); 38 | } 39 | 40 | class AssignmentManager { 41 | public: // application lifespan 42 | AssignmentManager(); 43 | ~AssignmentManager(); 44 | 45 | // required for seastar::distributed interface 46 | seastar::future<> gracefulStop(); 47 | seastar::future<> start(); 48 | 49 | seastar::future> 50 | handleAssign(dto::AssignmentCreateRequest&& request); 51 | 52 | seastar::future> 53 | handleOffload(dto::AssignmentOffloadRequest&& request); 54 | 55 | private: 56 | std::unique_ptr _pmodule; 57 | String _collectionName; 58 | }; // class AssignmentManager 59 | 60 | } // namespace k2 61 | -------------------------------------------------------------------------------- /src/k2/assignmentManager/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(assignment_manager OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (assignment_manager PRIVATE common transport Seastar::seastar dto k23si) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS assignment_manager EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/assignmentManager) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS assignment_manager NAMESPACE k2:: FILE assignment_manager-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/cmd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(demo) 2 | add_subdirectory(tpcc) 3 | add_subdirectory(controlPlaneOracle) 4 | add_subdirectory(nodepool) 5 | add_subdirectory(persistence) 6 | add_subdirectory(plog) 7 | add_subdirectory(txbench) 8 | add_subdirectory(tso) 9 | add_subdirectory(ycsb) 10 | add_subdirectory(httpproxy) 11 | -------------------------------------------------------------------------------- /src/k2/cmd/controlPlaneOracle/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (cpo_main cpo_main.cpp) 2 | 3 | target_link_libraries (cpo_main PRIVATE appbase common transport Seastar::seastar cpo_service infrastructure dto) 4 | 5 | install (TARGETS cpo_main DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/demo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (k2_demo_service service.cpp) 2 | add_executable (k2_demo_rpc_echo rpc_echo.cpp) 3 | add_executable (k2_demo_rpc_service rpc_service.cpp) 4 | add_executable (k2_demo_tso tso_sample_app.cpp) 5 | 6 | target_link_libraries (k2_demo_service PRIVATE appbase transport common Seastar::seastar) 7 | target_link_libraries (k2_demo_rpc_echo PRIVATE appbase transport common Seastar::seastar) 8 | target_link_libraries (k2_demo_rpc_service PRIVATE appbase transport common Seastar::seastar) 9 | target_link_libraries (k2_demo_tso PRIVATE tso_service tso_client appbase transport common Seastar::seastar) 10 | 11 | #install (TARGETS k2_demo_service k2_demo_rpc_echo k2_demo_rpc_service k2_demo_tso DESTINATION bin) 12 | -------------------------------------------------------------------------------- /src/k2/cmd/demo/README.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | This folder contains demos for K2 6 | # Service 7 | This demo shows an example K2 service which: 8 | - registers RPC handlers 9 | - automatically discovers available interfaces and creates a new RPC server stack on a separate core, polling the same interface 10 | -------------------------------------------------------------------------------- /src/k2/cmd/httpproxy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (http_proxy http_proxy_main.cpp) 2 | 3 | target_link_libraries (http_proxy PRIVATE appbase k23si_client cpo_client tso_client httpproxy infrastructure dto transport Seastar::seastar) 4 | 5 | install (TARGETS http_proxy DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/httpproxy/http_proxy_main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | int main(int argc, char** argv) { 31 | k2::App app("HttpProxy"); 32 | app.addApplet(); 33 | app.addApplet(); 34 | app.addApplet(); 35 | app.addOptions() 36 | // config for dependencies 37 | ("partition_request_timeout", bpo::value(), "Timeout of K23SI operations, as chrono literals") 38 | ("cpo", bpo::value(), "URL of Control Plane Oracle (CPO), e.g. 'tcp+k2rpc://192.168.1.2:12345'") 39 | ("cpo_request_timeout", bpo::value(), "CPO request timeout") 40 | ("cpo_request_backoff", bpo::value(), "CPO request backoff") 41 | ("httpproxy_txn_timeout", bpo::value(), "Txn idle timeout") 42 | ("httpproxy_expiry_timer_interval", bpo::value(), "Periodic timer interval to check expired Txns"); 43 | return app.start(argc, argv); 44 | } 45 | -------------------------------------------------------------------------------- /src/k2/cmd/nodepool/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (nodepool nodepool_main.cpp) 2 | 3 | target_link_libraries (nodepool PRIVATE appbase k23si cpo_client common transport Seastar::seastar assignment_manager tso_client infrastructure dto) 4 | 5 | install (TARGETS nodepool DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/persistence/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (persistence persistence_main.cpp) 2 | 3 | target_link_libraries (persistence PRIVATE cpo_client appbase common transport Seastar::seastar persistence_service) 4 | 5 | install (TARGETS persistence DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/persistence/persistence_main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | int main(int argc, char** argv) { 29 | k2::App app("PersistenceService"); 30 | app.addApplet(); 31 | // pass the ss::distributed container to the PersistenceService constructor 32 | app.addApplet(); 33 | return app.start(argc, argv); 34 | } 35 | -------------------------------------------------------------------------------- /src/k2/cmd/plog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (plog_main plog_main.cpp) 2 | 3 | target_link_libraries (plog_main PRIVATE appbase common transport Seastar::seastar plog_service) 4 | 5 | install (TARGETS plog_main DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/plog/plog_main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | int main(int argc, char** argv) { 29 | k2::App app("PlogServer"); 30 | app.addApplet(); 31 | return app.start(argc, argv); 32 | } 33 | 34 | -------------------------------------------------------------------------------- /src/k2/cmd/tpcc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (tpcc_client ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (tpcc_client PRIVATE tso_client appbase transport common cpo_client k23si_client dto Seastar::seastar) 7 | 8 | install (TARGETS tpcc_client DESTINATION bin) 9 | -------------------------------------------------------------------------------- /src/k2/cmd/tpcc/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger tpcc("k2::tpcc"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/cmd/tpcc/README: -------------------------------------------------------------------------------- 1 | To run: 2 | 3 | Start CPO, TSO, K2 server, and persistence 4 | 5 | Run load phase: 6 | ./tpcc_client --tcp_remotes --cpo --tso_endpoint 7 | --data_load true --num_warehouses 8 | --txn_weights 9 | 10 | Wait for "Done with benchmark" message and kill process. 11 | 12 | 13 | Run bechmark phase: 14 | ./tpcc_client --tcp_remotes --cpo --tso_endpoint 15 | --data_load false --num_warehouses clients_per_core 16 | --txn_weights 17 | 18 | Wait for "Done with benchmark" message 19 | 20 | 21 | See cluster/configs/tpcc_load.cfg and tpcc_client.cfg for examples 22 | -------------------------------------------------------------------------------- /src/k2/cmd/tso/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (tso tso_main.cpp) 2 | 3 | target_link_libraries (tso PRIVATE tso_service cpo_client appbase common transport Seastar::seastar) 4 | 5 | install (TARGETS tso DESTINATION bin) 6 | -------------------------------------------------------------------------------- /src/k2/cmd/tso/tso_main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | int main(int argc, char** argv) { 29 | k2::App app("TSOService"); 30 | app.addApplet(); 31 | app.addApplet(); 32 | 33 | app.addOptions()("tso.clock_poller_cpu", bpo::value(), "CPU to which to pin the GPS clock polling thread"); 34 | return app.start(argc, argv); 35 | } 36 | -------------------------------------------------------------------------------- /src/k2/cmd/txbench/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable (txbench_client txbench_client.cpp txbench_common.h) 2 | add_executable (txbench_server txbench_server.cpp txbench_common.h) 3 | add_executable (txbench_combine txbench_combine.cpp txbench_common.h) 4 | 5 | add_executable (rpcbench_client rpcbench_client.cpp rpcbench_common.h) 6 | add_executable (rpcbench_server rpcbench_server.cpp rpcbench_common.h) 7 | 8 | add_executable (k23sibench_client k23sibench_client.cpp) 9 | 10 | target_link_libraries (txbench_client PRIVATE appbase transport common Seastar::seastar) 11 | target_link_libraries (txbench_server PRIVATE appbase transport common Seastar::seastar) 12 | target_link_libraries (txbench_combine PRIVATE appbase transport common Seastar::seastar) 13 | 14 | target_link_libraries (rpcbench_client PRIVATE appbase transport common Seastar::seastar) 15 | target_link_libraries (rpcbench_server PRIVATE appbase transport common Seastar::seastar) 16 | 17 | target_link_libraries (k23sibench_client PRIVATE appbase tso_client cpo_client k23si_client dto transport Seastar::seastar) 18 | 19 | #install (TARGETS txbench_client txbench_server txbench_combine rpcbench_client rpcbench_server k23sibench_client DESTINATION bin) 20 | -------------------------------------------------------------------------------- /src/k2/cmd/txbench/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger txbench("k2::txbench"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/cmd/txbench/rpcbench_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | 27 | #pragma once 28 | using namespace k2; 29 | 30 | // The message verbs supported by this service 31 | enum MsgVerbs: Verb { 32 | REQUEST_COPY = 10, // issue request with a copy 33 | REQUEST = 11, // incoming requests 34 | START_SESSION = 12 // used to start a new test session 35 | }; 36 | 37 | struct BenchSession { 38 | BenchSession()=default; 39 | BenchSession(BenchSession&&)=default; 40 | BenchSession& operator=(BenchSession&&)=default; 41 | BenchSession(uint64_t sessionId, size_t dataSize): sessionId(sessionId) { 42 | dataCopy = String(dataSize, '.'); 43 | dataShare.write(dataCopy); 44 | } 45 | 46 | uint64_t sessionId = 0; 47 | uint64_t totalSize=0; 48 | uint64_t totalCount=0; 49 | Payload dataShare{Payload::DefaultAllocator()}; 50 | String dataCopy; 51 | std::vector> endpoints; 52 | }; 53 | 54 | struct TXBenchStartSession { 55 | uint64_t responseSize = 0; 56 | K2_PAYLOAD_FIELDS(responseSize); 57 | }; 58 | 59 | struct TXBenchStartSessionAck { 60 | uint64_t sessionId=0; 61 | K2_PAYLOAD_FIELDS(sessionId); 62 | }; 63 | 64 | template 65 | struct TXBenchRequest { 66 | uint64_t sessionId=0; 67 | SerializeAsPayload data; 68 | K2_PAYLOAD_FIELDS(sessionId, data); 69 | }; 70 | 71 | template 72 | struct TXBenchResponse { 73 | uint64_t sessionId=0; 74 | SerializeAsPayload data; 75 | K2_PAYLOAD_FIELDS(sessionId, data); 76 | }; 77 | -------------------------------------------------------------------------------- /src/k2/cmd/ycsb/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (ycsb_client ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (ycsb_client PRIVATE tso_client appbase common cpo_client k23si_client dto transport Seastar::seastar) 7 | 8 | install (TARGETS ycsb_client DESTINATION bin) 9 | -------------------------------------------------------------------------------- /src/k2/cmd/ycsb/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger ycsb("k2::ycsb"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/cmd/ycsb/README: -------------------------------------------------------------------------------- 1 | To run: 2 | 3 | Start CPO, TSO, K2 server, and persistence 4 | 5 | Run load phase: 6 | ./ycsb_client --tcp_remotes --cpo --tso_endpoint 7 | --data_load true --num_records 8 | --num_records_insert 9 | --request_dist --tcp_remotes --cpo --tso_endpoint 23 | --data_load false --clients_per_core 24 | --num_records --num_records_insert --request_dist 25 | --ops_per_txn 26 | --read_proportion --update_proportion --scan_proportion --insert_proportion --delete_propotion 27 | 28 | Wait for "Done with benchmark" message 29 | -------------------------------------------------------------------------------- /src/k2/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(common INTERFACE ${HEADERS} ${SOURCES}) 5 | target_link_libraries (common INTERFACE Seastar::seastar fmt::fmt skvhttp::common) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS common EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/common) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS common NAMESPACE k2:: FILE common-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/common/Defer.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | 28 | namespace k2::log { 29 | inline thread_local k2::logging::Logger defer("k2::defer"); 30 | } 31 | 32 | namespace k2 { 33 | // Simple utility class used to defer execution of a function until the end of a scope 34 | // the function is executed when the Defer instance is destroyed. 35 | 36 | template 37 | class Defer { 38 | public: 39 | Defer(Func&& func) : _func(std::forward(func)) {} 40 | ~Defer() { 41 | try { 42 | (void)_func(); 43 | } catch (std::exception& exc) { 44 | K2LOG_E(log::defer, "deferred func threw exception: {}", exc.what()); 45 | } catch (...) { 46 | K2LOG_E(log::defer, "deferred func threw unknown exception"); 47 | } 48 | } 49 | 50 | private: 51 | Func _func; 52 | }; 53 | 54 | } // ns k2 55 | -------------------------------------------------------------------------------- /src/k2/common/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2022 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger common("k2::common"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/common/MapUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2022 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | namespace k2 { 26 | 27 | // return a tuple of iterators to the elements before, at, and after the given key. 28 | template 29 | auto keyRange(const KeyT& key, MapT& mm) { 30 | auto end = mm.end(); 31 | // gives iterators (lower, upper) 32 | // lower is an interator to the first element, not less than key 33 | // upper is an iterator to the first element greater than key 34 | auto [at, after] = mm.equal_range(key); 35 | 36 | // to get the element before "key", we need to rewind "at" 37 | auto before = at; 38 | if (before != mm.begin() && mm.size() > 0) { 39 | // this is only safe to do if we do have some elements present 40 | --before; 41 | } else { 42 | // we can't rewind begin. Set it to end() directly 43 | before = mm.end(); 44 | } 45 | 46 | // if key exists, "at" and "after" will be different 47 | auto found = at == after ? end : at; 48 | 49 | return std::tuple(before, found, after); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/k2/common/SpinMutex.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #include 24 | #include 25 | 26 | namespace k2 { 27 | // Spin-based mutex implementation 28 | class SpinMutex { 29 | private: 30 | std::atomic_flag _flag = ATOMIC_FLAG_INIT; 31 | 32 | public: 33 | void lock() { 34 | while (_flag.test_and_set(std::memory_order_acquire)); 35 | } 36 | void unlock() { 37 | _flag.clear(std::memory_order_release); 38 | } 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /src/k2/common/TypeMap.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | namespace k2 { 28 | 29 | // Hash-map for type->value mappings. 30 | // Internally, we assign a unique id for each type and instantiate template methods as needed for the type 31 | template 32 | class TypeMap { 33 | typedef std::unordered_map __TypeMap; 34 | 35 | public: // iterator stuff 36 | typedef typename __TypeMap::iterator iterator; 37 | typedef typename __TypeMap::const_iterator const_iterator; 38 | 39 | const_iterator begin() const { return _map.begin(); } 40 | const_iterator end() const { return _map.end(); } 41 | iterator begin() { return _map.begin(); } 42 | iterator end() { return _map.end(); } 43 | template 44 | iterator find() { return _map.find(getTypeId()); } 45 | template 46 | const_iterator find() const { return _map.find(getTypeId()); } 47 | // Associates a value with the type "Key" 48 | template 49 | void put(ValueT&& value) { _map.emplace(getTypeId(), std::forward(value)); } 50 | 51 | private: 52 | // This function creates the unique id for each type. It allows us to discover the id for any 53 | // type at runtime since the compiler generates a unique getTypeId for each type. 54 | template 55 | int getTypeId() { 56 | // the id for this type. Since this is static, it gets assigned exactly once, on first call with this type 57 | static const int id = _typeIdx++; 58 | return id; 59 | } 60 | int _typeIdx = 0; // the latest id to use 61 | __TypeMap _map; 62 | }; 63 | 64 | } // namespace k2 65 | -------------------------------------------------------------------------------- /src/k2/common/VecUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | namespace k2 { 28 | 29 | // base recursion - no args to append 30 | template 31 | void _append_vec(std::vector&) {} 32 | 33 | // Append rvalues to a vector with variadic template expansion 34 | template 35 | void _append_vec(std::vector& v, T&& obj, ObjT&&... objs) { 36 | v.push_back(std::forward(obj)); 37 | _append_vec(v, std::forward(objs)...); 38 | } 39 | 40 | template 41 | void append_vec(std::vector& v, ObjT&&... objs) { 42 | constexpr std::size_t sz = std::tuple_size>::value; 43 | v.reserve(sz); 44 | _append_vec(v, std::forward(objs)...); 45 | } 46 | 47 | // create a vector using rvalues of type. This is needed to aid in creating rvalue vectors from 48 | // non-copyable types since the standard lib initializer list doesn't support non-copyable types 49 | template 50 | std::vector make_vec(ObjT&&... objs) { 51 | std::vector result; 52 | constexpr std::size_t sz = std::tuple_size>::value; 53 | result.reserve(sz); 54 | append_vec(result, std::forward(objs)...); 55 | return result; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/k2/config/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(config INTERFACE ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(config PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (config INTERFACE common) 12 | 13 | # export the library in the common k2Targets 14 | install(TARGETS config EXPORT k2Targets DESTINATION lib/k2) 15 | install(FILES ${HEADERS} DESTINATION include/k2/config) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS config NAMESPACE k2:: FILE config-config.cmake) 18 | -------------------------------------------------------------------------------- /src/k2/cpo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (service) 2 | add_subdirectory (client) 3 | -------------------------------------------------------------------------------- /src/k2/cpo/client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(cpo_client OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (cpo_client PRIVATE common Seastar::seastar dto transport) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS cpo_client EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/cpo/client) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS cpo_client NAMESPACE k2:: FILE cpo_client-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/cpo/client/Heartbeat.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | 26 | // third-party 27 | #include 28 | #include // for future stuff 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | 36 | namespace k2::cpo { 37 | 38 | class HeartbeatResponder { 39 | private: 40 | SingleTimer _nextHeartbeatExpire; 41 | Duration _HBInterval; // Will be obtained from the heartbeat requests 42 | uint32_t _HBDeadThreshold; // Will be obtained from the heartbeat requests 43 | uint32_t _missedHBs{0}; 44 | uint64_t _lastSeq{0}; 45 | bool _running{false}; 46 | bool _up{false}; 47 | 48 | // Returned in heartbeat response 49 | String _ID; 50 | std::vector _eps; 51 | String _metadata; 52 | 53 | // For metrics 54 | void _registerMetrics(); 55 | sm::metric_groups _metric_groups; 56 | k2::ExponentialHistogram _heartbeatInterarrival; 57 | k2::TimePoint _lastHeartbeatTime; 58 | 59 | seastar::future> 60 | _handleHeartbeat(dto::HeartbeatRequest&& request); 61 | 62 | public: 63 | // required for seastar::distributed interface 64 | seastar::future<> gracefulStop(); 65 | seastar::future<> start(); 66 | 67 | // Used by other applets to know if they should soft-down themselves 68 | bool isUp(); 69 | // Used by other applets to set role-specific metadata, 70 | // which is passed on to the monitor in the HB response 71 | void setRoleMetadata(String metadata); 72 | }; // class HealthMonitor 73 | 74 | } // namespace k2 75 | -------------------------------------------------------------------------------- /src/k2/cpo/service/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(cpo_service OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (cpo_service PRIVATE common transport Seastar::seastar dto) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS cpo_service EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/cpo/service) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS cpo_service NAMESPACE k2:: FILE cpo_service-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/cpo/service/Log.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace k2::cpo { 4 | namespace log { 5 | inline thread_local k2::logging::Logger cposvr("k2::cpo_service"); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/k2/dto/AssignmentManager.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include "Collection.h" 26 | #include "Log.h" 27 | // This file contains DTOs for K2 AssignmentManager 28 | 29 | namespace k2 { 30 | namespace dto { 31 | 32 | // Request to create a collection 33 | struct AssignmentCreateRequest { 34 | CollectionMetadata collectionMeta; 35 | Partition partition; 36 | std::vector cpoEndpoints; 37 | K2_PAYLOAD_FIELDS(collectionMeta, partition, cpoEndpoints); 38 | }; 39 | 40 | // Response to AssignmentCreateRequest 41 | struct AssignmentCreateResponse { 42 | Partition assignedPartition; 43 | K2_PAYLOAD_FIELDS(assignedPartition); 44 | }; 45 | 46 | // Request to offload a collection 47 | struct AssignmentOffloadRequest { 48 | String collectionName; 49 | K2_PAYLOAD_FIELDS(collectionName); 50 | }; 51 | 52 | // Response to AssignmentOffloadRequest 53 | struct AssignmentOffloadResponse { 54 | K2_PAYLOAD_EMPTY; 55 | }; 56 | 57 | } // namespace dto 58 | } // namespace k2 59 | -------------------------------------------------------------------------------- /src/k2/dto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(dto OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (dto PRIVATE transport common) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS dto EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/dto) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS dto NAMESPACE k2:: FILE dto-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/dto/K23SI.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "K23SI.h" 25 | namespace k2 { 26 | namespace dto { 27 | 28 | bool K23SI_MTR::operator==(const K23SI_MTR& o) const { 29 | return timestamp == o.timestamp && priority == o.priority; 30 | } 31 | bool K23SI_MTR::operator!=(const K23SI_MTR& o) const { 32 | return !(operator==(o)); 33 | } 34 | 35 | size_t K23SI_MTR::hash() const { 36 | return hash_combine(timestamp, priority); 37 | } 38 | 39 | } // ns dto 40 | } // ns k2 41 | -------------------------------------------------------------------------------- /src/k2/dto/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger dto("k2::dto"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/dto/PersistenceCluster.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | namespace k2 { 26 | namespace dto { 27 | 28 | struct PersistenceGroup{ 29 | String name; 30 | std::vector plogServerEndpoints; 31 | K2_PAYLOAD_FIELDS(name, plogServerEndpoints); 32 | }; 33 | 34 | 35 | struct PersistenceCluster{ 36 | String name; 37 | std::vector persistenceGroupVector; 38 | K2_PAYLOAD_FIELDS(name, persistenceGroupVector); 39 | }; 40 | 41 | 42 | // Request to create a Partition Cluster 43 | struct PersistenceClusterCreateRequest { 44 | PersistenceCluster cluster; 45 | K2_PAYLOAD_FIELDS(cluster); 46 | }; 47 | 48 | struct PersistenceClusterCreateResponse { 49 | K2_PAYLOAD_EMPTY; 50 | }; 51 | 52 | // Request to obtain a Partition Cluster 53 | struct PersistenceClusterGetRequest { 54 | String name; 55 | K2_PAYLOAD_FIELDS(name); 56 | }; 57 | 58 | struct PersistenceClusterGetResponse { 59 | PersistenceCluster cluster; 60 | K2_PAYLOAD_FIELDS(cluster); 61 | }; 62 | 63 | } // namespace dto 64 | } // namespace k2 65 | -------------------------------------------------------------------------------- /src/k2/dto/README.md: -------------------------------------------------------------------------------- 1 | This folder describes the wire interfaces for K2, and provides the types (DTOs) needed to send requests and receive 2 | responses when dealing with K2. 3 | 4 | The reason why we have all of these interfaces in one place is to facilitate mutual dependency on the types. For example, the ControlPlaneOracle will send ASSIGN message to K2 nodes in order to assign partitions. K2 nodes on the other hand will call ControlPlaneOracle to discover new/changed Collections (e.g. needed for the PUSH operation in K2-3SI transactions) 5 | 6 | - Each component which wants to expose a DTO interface, should create a separate file here with its DTOs 7 | - The verbs used to access the interface should be added to the common verbs in MessageVerbs.h 8 | - For objects used in the JSON API HTTP server (see infrastructure/APIServer.h), to and from json methods need to be added. They are not needed for K2 RPCs so can otherwise be omitted. 9 | 10 | TODO: For now we keep all interfaces here. We should consider refactoring into internal/external 11 | -------------------------------------------------------------------------------- /src/k2/dto/TSO.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include "Timestamp.h" 26 | namespace k2::dto { 27 | 28 | struct GetTimestampRequest { 29 | K2_PAYLOAD_EMPTY; 30 | K2_DEF_FMT(GetTimestampRequest); 31 | }; 32 | 33 | struct GetTimestampResponse{ 34 | Timestamp timestamp; 35 | 36 | K2_PAYLOAD_FIELDS(timestamp); 37 | K2_DEF_FMT(GetTimestampResponse, timestamp); 38 | }; 39 | 40 | } // k2::dto 41 | -------------------------------------------------------------------------------- /src/k2/httpproxy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(httpproxy OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (httpproxy PRIVATE cpo_client k23si_client dto infrastructure common config Seastar::seastar skvhttp::dto) 7 | 8 | # export the library in the common k2Targets 9 | install(TARGETS httpproxy EXPORT k2Targets DESTINATION lib/k2) 10 | install(FILES ${HEADERS} DESTINATION include/k2/httpproxy) 11 | # export the cmake config in the build tree for any users who want to use this project from source 12 | export(TARGETS httpproxy NAMESPACE k2:: FILE httpproxy-config.cmake) 13 | -------------------------------------------------------------------------------- /src/k2/indexer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(indexer OBJECT ${HEADERS} ${SOURCES}) 5 | set_target_properties(indexer PROPERTIES LINKER_LANGUAGE CXX) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS indexer EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/indexer) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS indexer NAMESPACE k2:: FILE indexer-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/indexer/IndexerInterface.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | namespace k2 28 | { 29 | // 30 | // K2 internal MVCC representation 31 | // 32 | template 33 | struct VersionedTreeNode { 34 | uint64_t version; 35 | ValueType value; 36 | std::unique_ptr> next; 37 | }; 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/k2/infrastructure/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(infrastructure OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (infrastructure PRIVATE common config Seastar::seastar transport) 7 | 8 | # export the library in the common k2Targets 9 | install(TARGETS infrastructure EXPORT k2Targets DESTINATION lib/k2) 10 | install(FILES ${HEADERS} DESTINATION include/k2/infrastructure) 11 | # export the cmake config in the build tree for any users who want to use this project from source 12 | export(TARGETS infrastructure NAMESPACE k2:: FILE infrastructure-config.cmake) 13 | -------------------------------------------------------------------------------- /src/k2/module/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (k23si) 2 | -------------------------------------------------------------------------------- /src/k2/module/k23si/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(k23si OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (k23si PRIVATE indexer transport cpo_client infrastructure dto common Seastar::seastar) 6 | add_subdirectory (client) 7 | 8 | # export the library in the common k2Targets 9 | install(TARGETS k23si EXPORT k2Targets DESTINATION lib/k2) 10 | install(FILES ${HEADERS} DESTINATION include/k2/module/k23si) 11 | # export the cmake config in the build tree for any users who want to use this project from source 12 | export(TARGETS k23si NAMESPACE k2:: FILE k23si-config.cmake) 13 | -------------------------------------------------------------------------------- /src/k2/module/k23si/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | #include 25 | 26 | namespace k2::log { 27 | inline thread_local k2::logging::Logger skvsvr("k2::skv_server"); 28 | } 29 | -------------------------------------------------------------------------------- /src/k2/module/k23si/client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(k23si_client OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (k23si_client PRIVATE cpo_client dto transport common Seastar::seastar) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS k23si_client EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/module/k23si/client) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS k23si_client NAMESPACE k2:: FILE k23si_client-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/module/k23si/client/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | namespace k2::log { 25 | inline thread_local k2::logging::Logger skvclient("k2::skv_client"); 26 | } 27 | -------------------------------------------------------------------------------- /src/k2/persistence/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/futurewei-cloud/chogori-platform/30291e5840fb9ccb2be75cde5b091856a8eea552/src/k2/persistence/.DS_Store -------------------------------------------------------------------------------- /src/k2/persistence/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (service) 2 | add_subdirectory (plog_service) 3 | add_subdirectory (plog_client) 4 | add_subdirectory (logStream) 5 | -------------------------------------------------------------------------------- /src/k2/persistence/logStream/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(log_stream OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (log_stream PRIVATE transport common cpo_client plog_client) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS log_stream EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/persistence/logStream) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS log_stream NAMESPACE k2:: FILE log_stream-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/persistence/plog_client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(plog_client OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (plog_client PRIVATE transport common cpo_client) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS plog_client EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/persistence/plog_client) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS plog_client NAMESPACE k2:: FILE plog_client-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/persistence/plog_service/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(plog_service OBJECT ${HEADERS} ${SOURCES}) 5 | target_link_libraries (plog_service PRIVATE transport common cpo_client) 6 | 7 | # export the library in the common k2Targets 8 | install(TARGETS plog_service EXPORT k2Targets DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/persistence/plog_service) 10 | # export the cmake config in the build tree for any users who want to use this project from source 11 | export(TARGETS plog_service NAMESPACE k2:: FILE plog_service-config.cmake) 12 | -------------------------------------------------------------------------------- /src/k2/persistence/service/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(persistence_service OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(persistence_service PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (persistence_service PRIVATE common transport dto Seastar::seastar ) 12 | 13 | # export the library in the common k2Targets 14 | install(TARGETS persistence_service EXPORT k2Targets DESTINATION lib/k2) 15 | install(FILES ${HEADERS} DESTINATION include/k2/persistence/service) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS persistence_service NAMESPACE k2:: FILE persistence_service-config.cmake) 18 | -------------------------------------------------------------------------------- /src/k2/persistence/service/PersistenceService.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "PersistenceService.h" 25 | #include 26 | #include 27 | 28 | #include 29 | #include // for RPC 30 | 31 | namespace k2 { 32 | 33 | PersistenceService::PersistenceService() { 34 | K2LOG_I(log::psvc, "ctor"); 35 | } 36 | 37 | PersistenceService::~PersistenceService() { 38 | K2LOG_I(log::psvc, "dtor"); 39 | } 40 | 41 | seastar::future<> PersistenceService::gracefulStop() { 42 | K2LOG_I(log::psvc, "stop"); 43 | return seastar::make_ready_future(); 44 | } 45 | 46 | seastar::future<> PersistenceService::start() { 47 | K2LOG_I(log::psvc, "Registering message handlers"); 48 | RPC().registerRPCObserver, dto::K23SI_PersistenceResponse> 49 | (dto::Verbs::K23SI_Persist, [this](dto::K23SI_PersistenceRequest&& request) { 50 | (void) request; 51 | return RPCResponse(Statuses::S200_OK("persistence success"), dto::K23SI_PersistenceResponse{}); 52 | }); 53 | 54 | return seastar::make_ready_future(); 55 | } 56 | 57 | } // namespace k2 58 | -------------------------------------------------------------------------------- /src/k2/persistence/service/PersistenceService.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | 26 | // third-party 27 | #include // for distributed<> 28 | #include // for future stuff 29 | #include 30 | 31 | namespace k2 { 32 | namespace log { 33 | inline thread_local k2::logging::Logger psvc("k2::persistence_svc"); 34 | } 35 | 36 | class PersistenceService { 37 | public : // application lifespan 38 | PersistenceService(); 39 | ~PersistenceService(); 40 | 41 | // required for seastar::distributed interface 42 | seastar::future<> gracefulStop(); 43 | seastar::future<> start(); 44 | }; // class PersistenceService 45 | 46 | } // namespace k2 47 | -------------------------------------------------------------------------------- /src/k2/transport/BaseTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | #include 27 | 28 | #include // for socket_address 29 | #include 30 | 31 | #include "Payload.h" 32 | #include 33 | #include "Request.h" 34 | #include "Status.h" 35 | #include "TXEndpoint.h" 36 | 37 | namespace k2 { 38 | // This file contains definitions for the base types we may want to use in the transport codebase 39 | 40 | // SocketAddress alias 41 | typedef seastar::socket_address SocketAddress; 42 | 43 | // The type for Message observers 44 | // TODO See if we can use something faster than std::function. 45 | // Benchmark indicates 20ns penalty per runtime call 46 | // See https://www.boost.org/doc/libs/1_69_0/doc/html/function/faq.html 47 | typedef std::function RequestObserver_t; 48 | 49 | // The type for RPC request observers. This is meant to be used with the RPC* API of RPCDispatcher 50 | template 51 | using RPCRequestObserver_t = std::function>(RequestType_t&& request)>; 52 | 53 | // The type for observers of channel failures 54 | typedef std::function FailureObserver_t; 55 | 56 | // the type of a low memory observer. This function will be called when a transport requires a release of 57 | // some memory 58 | typedef std::function LowTransportMemoryObserver_t; 59 | 60 | // the type of a function we can call to notify that we're low on memory. Function is called with required 61 | // number of bytes to release 62 | typedef std::function LowMemoryObserver_t; 63 | 64 | } // namespace k2 65 | -------------------------------------------------------------------------------- /src/k2/transport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(transport OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (transport PRIVATE common config Seastar::seastar crc32c skvhttp::common skvhttp::mpack) 7 | 8 | # export the library in the common k2Targets 9 | install(TARGETS transport EXPORT k2Targets DESTINATION lib/k2) 10 | install(FILES ${HEADERS} DESTINATION include/k2/transport) 11 | # export the cmake config in the build tree for any users who want to use this project from source 12 | export(TARGETS transport NAMESPACE k2:: FILE transport-config.cmake) 13 | -------------------------------------------------------------------------------- /src/k2/transport/Discovery.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "Discovery.h" 25 | 26 | #include 27 | 28 | #include "AutoRRDMARPCProtocol.h" 29 | #include "DiscoveryDTO.h" 30 | #include "Status.h" 31 | 32 | namespace k2 { 33 | 34 | Discovery::Discovery() { 35 | K2LOG_I(log::tx, "ctor"); 36 | } 37 | 38 | Discovery::~Discovery() { 39 | K2LOG_I(log::tx, "dtor"); 40 | } 41 | 42 | seastar::future<> Discovery::gracefulStop() { 43 | K2LOG_I(log::tx, "graceful stop"); 44 | return seastar::make_ready_future(); 45 | } 46 | 47 | seastar::future<> Discovery::start() { 48 | K2LOG_I(log::tx, "Registering message handlers"); 49 | RPC().registerRPCObserver(InternalVerbs::LIST_ENDPOINTS, 50 | [this](ListEndpointsRequest&&) { 51 | ListEndpointsResponse response{}; 52 | for (auto& serverEndpoint : RPC().getServerEndpoints()) { 53 | if (serverEndpoint) { 54 | response.endpoints.push_back(serverEndpoint->url); 55 | } 56 | } 57 | return RPCResponse(Statuses::S200_OK("list endpoints success"), std::move(response)); 58 | }); 59 | 60 | return seastar::make_ready_future(); 61 | } 62 | 63 | } // namespace k2 64 | -------------------------------------------------------------------------------- /src/k2/transport/DiscoveryDTO.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | #include "PayloadSerialization.h" 27 | 28 | namespace k2 { 29 | 30 | // Request for listing the supported endpoints by a node 31 | struct ListEndpointsRequest { 32 | K2_PAYLOAD_EMPTY; 33 | }; 34 | 35 | // Response, specifying the supported endpoints by a node 36 | struct ListEndpointsResponse { 37 | std::vector endpoints; 38 | K2_PAYLOAD_FIELDS(endpoints); 39 | }; 40 | 41 | } // ns k2 42 | -------------------------------------------------------------------------------- /src/k2/transport/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::log { 28 | inline thread_local k2::logging::Logger tx("k2::transport"); 29 | } 30 | -------------------------------------------------------------------------------- /src/k2/transport/PayloadFileUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "Payload.h" 32 | 33 | #include 34 | #include "Log.h" 35 | 36 | namespace k2 { 37 | struct fileutil { 38 | 39 | // create a directory if it doesn't exist already 40 | static bool makeDir(String path, int mode=0777); 41 | 42 | // check to see if a file exists 43 | static bool fileExists(String path); 44 | 45 | // read an entire file into a payload 46 | static bool readFile(Payload& payload, String path); 47 | 48 | // write an entire payload into the given file 49 | static bool writeFile(Payload&& payload, String path); 50 | 51 | }; // struct fileutil 52 | } // ns k2 53 | -------------------------------------------------------------------------------- /src/k2/transport/README.md: -------------------------------------------------------------------------------- 1 | 4 | # K2 Transport 5 | The K2 Transport deals with transport(e.g. TCP/eRPC/UDP) and message serialization(e.g. SEASTAR). It provides API on top of Seastar which should be used to send and receive messages 6 | 7 | 8 | # Structure 9 | See the `src/cmd/demo/service.cpp` demo app for example on how to use the module 10 | -------------------------------------------------------------------------------- /src/k2/transport/RPCHeader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "RPCHeader.h" 25 | 26 | namespace k2 { 27 | 28 | void MessageMetadata::setPayloadSize(uint32_t payloadSize) { 29 | if (payloadSize > 0) { 30 | this->payloadSize = payloadSize; 31 | this->features |= (1 << 0); // bit0 32 | } 33 | } 34 | 35 | bool MessageMetadata::isPayloadSizeSet() const { 36 | return this->features & (1 << 0); // bit0 37 | } 38 | 39 | void MessageMetadata::setRequestID(uint32_t requestID) { 40 | this->requestID = requestID; 41 | this->features |= (1 << 1); // bit1 42 | } 43 | 44 | bool MessageMetadata::isRequestIDSet() const { 45 | return this->features & (1 << 1); // bit1 46 | } 47 | 48 | void MessageMetadata::setResponseID(uint32_t responseID) { 49 | this->responseID = responseID; 50 | this->features |= (1 << 2); // bit2 51 | } 52 | 53 | bool MessageMetadata::isResponseIDSet() const { 54 | return this->features & (1 << 2); // bit2 55 | } 56 | 57 | void MessageMetadata::setChecksum(uint32_t checksum) { 58 | this->checksum = checksum; 59 | this->features |= (1 << 3); // bit3 60 | } 61 | 62 | bool MessageMetadata::isChecksumSet() const { 63 | return this->features & (1 << 3); // bit3 64 | } 65 | 66 | size_t MessageMetadata::wireByteCount() { 67 | return isPayloadSizeSet() * sizeof(payloadSize) + 68 | isRequestIDSet() * sizeof(requestID) + 69 | isResponseIDSet() * sizeof(responseID) + 70 | isChecksumSet() * sizeof(checksum); 71 | } 72 | 73 | } // namespace k2 74 | -------------------------------------------------------------------------------- /src/k2/transport/RPCProtocolFactory.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | 26 | #include "RPCProtocolFactory.h" 27 | #include "Log.h" 28 | 29 | namespace k2 { 30 | RPCProtocolFactory::RPCProtocolFactory(BuilderFunc_t builder): _builder(builder) { 31 | K2LOG_D(log::tx, "ctor"); 32 | } 33 | 34 | RPCProtocolFactory::~RPCProtocolFactory() { 35 | K2LOG_D(log::tx, "dtor"); 36 | } 37 | 38 | void RPCProtocolFactory::start() { 39 | K2LOG_D(log::tx, "start"); 40 | // Create the protocol instance 41 | _instance = _builder(); 42 | if (_instance) { 43 | _instance->start(); 44 | } 45 | } 46 | 47 | seastar::future<> RPCProtocolFactory::stop() { 48 | K2LOG_D(log::tx, "stop"); 49 | if (_instance) { 50 | // pass-on the signal to stop 51 | auto result = _instance->stop(); 52 | _instance = nullptr; 53 | return result; 54 | } 55 | return seastar::make_ready_future<>(); 56 | } 57 | 58 | seastar::shared_ptr RPCProtocolFactory::instance() { return _instance; } 59 | 60 | } // k2 61 | -------------------------------------------------------------------------------- /src/k2/transport/RPCTypes.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | // stl 26 | #include // for int types 27 | #include 28 | 29 | namespace k2 { 30 | // This file contains definitions for RPC types 31 | // The type for verbs in the RPC system 32 | 33 | // 34 | // Verb describes particular service within K2 endpoint 35 | // 36 | typedef uint8_t Verb; 37 | 38 | // Verbs used by K2 internally 39 | enum InternalVerbs : k2::Verb { 40 | LIST_ENDPOINTS = 249, // used to discover the endpoints of a node 41 | MAX_VERB = 250, // something we can use to prevent override of internal verbs. 42 | NIL // used for messages where the verb doesn't matter 43 | }; 44 | 45 | } // namespace k2 46 | -------------------------------------------------------------------------------- /src/k2/transport/Request.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "Request.h" 25 | #include 26 | #include "Log.h" 27 | 28 | namespace k2 { 29 | 30 | Request::Request(Verb verb, TXEndpoint& endpoint, MessageMetadata metadata, std::unique_ptr payload): 31 | verb(verb), 32 | endpoint(endpoint), 33 | metadata(std::move(metadata)), 34 | payload(std::move(payload)) { 35 | K2LOG_D(log::tx, "ctor Request @{}, with verb={}, from {}", ((void*)this), int(verb), endpoint.url); 36 | } 37 | 38 | Request::~Request() { 39 | K2LOG_D(log::tx, "dtor Request @{}", ((void*)this)); 40 | } 41 | 42 | Request::Request(Request&& o): 43 | verb(o.verb), 44 | endpoint(std::move(o.endpoint)), 45 | metadata(std::move(o.metadata)), 46 | payload(std::move(o.payload)) { 47 | o.verb = InternalVerbs::NIL; 48 | K2LOG_D(log::tx, "move Request @{}, with verb={}, from {}", ((void*)this), int(verb), endpoint.url); 49 | } 50 | 51 | } // k2tx 52 | -------------------------------------------------------------------------------- /src/k2/transport/Request.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | #include 28 | #include "RPCHeader.h" 29 | #include "RPCTypes.h" 30 | #include "TXEndpoint.h" 31 | 32 | namespace k2 { 33 | 34 | // This class is used to deliver a request to a message handler. It contains the message payload and some metadata 35 | class Request{ 36 | public: // lifecycle 37 | // construct a request with the given verb, endpoint, metadata and payload 38 | Request(Verb verb, TXEndpoint& endpoint, MessageMetadata metadata, std::unique_ptr payload); 39 | 40 | // move constructor 41 | Request(Request&& o); 42 | 43 | // destructor 44 | ~Request(); 45 | 46 | public: // fields 47 | 48 | // the verb for the request 49 | Verb verb; 50 | 51 | // the endpoint which sent the request 52 | TXEndpoint endpoint; 53 | 54 | // some message metadata 55 | MessageMetadata metadata; 56 | 57 | // the payload of this request 58 | std::unique_ptr payload; 59 | 60 | private: // don't need 61 | Request() = delete; 62 | Request(const Request& o) = delete; 63 | Request& operator=(const Request& o) = delete; 64 | Request& operator=(Request&& o) = delete; 65 | }; 66 | } // k2 67 | -------------------------------------------------------------------------------- /src/k2/transport/RetryStrategy.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include "RetryStrategy.h" 25 | 26 | namespace k2 { 27 | 28 | // Set the desired number of retries 29 | ExponentialBackoffStrategy& ExponentialBackoffStrategy::withRetries(int retries) { 30 | K2LOG_D(log::tx, "retries: {}", retries); 31 | _retries = retries; 32 | return *this; 33 | } 34 | 35 | // Set the exponential increase rate 36 | ExponentialBackoffStrategy& ExponentialBackoffStrategy::withRate(double rate) { 37 | K2LOG_D(log::tx, "rate: {}", rate); 38 | _rate = rate; 39 | return *this; 40 | } 41 | 42 | // Set the desired starting value 43 | ExponentialBackoffStrategy& ExponentialBackoffStrategy::withBaseBackoffTime(Duration baseBackoffTime) { 44 | K2LOG_D(log::tx, "base backoff time: {}", baseBackoffTime); 45 | _currentBackoffTime = baseBackoffTime; 46 | return *this; 47 | } 48 | 49 | } // namespace k2 50 | -------------------------------------------------------------------------------- /src/k2/transport/Util.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace k2 { 4 | 5 | inline String getHostName() { 6 | static const long max_hostname = sysconf(_SC_HOST_NAME_MAX); 7 | static const long size = (max_hostname > 255) ? max_hostname + 1 : 256; 8 | String hostname(size, '\0'); 9 | ::gethostname(hostname.data(), size - 1); 10 | hostname.resize(strlen(hostname.c_str())); 11 | return hostname; 12 | } 13 | 14 | } // ns k2 15 | -------------------------------------------------------------------------------- /src/k2/tso/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (service) 2 | add_subdirectory (client) 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/k2/tso/client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(tso_client OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (tso_client PRIVATE appbase dto config common transport Seastar::seastar) 7 | 8 | # export the library in the common k2Targets 9 | install(TARGETS tso_client EXPORT k2targets DESTINATION lib/k2) 10 | install(FILES ${HEADERS} DESTINATION include/k2/tso/client) 11 | # export the cmake config in the build tree for any users who want to use this project from source 12 | export(TARGETS tso_client NAMESPACE k2:: FILE tso_client-config.cmake) 13 | -------------------------------------------------------------------------------- /src/k2/tso/service/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library (tso_service OBJECT ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (tso_service PRIVATE appbase common dto transport Seastar::seastar) 7 | 8 | install(TARGETS tso_service DESTINATION lib/k2) 9 | install(FILES ${HEADERS} DESTINATION include/k2/tso/service) 10 | -------------------------------------------------------------------------------- /src/k2/tso/service/Exceptions.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | #include "Log.h" 25 | namespace k2::tso { 26 | // TSO service should be started with at least two cores, one controller and rest are workers. 27 | class NotEnoughCoresException : public std::exception {}; 28 | 29 | // TSO server not ready yet to issue timestamp(batch) 30 | class NotReadyException : public std::exception {}; 31 | 32 | // operations invalid during server shutdown 33 | class ShutdownException : public std::exception {}; 34 | } 35 | -------------------------------------------------------------------------------- /src/k2/tso/service/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace k2::tso::log { 28 | inline thread_local logging::Logger tsoserver("k2::tso_server"); 29 | } 30 | -------------------------------------------------------------------------------- /src/logging/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.13) 2 | 3 | project (yule VERSION 0.0.1 LANGUAGES CXX) 4 | set(CMAKE_CXX_STANDARD 17) 5 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 6 | add_compile_options(-O3 -g -Werror -Wall -Wextra) 7 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 8 | 9 | # K2_VERBOSE_LOGGING enables some verbose logging for debug purposes. By default it is disabled and compiled out 10 | if(DEFINED ENV{K2_VERBOSE_LOGGING}) 11 | set(K2_VERBOSE_LOGGING $ENV{K2_VERBOSE_LOGGING}) 12 | 13 | else() 14 | set(K2_VERBOSE_LOGGING 0) 15 | endif() 16 | 17 | include_directories(src) 18 | 19 | find_package (fmt REQUIRED) 20 | 21 | add_subdirectory (src) 22 | 23 | # install the cmake config for the entire project so that it can be used by upstream 24 | # via find_package(yule REQUIRED) 25 | # and pulling individual dependencies like so: 26 | # DEPENDS ..... yule::logging 27 | install( 28 | EXPORT yuleTargets 29 | FILE yule-config.cmake 30 | DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/yule" 31 | NAMESPACE yule:: 32 | ) 33 | -------------------------------------------------------------------------------- /src/logging/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(k2) 2 | -------------------------------------------------------------------------------- /src/logging/src/k2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(logging) 2 | -------------------------------------------------------------------------------- /src/logging/src/k2/logging/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | set(AUTOGEN_HEADER 5 | "${CMAKE_CURRENT_SOURCE_DIR}/AutoGenFormattingUtils.h" 6 | ) 7 | 8 | add_custom_command( 9 | OUTPUT ${AUTOGEN_HEADER} 10 | COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/gen_macros.py > ${AUTOGEN_HEADER} 11 | DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/gen_macros.py 12 | ) 13 | 14 | add_library(logging STATIC ${HEADERS} ${SOURCES} ${AUTOGEN_HEADER}) 15 | 16 | set_target_properties(logging PROPERTIES 17 | VERSION ${PROJECT_VERSION} 18 | SOVERSION 1 19 | ) 20 | 21 | set_target_properties(logging PROPERTIES LINKER_LANGUAGE CXX) 22 | 23 | target_link_libraries (logging PRIVATE fmt::fmt) 24 | 25 | # export the library in the logging yuleTargets 26 | install(TARGETS logging EXPORT yuleTargets DESTINATION lib/k2) 27 | install(FILES ${HEADERS} DESTINATION include/k2/logging) 28 | # export the cmake config in the build tree for any users who want to use this project from source 29 | export(TARGETS logging NAMESPACE k2:: FILE common-config.cmake) 30 | -------------------------------------------------------------------------------- /src/logging/src/k2/logging/Decimal.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2022 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace boost::multiprecision { 28 | typedef number > cpp_dec_float_25; 29 | } 30 | 31 | namespace k2 { 32 | using DecimalD25 = boost::multiprecision::cpp_dec_float_25; 33 | using DecimalD50 = boost::multiprecision::cpp_dec_float_50; 34 | using DecimalD100 = boost::multiprecision::cpp_dec_float_100; 35 | } 36 | -------------------------------------------------------------------------------- /src/logging/src/k2/logging/MacroUtils.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | // utility macro to generate overloads for up to 64-argument macros (including no-arg: 0 macro) 26 | #define _K2_OVERLOADED_MACRO(M, ...) _K2_OVR(M, _K2_NARGS(__VA_ARGS__))(__VA_ARGS__) 27 | #define _K2_OVR(macro_name, number_of_args) _K2_OVR_EXPAND(macro_name, number_of_args) 28 | #define _K2_OVR_EXPAND(macro_name, number_of_args) macro_name##number_of_args 29 | 30 | #define _K2_NARGS(...) _K2_RARGS(_, ##__VA_ARGS__, _, \ 31 | 63, 62, 61, 60, \ 32 | 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, \ 33 | 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, \ 34 | 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) 35 | #define _K2_RARGS(_, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, \ 36 | _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, \ 37 | _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, \ 38 | _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, N, ...) N 39 | -------------------------------------------------------------------------------- /src/skvhttpclient/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.13) 2 | 3 | project (skvhttpClient VERSION 0.0.1 LANGUAGES CXX) 4 | 5 | ## 6 | ## MAIN_PROJECT CHECK 7 | ## determine if we are built as a subproject (using add_subdirectory) or as the main project 8 | ## 9 | set(MAIN_PROJECT OFF) 10 | if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) 11 | set(MAIN_PROJECT ON) 12 | endif() 13 | 14 | set(CMAKE_CXX_STANDARD 17) 15 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 16 | add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) 17 | add_compile_options(-O3 -g -Werror -Wall -Wextra) 18 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 19 | 20 | include_directories(src) 21 | 22 | set(Boost_USE_STATIC_LIBS OFF) 23 | set(Boost_USE_MULTITHREADED ON) 24 | set(Boost_USE_STATIC_RUNTIME OFF) 25 | find_package(Boost COMPONENTS thread system REQUIRED) # header only libraries must not be added here 26 | find_package(Threads REQUIRED) 27 | find_package(fmt REQUIRED) 28 | find_package(yule REQUIRED) 29 | 30 | add_subdirectory (src) 31 | 32 | enable_testing() 33 | add_subdirectory (test) 34 | 35 | # install the cmake config for the entire project so that it can be used by upstream 36 | # via find_package(skv REQUIRED) 37 | # and pulling individual dependencies like so: 38 | # DEPENDS ..... skvhttp::dto skvhttp::client 39 | install( 40 | EXPORT skvhttpTargets 41 | FILE skvhttp-config.cmake 42 | DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/cmake/skvhttp" 43 | NAMESPACE skvhttp:: 44 | ) 45 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(skvhttp) 2 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory (client) 2 | add_subdirectory (common) 3 | add_subdirectory (dto) 4 | add_subdirectory (mpack) 5 | add_subdirectory (httplib) 6 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(client STATIC ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(client PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (client PRIVATE common dto mpack httplib) 12 | 13 | # export the library in the common skvhttpTargets 14 | install(TARGETS client EXPORT skvhttpTargets DESTINATION lib/skvhttp) 15 | install(FILES ${HEADERS} DESTINATION include/skvhttp/client) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS client NAMESPACE skvhttp:: FILE client-config.cmake) 18 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(common STATIC ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(common PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (common PRIVATE Boost::thread Threads::Threads fmt::fmt yule::logging) 12 | 13 | # export the library in the common skvhttpTargets 14 | install(TARGETS common EXPORT skvhttpTargets DESTINATION lib/skvhttp) 15 | install(FILES ${HEADERS} DESTINATION include/skvhttp/common) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS common NAMESPACE skvhttp:: FILE common-config.cmake) 18 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/common/VecUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | 27 | namespace skv::http { 28 | 29 | // base recursion - no args to append 30 | template 31 | void _append_vec(std::vector&) {} 32 | 33 | // Append rvalues to a vector with variadic template expansion 34 | template 35 | void _append_vec(std::vector& v, T&& obj, ObjT&&... objs) { 36 | v.push_back(std::forward(obj)); 37 | _append_vec(v, std::forward(objs)...); 38 | } 39 | 40 | template 41 | void append_vec(std::vector& v, ObjT&&... objs) { 42 | constexpr std::size_t sz = std::tuple_size>::value; 43 | v.reserve(sz); 44 | _append_vec(v, std::forward(objs)...); 45 | } 46 | 47 | // create a vector using rvalues of type. This is needed to aid in creating rvalue vectors from 48 | // non-copyable types since the standard lib initializer list doesn't support non-copyable types 49 | template 50 | std::vector make_vec(ObjT&&... objs) { 51 | std::vector result; 52 | constexpr std::size_t sz = std::tuple_size>::value; 53 | result.reserve(sz); 54 | append_vec(result, std::forward(objs)...); 55 | return result; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/dto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(dto STATIC ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(dto PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (dto PRIVATE common) 12 | 13 | # export the library in the common skvhttpTargets 14 | install(TARGETS dto EXPORT skvhttpTargets DESTINATION lib/skvhttp) 15 | install(FILES ${HEADERS} DESTINATION include/skvhttp/dto) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS dto NAMESPACE skvhttp:: FILE dto-config.cmake) 18 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/dto/Collection.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | 27 | #include "Collection.h" 28 | 29 | namespace skv::http::dto { 30 | 31 | int Key::compare(const Key& o) const noexcept { 32 | auto scomp = schemaName.compare(o.schemaName); 33 | if (scomp != 0) { 34 | return scomp; 35 | } 36 | 37 | auto pkcomp = partitionKey.compare(o.partitionKey); 38 | if (pkcomp == 0) { 39 | // if the partition keys are equal, return the comparison of the range keys 40 | return rangeKey.compare(o.rangeKey); 41 | } 42 | return pkcomp; 43 | } 44 | 45 | bool Key::operator<(const Key& o) const noexcept { 46 | return compare(o) < 0; 47 | } 48 | bool Key::operator<=(const Key& o) const noexcept { 49 | return compare(o) <= 0; 50 | } 51 | bool Key::operator>(const Key& o) const noexcept { 52 | return compare(o) > 0; 53 | } 54 | bool Key::operator>=(const Key& o) const noexcept { 55 | return compare(o) >= 0; 56 | } 57 | bool Key::operator==(const Key& o) const noexcept { 58 | return compare(o) == 0; 59 | } 60 | bool Key::operator!=(const Key& o) const noexcept { 61 | return compare(o) != 0; 62 | } 63 | size_t Key::hash() const noexcept { 64 | return k2::hash_combine(schemaName, partitionKey, rangeKey); 65 | } 66 | 67 | } // namespace skv::http::dto 68 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/dto/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #pragma once 24 | 25 | #include 26 | 27 | namespace skv::http::log { 28 | inline thread_local k2::logging::Logger dto("skv::http::dto"); 29 | } 30 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/httplib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(httplib STATIC ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(httplib PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (httplib PRIVATE common) 12 | 13 | # export the library in the common skvhttpTargets 14 | install(TARGETS httplib EXPORT skvhttpTargets DESTINATION lib/skvhttp) 15 | install(FILES ${HEADERS} DESTINATION include/skvhttp/httplib) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS httplib NAMESPACE skvhttp:: FILE httplib-config.cmake) 18 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/httplib/httplib.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2022 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | #include "httplib.h" 24 | -------------------------------------------------------------------------------- /src/skvhttpclient/src/skvhttp/mpack/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_library(mpack STATIC ${HEADERS} ${SOURCES}) 5 | 6 | set_target_properties(mpack PROPERTIES 7 | VERSION ${PROJECT_VERSION} 8 | SOVERSION 1 9 | ) 10 | 11 | target_link_libraries (mpack PRIVATE common) 12 | 13 | # export the library in the common skvhttpTargets 14 | install(TARGETS mpack EXPORT skvhttpTargets DESTINATION lib/skvhttp) 15 | install(FILES ${HEADERS} DESTINATION include/skvhttp/mpack) 16 | # export the cmake config in the build tree for any users who want to use this project from source 17 | export(TARGETS mpack NAMESPACE skvhttp:: FILE mpack-config.cmake) 18 | -------------------------------------------------------------------------------- /src/skvhttpclient/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(include) 2 | add_compile_options(-fno-var-tracking) 3 | 4 | add_executable (mpack_test mpacktest.cpp) 5 | add_executable (skv_record_test ${HEADERS} SKVRecordTest.cpp) 6 | add_executable (expression_test ${HEADERS} ExpressionTest.cpp) 7 | add_executable (client_integration_test ${HEADERS} ClientIntegrationTest.cpp) 8 | 9 | target_link_libraries (mpack_test PRIVATE mpack) 10 | target_link_libraries (skv_record_test PRIVATE dto mpack common) 11 | target_link_libraries (expression_test PRIVATE dto mpack common) 12 | target_link_libraries (client_integration_test PRIVATE client dto mpack common) 13 | 14 | add_test(NAME mpack_tst COMMAND mpack_test) 15 | add_test(NAME skv_record COMMAND skv_record_test) 16 | add_test(NAME expression COMMAND expression_test) 17 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | enable_testing() 2 | include_directories(include) 3 | add_subdirectory (cpo) 4 | add_subdirectory (transport) 5 | add_subdirectory (k23si) 6 | add_subdirectory (plog) 7 | add_subdirectory (dto) 8 | add_subdirectory (common) 9 | add_subdirectory (integration) 10 | -------------------------------------------------------------------------------- /test/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (common_test ${HEADERS} ${SOURCES}) 5 | 6 | target_link_libraries (common_test PRIVATE transport) 7 | add_test(NAME transport COMMAND common_test) 8 | -------------------------------------------------------------------------------- /test/cpo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (cpo_test ${HEADERS} CPOTest.cpp Main.cpp) 5 | add_executable(cpo_client_test CPOClientTest.cpp) 6 | 7 | target_link_libraries (cpo_test PRIVATE appbase Seastar::seastar dto tso_client transport) 8 | target_link_libraries (cpo_client_test PRIVATE appbase Seastar::seastar dto tso_client cpo_client transport common) 9 | -------------------------------------------------------------------------------- /test/cpo/CPOTest.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | 31 | namespace k2::log { 32 | inline thread_local k2::logging::Logger cpotest("k2::cpotest"); 33 | } 34 | class CPOTest { 35 | public: // application lifespan 36 | CPOTest(); 37 | ~CPOTest(); 38 | 39 | // required for seastar::distributed interface 40 | seastar::future<> gracefulStop(); 41 | seastar::future<> start(); 42 | 43 | seastar::future<> runTest1(); 44 | seastar::future<> runTest2(); 45 | seastar::future<> runTest3(); 46 | seastar::future<> runTest4(); 47 | seastar::future<> runTest5(); 48 | seastar::future<> runTest6(); 49 | seastar::future<> runTest7(); 50 | seastar::future<> runTest8(); 51 | seastar::future<> runTest9(); 52 | seastar::future<> runTest10(); 53 | seastar::future<> runTest11(); 54 | 55 | private: 56 | int exitcode = -1; 57 | std::unique_ptr _cpoEndpoint; 58 | seastar::future<> _testFuture = seastar::make_ready_future(); 59 | seastar::timer<> _testTimer; 60 | seastar::shared_ptr _tsoClient; 61 | }; 62 | -------------------------------------------------------------------------------- /test/cpo/Main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include "CPOTest.h" 27 | 28 | int main(int argc, char** argv) { 29 | k2::App app("CPOTest"); 30 | app.addOptions()("cpo", bpo::value(), "The endpoint of the CPO service"); 31 | app.addApplet(); 32 | app.addApplet(); 33 | return app.start(argc, argv); 34 | } 35 | -------------------------------------------------------------------------------- /test/dto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (partition_test ${HEADERS} PartitionTest.cpp) 5 | 6 | target_link_libraries (partition_test PRIVATE Seastar::seastar dto transport cpo_client tso_client k23si_client appbase) 7 | -------------------------------------------------------------------------------- /test/include/TestUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2020 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | #pragma once 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | namespace k2::log { 31 | inline thread_local k2::logging::Logger testutil("k2::testutil"); 32 | } 33 | std::string generateTempFolderPath(const char* id) 34 | { 35 | char folder[255]; 36 | int res = snprintf(folder, sizeof(folder), "/%s_%lx%x%x/", id ? id : "", pthread_self(), (uint32_t)time(nullptr), (uint32_t)rand()); 37 | K2ASSERT(k2::log::testutil, res > 0 && res < (int)sizeof(folder), "unable to construct folder name"); 38 | 39 | return std::filesystem::temp_directory_path().concat(folder); 40 | } 41 | -------------------------------------------------------------------------------- /test/include/catch2/catch_reporter_automake.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Created by Justin R. Wilson on 2/19/2017. 3 | * Copyright 2017 Justin R. Wilson. All rights reserved. 4 | * 5 | * Distributed under the Boost Software License, Version 1.0. (See accompanying 6 | * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 7 | */ 8 | #ifndef TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED 9 | #define TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED 10 | 11 | // Don't #include any Catch headers here - we can assume they are already 12 | // included before this header. 13 | // This is not good practice in general but is necessary in this case so this 14 | // file can be distributed as a single header that works with the main 15 | // Catch single header. 16 | 17 | namespace Catch { 18 | 19 | struct AutomakeReporter : StreamingReporterBase { 20 | AutomakeReporter( ReporterConfig const& _config ) 21 | : StreamingReporterBase( _config ) 22 | {} 23 | 24 | ~AutomakeReporter() override; 25 | 26 | static std::string getDescription() { 27 | return "Reports test results in the format of Automake .trs files"; 28 | } 29 | 30 | void assertionStarting( AssertionInfo const& ) override {} 31 | 32 | bool assertionEnded( AssertionStats const& /*_assertionStats*/ ) override { return true; } 33 | 34 | void testCaseEnded( TestCaseStats const& _testCaseStats ) override { 35 | // Possible values to emit are PASS, XFAIL, SKIP, FAIL, XPASS and ERROR. 36 | stream << ":test-result: "; 37 | if (_testCaseStats.totals.assertions.allPassed()) { 38 | stream << "PASS"; 39 | } else if (_testCaseStats.totals.assertions.allOk()) { 40 | stream << "XFAIL"; 41 | } else { 42 | stream << "FAIL"; 43 | } 44 | stream << ' ' << _testCaseStats.testInfo.name << '\n'; 45 | StreamingReporterBase::testCaseEnded( _testCaseStats ); 46 | } 47 | 48 | void skipTest( TestCaseInfo const& testInfo ) override { 49 | stream << ":test-result: SKIP " << testInfo.name << '\n'; 50 | } 51 | 52 | }; 53 | 54 | #ifdef CATCH_IMPL 55 | AutomakeReporter::~AutomakeReporter() {} 56 | #endif 57 | 58 | CATCH_REGISTER_REPORTER( "automake", AutomakeReporter) 59 | 60 | } // end namespace Catch 61 | 62 | #endif // TWOBLUECUBES_CATCH_REPORTER_AUTOMAKE_HPP_INCLUDED 63 | -------------------------------------------------------------------------------- /test/integration/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | install(PROGRAMS run_k2_cluster.sh DESTINATION bin) 2 | -------------------------------------------------------------------------------- /test/integration/common_defs.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | CPODIR=/tmp/___cpo_integ_test 3 | rm -rf ${CPODIR} 4 | EPS=("tcp+k2rpc://0.0.0.0:10000" "tcp+k2rpc://0.0.0.0:10001" "tcp+k2rpc://0.0.0.0:10002" "tcp+k2rpc://0.0.0.0:10003" "tcp+k2rpc://0.0.0.0:10004") 5 | NUMCORES=`nproc` 6 | # core on which to run the TSO poller thread. Pick 4 if we have that many, or the highest-available otherwise 7 | #TSO_POLLER_CORE=$(( 5 > $NUMCORES ? $NUMCORES-1 : 4 )) 8 | # poll on random core 9 | TSO_POLLER_CORE=-1 10 | PERSISTENCE=tcp+k2rpc://0.0.0.0:12001 11 | CPO=tcp+k2rpc://0.0.0.0:9000 12 | TSO=tcp+k2rpc://0.0.0.0:13000 13 | COMMON_ARGS="--reactor-backend=epoll --enable_tx_checksum true --thread-affinity false" 14 | HTTP=tcp+k2rpc://0.0.0.0:20000 15 | -------------------------------------------------------------------------------- /test/integration/failing_test_logstream.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start CPO 8 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} 9001 --data_dir ${CPODIR} --prometheus_port 63000 & 9 | cpo_child_pid=$! 10 | 11 | # start plog 12 | ./build/src/k2/cmd/plog/plog_main ${COMMON_ARGS} -c3 -m500M --tcp_endpoints 10000 10001 10002 --prometheus_port=63001 & 13 | plog_child_pid=$! 14 | 15 | function finish { 16 | # cleanup code 17 | rm -rf ${CPODIR} 18 | 19 | kill ${cpo_child_pid} 20 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 21 | wait ${cpo_child_pid} 22 | 23 | kill ${plog_child_pid} 24 | echo "Waiting for plog child pid: ${plog_child_pid}" 25 | wait ${plog_child_pid} 26 | } 27 | trap finish EXIT 28 | 29 | sleep 1 30 | 31 | ./build/test/plog/logstream_test ${COMMON_ARGS} -m 500M --cpo_url ${CPO} --tcp_endpoints 12345 --plog_server_endpoints tcp+k2rpc://0.0.0.0:10000 tcp+k2rpc://0.0.0.0:10001 tcp+k2rpc://0.0.0.0:10002 --prometheus_port=63002 32 | -------------------------------------------------------------------------------- /test/integration/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | topname=$(dirname "$0") 3 | cd ${topname} 4 | 5 | set -e 6 | 7 | for test in test_*.sh; do 8 | echo ">>> Running integration test: ${test}"; 9 | ./${test}; 10 | echo ">>> Done running test ${test}"; 11 | done 12 | -------------------------------------------------------------------------------- /test/integration/test_3si_txn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} --log_level Info k2::skv_server=Info -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --partition_request_timeout=30s & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | # start CPO 20 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 21 | cpo_child_pid=$! 22 | 23 | 24 | function finish { 25 | rv=$? 26 | # cleanup code 27 | rm -rf ${CPODIR} 28 | 29 | kill ${cpo_child_pid} 30 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 31 | wait ${cpo_child_pid} 32 | 33 | kill ${nodepool_child_pid} 34 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 35 | wait ${nodepool_child_pid} 36 | 37 | kill ${persistence_child_pid} 38 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 39 | wait ${persistence_child_pid} 40 | 41 | kill ${tso_child_pid} 42 | echo "Waiting for tso child pid: ${tso_child_pid}" 43 | wait ${tso_child_pid} 44 | echo ">>>> Test ${0} finished with code ${rv}" 45 | } 46 | trap finish EXIT 47 | 48 | sleep 1 49 | 50 | ./build/test/k23si/3si_txn_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 51 | -------------------------------------------------------------------------------- /test/integration/test_collection.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=20us --persistence_endpoints ${PERSISTENCE} & 20 | cpo_child_pid=$! 21 | 22 | function finish { 23 | rv=$? 24 | # cleanup code 25 | rm -rf ${CPODIR} 26 | 27 | kill ${cpo_child_pid} 28 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 29 | wait ${cpo_child_pid} 30 | 31 | kill ${nodepool_child_pid} 32 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 33 | wait ${nodepool_child_pid} 34 | 35 | kill ${persistence_child_pid} 36 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 37 | wait ${persistence_child_pid} 38 | 39 | kill ${tso_child_pid} 40 | echo "Waiting for tso child pid: ${tso_child_pid}" 41 | wait ${tso_child_pid} 42 | echo ">>>> Test ${0} finished with code ${rv}" 43 | } 44 | trap finish EXIT 45 | 46 | sleep 1 47 | 48 | ./build/test/cpo/cpo_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 49 | -------------------------------------------------------------------------------- /test/integration/test_heartbeat_monitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} --log_level Info k2::skv_server=Info -c1 --tcp_endpoints ${EPS[0]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --partition_request_timeout=30s & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} 13001 --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | # start CPO 20 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[0]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 21 | cpo_child_pid=$! 22 | 23 | 24 | function finish { 25 | rv=$? 26 | # cleanup code 27 | rm -rf ${CPODIR} 28 | 29 | kill ${cpo_child_pid} 30 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 31 | wait ${cpo_child_pid} 32 | 33 | kill ${persistence_child_pid} 34 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 35 | wait ${persistence_child_pid} 36 | 37 | kill ${tso_child_pid} 38 | echo "Waiting for tso child pid: ${tso_child_pid}" 39 | wait ${tso_child_pid} 40 | echo ">>>> Test ${0} finished with code ${rv}" 41 | } 42 | trap finish EXIT 43 | 44 | sleep 1 45 | 46 | ./test/integration/heartbeat_monitor.py --nodepool_pid=${nodepool_child_pid} --prometheus_port=63000 47 | -------------------------------------------------------------------------------- /test/integration/test_http_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start CPO 8 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} 9001 --data_dir ${CPODIR} --prometheus_port 63000 --assignment_timeout=3s --txn_heartbeat_deadline=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE}& 9 | cpo_child_pid=$! 10 | 11 | # start nodepool 12 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --memory=1G --partition_request_timeout=6s & 13 | nodepool_child_pid=$! 14 | 15 | # start persistence 16 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 17 | persistence_child_pid=$! 18 | 19 | # start tso 20 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 21 | tso_child_pid=$! 22 | 23 | sleep 1 24 | 25 | # start http proxy with increased cpo request timeout as delete collection takes more time 26 | ./build/src/k2/cmd/httpproxy/http_proxy ${COMMON_ARGS} -c1 --tcp_endpoints ${HTTP} --memory=1G --cpo ${CPO} --httpproxy_txn_timeout=100ms --httpproxy_expiry_timer_interval=50ms --cpo_request_timeout=6s& 27 | http_child_pid=$! 28 | 29 | function finish { 30 | rv=$? 31 | # cleanup code 32 | rm -rf ${CPODIR} 33 | 34 | kill ${nodepool_child_pid} 35 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 36 | wait ${nodepool_child_pid} 37 | 38 | kill ${cpo_child_pid} 39 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 40 | wait ${cpo_child_pid} 41 | 42 | kill ${persistence_child_pid} 43 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 44 | wait ${persistence_child_pid} 45 | 46 | kill ${tso_child_pid} 47 | echo "Waiting for tso child pid: ${tso_child_pid}" 48 | wait ${tso_child_pid} 49 | 50 | kill ${http_child_pid} 51 | echo "Waiting for http child pid: ${http_child_pid}" 52 | wait ${http_child_pid} 53 | 54 | echo ">>>> Test ${0} finished with code ${rv}" 55 | } 56 | trap finish EXIT 57 | sleep 1 58 | 59 | echo ">>> Starting http test ..." 60 | PYTHONPATH=${PYTHONPATH}:./test/integration ./test/integration/test_http.py --http http://127.0.0.1:30000 61 | -------------------------------------------------------------------------------- /test/integration/test_http_cpp_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start CPO 8 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} 9001 --data_dir ${CPODIR} --prometheus_port 63000 --assignment_timeout=3s --txn_heartbeat_deadline=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE}& 9 | cpo_child_pid=$! 10 | 11 | # start nodepool 12 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --memory=1G --partition_request_timeout=6s & 13 | nodepool_child_pid=$! 14 | 15 | # start persistence 16 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 17 | persistence_child_pid=$! 18 | 19 | # start tso 20 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 21 | tso_child_pid=$! 22 | 23 | sleep 1 24 | # start http proxy with increased cpo request timeout as delete collection takes more time 25 | ./build/src/k2/cmd/httpproxy/http_proxy ${COMMON_ARGS} -c1 --tcp_endpoints ${HTTP} --log_level=Info k2::transport=Info --memory=1G --cpo ${CPO} --cpo_request_timeout=6s& 26 | http_child_pid=$! 27 | 28 | function finish { 29 | rv=$? 30 | # cleanup code 31 | rm -rf ${CPODIR} 32 | 33 | kill ${nodepool_child_pid} 34 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 35 | wait ${nodepool_child_pid} 36 | 37 | kill ${cpo_child_pid} 38 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 39 | wait ${cpo_child_pid} 40 | 41 | kill ${persistence_child_pid} 42 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 43 | wait ${persistence_child_pid} 44 | 45 | kill ${tso_child_pid} 46 | echo "Waiting for tso child pid: ${tso_child_pid}" 47 | wait ${tso_child_pid} 48 | 49 | kill ${http_child_pid} 50 | echo "Waiting for http child pid: ${http_child_pid}" 51 | wait ${http_child_pid} 52 | 53 | echo ">>>> Test ${0} finished with code ${rv}" 54 | } 55 | trap finish EXIT 56 | sleep 1 57 | 58 | echo ">>> Starting http test ..." 59 | ./src/skvhttpclient/build/test/client_integration_test 60 | -------------------------------------------------------------------------------- /test/integration/test_k23si.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} --log_level Info k2::skv_server=Info -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 20 | cpo_child_pid=$! 21 | 22 | function finish { 23 | rv=$? 24 | # cleanup code 25 | rm -rf ${CPODIR} 26 | 27 | kill ${cpo_child_pid} 28 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 29 | wait ${cpo_child_pid} 30 | 31 | kill ${nodepool_child_pid} 32 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 33 | wait ${nodepool_child_pid} 34 | 35 | kill ${persistence_child_pid} 36 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 37 | wait ${persistence_child_pid} 38 | 39 | kill ${tso_child_pid} 40 | echo "Waiting for tso child pid: ${tso_child_pid}" 41 | wait ${tso_child_pid} 42 | echo ">>>> Test ${0} finished with code ${rv}" 43 | } 44 | trap finish EXIT 45 | 46 | sleep 1 47 | 48 | ./build/test/k23si/k23si_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 49 | -------------------------------------------------------------------------------- /test/integration/test_k23si_tpcc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --memory=3G --partition_request_timeout=6s & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} --heartbeat_interval=1s & 20 | cpo_child_pid=$! 21 | 22 | 23 | function finish { 24 | rv=$? 25 | # cleanup code 26 | rm -rf ${CPODIR} 27 | 28 | kill ${nodepool_child_pid} 29 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 30 | wait ${nodepool_child_pid} 31 | 32 | kill ${cpo_child_pid} 33 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 34 | wait ${cpo_child_pid} 35 | 36 | kill ${persistence_child_pid} 37 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 38 | wait ${persistence_child_pid} 39 | 40 | kill ${tso_child_pid} 41 | echo "Waiting for tso child pid: ${tso_child_pid}" 42 | wait ${tso_child_pid} 43 | echo ">>>> Test ${0} finished with code ${rv}" 44 | } 45 | trap finish EXIT 46 | 47 | sleep 1 48 | 49 | NUMWH=1 50 | NUMDIST=1 51 | echo ">>> Starting load ..." 52 | ./build/src/k2/cmd/tpcc/tpcc_client ${COMMON_ARGS} -c1 --cpo ${CPO} --data_load true --num_warehouses ${NUMWH} --districts_per_warehouse ${NUMDIST} --prometheus_port 63100 --memory=512M --partition_request_timeout=6s --dataload_txn_timeout=600s --do_verification true --num_concurrent_txns=2 --item_table_num_nodes=1 --txn_weights={43,4,4,45,4} --warehouse_tables_num_nodes=2 --log_level=Info k2::tpcc=Info k2::skv_client=Info log::txretry=Info 53 | 54 | sleep 1 55 | 56 | echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" 57 | echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" 58 | echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" 59 | echo ">>> Starting benchmark ..." 60 | ./build/src/k2/cmd/tpcc/tpcc_client ${COMMON_ARGS} -c1 --cpo ${CPO} --num_warehouses ${NUMWH} --districts_per_warehouse ${NUMDIST} --prometheus_port 63101 --memory=512M --partition_request_timeout=6s --num_concurrent_txns=1 --do_verification true --delivery_txn_batch_size=10 --txn_weights={43,4,4,45,4} 61 | -------------------------------------------------------------------------------- /test/integration/test_partition.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start CPO 8 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --prometheus_port 63000 --assignment_timeout=100ms --tso_error_bound=100us --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} & 9 | cpo_child_pid=$! 10 | 11 | # start tso 12 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 13 | tso_child_pid=$! 14 | 15 | # start persistence 16 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 17 | persistence_child_pid=$! 18 | 19 | # start nodepool 20 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 21 | nodepool_child_pid=$! 22 | 23 | function finish { 24 | rv=$? 25 | # cleanup code 26 | rm -rf ${CPODIR} 27 | 28 | kill ${cpo_child_pid} 29 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 30 | wait ${cpo_child_pid} 31 | 32 | kill ${nodepool_child_pid} 33 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 34 | wait ${nodepool_child_pid} 35 | 36 | kill ${persistence_child_pid} 37 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 38 | wait ${persistence_child_pid} 39 | 40 | kill ${tso_child_pid} 41 | echo "Waiting for tso child pid: ${tso_child_pid}" 42 | wait ${tso_child_pid} 43 | echo ">>>> Test ${0} finished with code ${rv}" 44 | } 45 | trap finish EXIT 46 | 47 | ./build/test/dto/partition_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 48 | -------------------------------------------------------------------------------- /test/integration/test_partition_assigned.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | new_nodepool="tcp+k2rpc://0.0.0.0:10099" 7 | 8 | # start nodepool 9 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 10 | nodepool_child_pid=$! 11 | 12 | # start persistence 13 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 14 | persistence_child_pid=$! 15 | 16 | # start tso 17 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 18 | tso_child_pid=$! 19 | 20 | # starting the CPO with a nodepool that won't exist at start up 21 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} $new_nodepool --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 22 | cpo_child_pid=$! 23 | 24 | function finish { 25 | rv=$? 26 | # cleanup code 27 | rm -rf ${CPODIR} 28 | 29 | kill ${cpo_child_pid} 30 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 31 | wait ${cpo_child_pid} 32 | 33 | kill ${nodepool_child_pid} 34 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 35 | wait ${nodepool_child_pid} 36 | 37 | kill ${persistence_child_pid} 38 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 39 | wait ${persistence_child_pid} 40 | 41 | kill ${tso_child_pid} 42 | echo "Waiting for tso child pid: ${tso_child_pid}" 43 | wait ${tso_child_pid} 44 | echo ">>>> Test ${0} finished with code ${rv}" 45 | } 46 | trap finish EXIT 47 | 48 | sleep 1 49 | 50 | ./build/test/cpo/cpo_client_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 51 | -------------------------------------------------------------------------------- /test/integration/test_plog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start CPO 8 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} 9001 --data_dir ${CPODIR} --prometheus_port 63000 --assignment_timeout=1s & 9 | cpo_child_pid=$! 10 | 11 | # start plog 12 | ./build/src/k2/cmd/plog/plog_main ${COMMON_ARGS} -c3 --tcp_endpoints 10000 10001 10002 --prometheus_port=63001 & 13 | plog_child_pid=$! 14 | 15 | function finish { 16 | rv=$? 17 | # cleanup code 18 | rm -rf ${CPODIR} 19 | 20 | kill ${cpo_child_pid} 21 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 22 | wait ${cpo_child_pid} 23 | 24 | kill ${plog_child_pid} 25 | echo "Waiting for plog child pid: ${plog_child_pid}" 26 | wait ${plog_child_pid} 27 | echo ">>>> Test ${0} finished with code ${rv}" 28 | } 29 | trap finish EXIT 30 | 31 | sleep 1 32 | 33 | ./build/test/plog/plog_test ${COMMON_ARGS} --cpo_url ${CPO} --tcp_endpoints 12345 --plog_server_endpoints tcp+k2rpc://0.0.0.0:10000 tcp+k2rpc://0.0.0.0:10001 tcp+k2rpc://0.0.0.0:10002 --prometheus_port=63002 34 | -------------------------------------------------------------------------------- /test/integration/test_query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c2 --tcp_endpoints ${EPS[@]:0:2} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 --k23si_query_pagination_limit 2 --k23si_query_scan_limit 6 & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]:0:2} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 20 | cpo_child_pid=$! 21 | 22 | function finish { 23 | rv=$? 24 | # cleanup code 25 | rm -rf ${CPODIR} 26 | 27 | kill ${cpo_child_pid} 28 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 29 | wait ${cpo_child_pid} 30 | 31 | kill ${nodepool_child_pid} 32 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 33 | wait ${nodepool_child_pid} 34 | 35 | kill ${persistence_child_pid} 36 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 37 | wait ${persistence_child_pid} 38 | 39 | kill ${tso_child_pid} 40 | echo "Waiting for tso child pid: ${tso_child_pid}" 41 | wait ${tso_child_pid} 42 | echo ">>>> Test ${0} finished with code ${rv}" 43 | } 44 | trap finish EXIT 45 | 46 | sleep 1 47 | 48 | ./build/test/k23si/query_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 49 | -------------------------------------------------------------------------------- /test/integration/test_schema_create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} 13001 --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 20 | cpo_child_pid=$! 21 | 22 | function finish { 23 | rv=$? 24 | # cleanup code 25 | rm -rf ${CPODIR} 26 | 27 | kill ${cpo_child_pid} 28 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 29 | wait ${cpo_child_pid} 30 | 31 | kill ${nodepool_child_pid} 32 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 33 | wait ${nodepool_child_pid} 34 | 35 | kill ${persistence_child_pid} 36 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 37 | wait ${persistence_child_pid} 38 | 39 | kill ${tso_child_pid} 40 | echo "Waiting for tso child pid: ${tso_child_pid}" 41 | wait ${tso_child_pid} 42 | echo ">>>> Test ${0} finished with code ${rv}" 43 | } 44 | trap finish EXIT 45 | 46 | sleep 1 47 | 48 | ./build/test/k23si/schema_creation_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 49 | -------------------------------------------------------------------------------- /test/integration/test_skv_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start nodepool 8 | ./build/src/k2/cmd/nodepool/nodepool ${COMMON_ARGS} --log_level Info k2::skv_server=Info -c${#EPS[@]} --tcp_endpoints ${EPS[@]} --k23si_persistence_endpoint ${PERSISTENCE} --prometheus_port 63001 & 9 | nodepool_child_pid=$! 10 | 11 | # start persistence 12 | ./build/src/k2/cmd/persistence/persistence ${COMMON_ARGS} -c1 --tcp_endpoints ${PERSISTENCE} --prometheus_port 63002 & 13 | persistence_child_pid=$! 14 | 15 | # start tso 16 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 17 | tso_child_pid=$! 18 | 19 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --nodepool_endpoints ${EPS[@]} --tso_endpoints ${TSO} --tso_error_bound=100us --persistence_endpoints ${PERSISTENCE} & 20 | cpo_child_pid=$! 21 | 22 | function finish { 23 | rv=$? 24 | # cleanup code 25 | rm -rf ${CPODIR} 26 | 27 | kill ${cpo_child_pid} 28 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 29 | wait ${cpo_child_pid} 30 | 31 | kill ${nodepool_child_pid} 32 | echo "Waiting for nodepool child pid: ${nodepool_child_pid}" 33 | wait ${nodepool_child_pid} 34 | 35 | kill ${persistence_child_pid} 36 | echo "Waiting for persistence child pid: ${persistence_child_pid}" 37 | wait ${persistence_child_pid} 38 | 39 | kill ${tso_child_pid} 40 | echo "Waiting for tso child pid: ${tso_child_pid}" 41 | wait ${tso_child_pid} 42 | echo ">>>> Test ${0} finished with code ${rv}" 43 | } 44 | trap finish EXIT 45 | 46 | sleep 1 47 | 48 | ./build/test/k23si/skv_client_test ${COMMON_ARGS} --cpo ${CPO} --prometheus_port 63100 49 | -------------------------------------------------------------------------------- /test/integration/test_tso_assignment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | allTSOs=("tcp+k2rpc://0.0.0.0:13000" "tcp+k2rpc://0.0.0.0:13001" "tcp+k2rpc://0.0.0.0:13002") 8 | # start tso 9 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 10 | tso_child_pid=$! 11 | 12 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --tso_endpoints ${allTSOs[@]} --tso_error_bound=100us & 13 | cpo_child_pid=$! 14 | 15 | 16 | function finish { 17 | rv=$? 18 | # cleanup code 19 | rm -rf ${CPODIR} 20 | 21 | kill ${cpo_child_pid} 22 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 23 | wait ${cpo_child_pid} 24 | 25 | kill ${tso_child_pid} 26 | echo "Waiting for tso child pid: ${tso_child_pid}" 27 | wait ${tso_child_pid} 28 | echo ">>>> Test ${0} finished with code ${rv}" 29 | } 30 | trap finish EXIT 31 | 32 | sleep 1 33 | 34 | ./test/integration/test_cpo_tso_assign.py --tso_child_pid=${tso_child_pid} --prometheus_port=63000 --cmd="./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c2 --tcp_endpoints \"tcp+k2rpc://0.0.0.0:13001\" \"tcp+k2rpc://0.0.0.0:13002\" --prometheus_port 63004 --tso.clock_poller_cpu=${TSO_POLLER_CORE}"\ 35 | -------------------------------------------------------------------------------- /test/integration/test_tso_errorbound.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | topname=$(dirname "$0") 4 | source ${topname}/common_defs.sh 5 | cd ${topname}/../.. 6 | 7 | # start tso 8 | ./build/src/k2/cmd/tso/tso ${COMMON_ARGS} -c1 --tcp_endpoints ${TSO} --prometheus_port 63003 --tso.clock_poller_cpu=${TSO_POLLER_CORE} & 9 | tso_child_pid=$! 10 | 11 | #assigning unrealistic error bound to tso 12 | ./build/src/k2/cmd/controlPlaneOracle/cpo_main ${COMMON_ARGS} -c1 --tcp_endpoints ${CPO} --data_dir ${CPODIR} --txn_heartbeat_deadline=10s --prometheus_port 63000 --assignment_timeout=1s --tso_endpoints ${TSO} --tso_error_bound=1us & 13 | cpo_child_pid=$! 14 | 15 | 16 | function finish { 17 | rv=$? 18 | # cleanup code 19 | rm -rf ${CPODIR} 20 | 21 | kill ${cpo_child_pid} 22 | echo "Waiting for cpo child pid: ${cpo_child_pid}" 23 | wait ${cpo_child_pid} 24 | 25 | kill ${tso_child_pid} 26 | echo "Waiting for tso child pid: ${tso_child_pid}" 27 | wait ${tso_child_pid} 28 | echo ">>>> Test ${0} finished with code ${rv}" 29 | } 30 | trap finish EXIT 31 | 32 | sleep 1 33 | 34 | ./test/integration/test_tso_errorbound_fail.py --prometheus_port=63003 35 | -------------------------------------------------------------------------------- /test/integration/test_tso_errorbound_fail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | MIT License 5 | 6 | Copyright (c) 2022 Futurewei Cloud 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | ''' 26 | 27 | import argparse, unittest, sys, os, signal, time 28 | import requests, json 29 | from urllib.parse import urlparse 30 | 31 | parser = argparse.ArgumentParser() 32 | parser.add_argument("--prometheus_port", help="TSO prometheus_port") 33 | args = parser.parse_args() 34 | url = "http://127.0.0.1:" + args.prometheus_port + "/metrics" 35 | 36 | TEST_TIMEOUT = 2 37 | TEST_RETRY_INTERVAL = 0.1 38 | 39 | def get_failed_errbound_times(): 40 | r = requests.get(url) 41 | count = 0 42 | for line in r.text.splitlines(): 43 | # print(line) 44 | if "TSOService_TSO_timestamp_errorbound_count" in line: 45 | try: 46 | count = int(float(line.split()[1])) 47 | except: 48 | continue 49 | return count 50 | 51 | class TestTSOErrorBoundFailure(unittest.TestCase): 52 | def test_tsoEBFailure(self): 53 | time_start = time.time() 54 | count = -1 55 | while time.time() < time_start + TEST_TIMEOUT: 56 | try: 57 | count = get_failed_errbound_times() 58 | # print("count is: ", count) 59 | if count > 0: 60 | break 61 | time.sleep(TEST_RETRY_INTERVAL) 62 | except: 63 | continue 64 | self.assertTrue(count > 0) 65 | 66 | del sys.argv[1:] 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /test/k23si/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (k23si_test ${HEADERS} K23SITest.cpp) 5 | add_executable (indexer_test ${HEADERS} IndexerTest.cpp) 6 | add_executable (skv_record_test ${HEADERS} SKVRecordTest.cpp) 7 | add_executable (key_encoding_test ${HEADERS} KeyEncodingTest.cpp) 8 | add_executable (schema_creation_test ${HEADERS} SchemaCreationTest.cpp) 9 | add_executable (skv_ser_test ${HEADERS} SKVSerTest.cpp) 10 | add_executable (3si_txn_test ${HEADERS} 3SITxnTest.cpp) 11 | add_executable (skv_client_test ${HEADERS} SKVClientTest.cpp) 12 | add_executable (query_test ${HEADERS} QueryTest.cpp) 13 | add_executable (expression_test ${HEADERS} ExpressionTest.cpp) 14 | 15 | target_link_libraries (k23si_test PRIVATE appbase Seastar::seastar tso_client k23si cpo_client infrastructure dto transport) 16 | target_link_libraries (indexer_test PRIVATE k23si tso_client cpo_client infrastructure dto transport appbase Seastar::seastar) 17 | target_link_libraries (skv_record_test PRIVATE dto transport) 18 | target_link_libraries (key_encoding_test PRIVATE dto transport) 19 | target_link_libraries (schema_creation_test PRIVATE appbase Seastar::seastar dto transport) 20 | target_link_libraries (skv_ser_test PRIVATE dto transport) 21 | target_link_libraries (3si_txn_test PRIVATE appbase dto transport tso_client Seastar::seastar) 22 | target_link_libraries (skv_client_test PRIVATE tso_client k23si_client cpo_client appbase dto transport Seastar::seastar) 23 | target_link_libraries (query_test PRIVATE tso_client k23si_client cpo_client appbase dto transport Seastar::seastar) 24 | target_link_libraries (expression_test PRIVATE dto transport Seastar::seastar) 25 | 26 | add_test(NAME indexer COMMAND indexer_test) 27 | add_test(NAME skv_record COMMAND skv_record_test) 28 | add_test(NAME key_encoding COMMAND key_encoding_test) 29 | add_test(NAME skv_ser COMMAND skv_ser_test) 30 | add_test(NAME expression COMMAND expression_test) 31 | -------------------------------------------------------------------------------- /test/k23si/Log.h: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright(c) 2021 Futurewei Cloud 5 | 6 | Permission is hereby granted, 7 | free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions : 8 | 9 | The above copyright notice and this permission notice shall be included in all copies 10 | or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", 14 | WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER 18 | LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | */ 23 | 24 | namespace k2::log { 25 | inline thread_local k2::logging::Logger k23si("k2::k23si_test"); 26 | } 27 | -------------------------------------------------------------------------------- /test/plog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (plog_test ${HEADERS} PlogClientTest.cpp) 5 | add_executable (logstream_test ${HEADERS} LogStreamTest.cpp) 6 | 7 | target_link_libraries (plog_test PRIVATE appbase cpo_client transport common plog_client Seastar::seastar dto) 8 | target_link_libraries (logstream_test PRIVATE appbase cpo_client plog_client transport common log_stream Seastar::seastar dto) 9 | -------------------------------------------------------------------------------- /test/transport/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB HEADERS "*.h") 2 | file(GLOB SOURCES "*.cpp") 3 | 4 | add_executable (payload_test ${HEADERS} PayloadTest.cpp) 5 | target_link_libraries (payload_test PRIVATE transport) 6 | add_test(NAME transport_payload COMMAND payload_test) 7 | 8 | add_executable (endpoint_test ${HEADERS} EndpointTest.cpp) 9 | target_link_libraries (endpoint_test PRIVATE transport) 10 | add_test(NAME transport_endpoint COMMAND endpoint_test) 11 | --------------------------------------------------------------------------------