├── .gitignore ├── .gitmodules ├── .travis.yml ├── CPPLINT.cfg ├── License.md ├── Makefile.am ├── README.md ├── benchmarks ├── .gitignore ├── 1_1.cpp ├── 1_2.cpp ├── 1_3.cpp ├── Makefile.am ├── README.md ├── cache_benchmark.cpp ├── cache_iops.cpp ├── cache_latency.cpp ├── iterator_benchmark.cpp ├── iterator_v2.cpp ├── outstanding_requests.cpp ├── remove_bulk.cpp ├── run_benchmarks.py ├── store_bandwidth.cpp └── throughput.cpp ├── bootstrap.sh ├── common.mk ├── configure.ac ├── docs ├── Doxyfile ├── addtl_documentation.dox └── mainpage.dox ├── examples ├── Makefile.am ├── graphs │ ├── .gitignore │ ├── Makefile.am │ ├── data │ │ ├── graph_1 │ │ ├── graph_2 │ │ └── graph_3 │ ├── page_rank │ │ ├── .gitignore │ │ ├── Input.cpp │ │ ├── Input.h │ │ ├── Makefile.am │ │ ├── PageRank.cpp │ │ ├── PageRank.h │ │ ├── Vertex.cpp │ │ ├── Vertex.h │ │ └── pr.cpp │ └── wcc │ │ ├── .gitignore │ │ ├── Input.cpp │ │ ├── Input.h │ │ ├── Makefile.am │ │ ├── Vertex.cpp │ │ ├── Vertex.h │ │ ├── Wcc.cpp │ │ ├── Wcc.h │ │ └── main.cpp └── ml │ ├── .gitignore │ ├── Checksum.cpp │ ├── Checksum.h │ ├── Configuration.cpp │ ├── Configuration.h │ ├── Dataset.cpp │ ├── Dataset.h │ ├── Input.cpp │ ├── Input.h │ ├── LRModel.cpp │ ├── LRModel.h │ ├── Makefile.am │ ├── Matrix.cpp │ ├── Matrix.h │ ├── MlUtils.cpp │ ├── MlUtils.h │ ├── Model.cpp │ ├── Model.h │ ├── ModelGradient.cpp │ ├── ModelGradient.h │ ├── Serializers.cpp │ ├── Serializers.h │ ├── SoftmaxModel.cpp │ ├── SoftmaxModel.h │ ├── Tasks.cpp │ ├── Tasks.h │ ├── Tasks_softmax.cpp │ ├── Tasks_softmax.h │ ├── Utils.cpp │ ├── Utils.h │ ├── bug_regression.cpp │ ├── configs │ ├── criteo_fbox.cfg │ ├── hosts │ └── mnist_fbox.cfg │ ├── configure.ac │ ├── configure.gnu │ ├── data │ └── day_0_test_1K.csv │ ├── parameter_server.cpp │ ├── parameter_server_softmax.cpp │ ├── run.sh │ ├── run_regression.sh │ ├── run_softmax.sh │ ├── two_clients_test.cpp │ └── vg_suppressions_cirrus ├── scripts ├── README.md ├── disable_swapping.sh └── setup_lxc.sh ├── src ├── Makefile.am ├── authentication │ ├── ApplicationKey.cpp │ ├── ApplicationKey.h │ ├── AuthenticationToken.cpp │ ├── AuthenticationToken.h │ ├── Authenticator.h │ ├── GrantingKey.cpp │ ├── GrantingKey.h │ └── Makefile.am ├── cache_manager │ ├── CacheManager.h │ ├── EvictionPolicy.cpp │ ├── EvictionPolicy.h │ ├── LRAddedEvictionPolicy.cpp │ ├── LRAddedEvictionPolicy.h │ ├── LRUEvictionPolicy.cpp │ ├── LRUEvictionPolicy.h │ ├── Makefile.am │ └── PrefetchPolicy.h ├── client │ ├── .gitignore │ ├── BladeClient.cpp │ ├── BladeClient.h │ ├── DataPointer.h │ ├── Makefile.am │ ├── RDMAClient.cpp │ ├── RDMAClient.h │ ├── TCPClient.cpp │ └── TCPClient.h ├── common │ ├── AllocationRecord.h │ ├── AllocatorMessage.h │ ├── AllocatorMessageGenerator.cpp │ ├── AllocatorMessageGenerator.h │ ├── Decls.h │ ├── Exception.h │ ├── FileAllocationRecord.h │ ├── Makefile.am │ ├── Serializer.h │ ├── Synchronization.cpp │ ├── Synchronization.h │ ├── ThreadPinning.h │ ├── config.h │ └── schemas │ │ ├── .gitignore │ │ ├── AllocatorMessage.fbs │ │ ├── BladeMessage.fbs │ │ ├── BladeObjectStoreMessage.fbs │ │ ├── Makefile.am │ │ └── TCPBladeMessage.fbs ├── coordination │ ├── LockManager.cpp │ └── LockManager.h ├── iterator │ ├── CirrusIterable.h │ ├── IteratorPolicy.h │ └── Makefile.am ├── memories │ ├── 3DXPoint.cpp │ ├── 3DXPoint.h │ └── Memory.h ├── object_store │ ├── EvictionPolicy.cpp │ ├── EvictionPolicy.h │ ├── FullBladeObjectStore.h │ ├── FullCacheStore.cpp │ ├── FullCacheStore.h │ ├── LRUEvictionPolicy.cpp │ ├── LRUEvictionPolicy.h │ ├── Makefile.am │ ├── ObjectStore.cpp │ ├── ObjectStore.h │ ├── RDMAObjectStore.cpp │ ├── RDMAObjectStore.h │ ├── RandomEvictionPolicy.cpp │ └── RandomEvictionPolicy.h ├── server │ ├── .gitignore │ ├── Allocation.h │ ├── ApplicationRecord.cpp │ ├── ApplicationRecord.h │ ├── BladeAllocServer.cpp │ ├── BladeAllocServer.h │ ├── BladeObjectStore.cpp │ ├── BladeObjectStore.h │ ├── BladePoolServer.cpp │ ├── BladePoolServer.h │ ├── ConnectionContext.cpp │ ├── ConnectionContext.h │ ├── ConnectionInfo.h │ ├── GeneralContext.h │ ├── Makefile.am │ ├── MemSlice.h │ ├── MemoryBackend.cpp │ ├── MemoryBackend.h │ ├── NVStorageBackend.cpp │ ├── NVStorageBackend.h │ ├── QuotaManager.h │ ├── RDMAServer.cpp │ ├── RDMAServer.h │ ├── ResourceAllocator.cpp │ ├── ResourceAllocator.h │ ├── Server.h │ ├── StorageBackend.h │ ├── TCPServer.cpp │ ├── TCPServer.h │ ├── TCPServerMain.cpp │ ├── allocmain.cpp │ ├── bladeallocmain.cpp │ ├── bladepoolmain.cpp │ ├── controllermain.cpp │ └── objectstoremain.cpp └── utils │ ├── CirrusTime.cpp │ ├── CirrusTime.h │ ├── InfinibandSupport.cpp │ ├── InfinibandSupport.h │ ├── Log.h │ ├── Makefile.am │ ├── Stats.cpp │ ├── Stats.h │ ├── StringUtils.h │ ├── logging.h │ └── utils.h ├── tests ├── .gitignore ├── Makefile.am ├── client │ ├── .gitignore │ ├── Makefile.am │ ├── RDMAClientmain.cpp │ └── TCPClientMain.cpp ├── mpi │ ├── .gitignore │ ├── Makefile.am │ ├── configure.ac │ ├── configure.gnu │ ├── run.sh │ └── test_mpi.cpp ├── object_store │ ├── .gitignore │ ├── Makefile.am │ ├── mem_exhaustion.cpp │ ├── object_store_internal.h │ ├── test_cache_manager.cpp │ ├── test_fullblade_store.cpp │ ├── test_iterator.cpp │ ├── test_mt.cpp │ ├── test_mult_clients.cpp │ ├── test_shards.cpp │ ├── test_store_bulk.cpp │ └── test_store_v2.cpp ├── raw │ └── test_atomics.cpp ├── test_bulk_transfer_TCP.py ├── test_cache_manager_RDMA.py ├── test_cache_manager_TCP.py ├── test_client_RDMA.py ├── test_client_TCP.py ├── test_iterator_RDMA.py ├── test_iterator_TCP.py ├── test_mem_exhaustion_RDMA.py ├── test_mem_exhaustion_TCP.py ├── test_mt_RDMA.py ├── test_mt_TCP.py ├── test_mult_clients_RDMA.py ├── test_mult_clients_TCP.py ├── test_runner.py ├── test_store_RDMA.py ├── test_store_TCP.py ├── test_store_simple_RDMA.py └── test_store_simple_TCP.py └── third_party ├── easylogging++.h └── get_eigen.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.log 3 | *.status 4 | *.in 5 | *.a 6 | *.sty 7 | *.scan 8 | core 9 | .deps 10 | *.out 11 | *.txt 12 | config.* 13 | Makefile 14 | *.tgz 15 | tags 16 | configure 17 | aclocal.m4 18 | autom4te.cache/ 19 | compile 20 | depcomp 21 | install-sh 22 | ltmain.sh 23 | missing 24 | docs/html/ 25 | *.pyc 26 | __pycache__/ 27 | *.dSYM/ 28 | *.gch 29 | third_party/eigen_source/* 30 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party/sparsehash"] 2 | path = third_party/sparsehash 3 | url = https://github.com/sparsehash/sparsehash.git 4 | [submodule "third_party/libcuckoo"] 5 | path = third_party/libcuckoo 6 | url = https://github.com/efficient/libcuckoo.git 7 | [submodule "third_party/flatbuffers"] 8 | path = third_party/flatbuffers 9 | url = https://github.com/google/flatbuffers.git 10 | [submodule "third_party/rocksdb"] 11 | path = third_party/rocksdb 12 | url = https://github.com/facebook/rocksdb.git 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: generic 4 | 5 | matrix: 6 | include: 7 | - os: linux 8 | dist: trusty 9 | 10 | env: 11 | global: 12 | # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created 13 | # via the "travis encrypt" command using the project repo's public key 14 | - secure: "TP4WteOFsNsV0hiCjKHlWsbOKl/tXIJla9w4Zd9x6x9JxHSSdT2orVA8KU2LlOjBY9em+Zb0+8PkCs/2sKG3TH10bZPkgEaxq6EXYWCxAlqALATtLJnuTJvVZgCAdHiraGgSiniHG0mjQS9NTqT8nwT4x60zBwPWiWkyjbpZXseBC+VJi3LqNRC1TFx/vet7hn8jEDVDM1UJW4qMnkVq+d2MXLicNHXUkvTwzwzscsKTagg3uDHjyOZ3NwylmW88BY2nKjZXuCTAokdbtszFgqafsucrXVo/YtAETgFp+gD3+06NLgEqyHoXbl64PE4orfvQ22JC0YqIK1xO+jpsra7XMTxziZGVoN8hyXjiU3siKSKkkOa2lEPa7TCX25IWPzxhoHG1CA2It6+YX0fmJ9AcJtHCaSmWM2XJ3qHPSM2n5Km0S84CyTBlyACLv6Bx4myC8f0eLKFKH+Q8PwLvvL9qXY+6q9due24jDaULZEFRF/OnHLKluH+mqupyV24mh2CIqpIPNqUUPXDkvqKVRTehM0N4b/MVT7ylmZK06S6WeUP8qd6k+44D1ULEpVKi1RR6Lw545zpk8z7afQ5/sbIKkNorc066ciP5c6TjmPMlRdciNak39Jgk2k85iQhMXZv4soHWJ0HHCdDyD1OrT6FTFTYeIE5TZzbvzlx4v7o=" 15 | 16 | before_install: 17 | - echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- 18 | 19 | addons: 20 | coverity_scan: 21 | project: 22 | name: "jcarreira/ddc" 23 | description: "Build submitted via Travis CI" 24 | notification_email: joao@berkeley.edu 25 | build_command_prepend: "./bootstrap.sh; ./configure; make clean" 26 | build_command: "make -j 8" 27 | branch_pattern: coverity_scan 28 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | filter=-legal/copyright,-build/c++11 2 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = foreign serial-tests 2 | SUBDIRS = src tests benchmarks examples 3 | 4 | TESTS = ./tests/test_client_TCP.py ./tests/test_mem_exhaustion_TCP.py \ 5 | ./tests/test_store_simple_TCP.py ./tests/test_cache_manager_TCP.py \ 6 | ./tests/test_iterator_TCP.py ./tests/test_store_TCP.py \ 7 | ./tests/test_mt_TCP.py ./tests/test_mult_clients_TCP.py \ 8 | ./tests/test_bulk_transfer_TCP.py 9 | 10 | if USE_RDMA 11 | TESTS += ./tests/test_client_RDMA.py ./tests/test_mem_exhaustion_RDMA.py \ 12 | ./tests/test_store_simple_RDMA.py ./tests/test_cache_manager_RDMA.py \ 13 | ./tests/test_iterator_RDMA.py ./tests/test_store_RDMA.py \ 14 | ./tests/test_mt_RDMA.py ./tests/test_mult_clients_RDMA.py 15 | endif 16 | 17 | 18 | source_dir = src/ 19 | DEFAULT_INCLUDES = -I$(source_dir) 20 | 21 | .PHONY: docs 22 | 23 | check_style: 24 | find . -path ./third_party -prune -o -path ./src/common/schemas -prune -o -name "*.h" -o -name "*.cpp" | xargs cpplint 25 | 26 | benchmark: all 27 | ./benchmarks/run_benchmarks.py 28 | 29 | docs: 30 | doxygen ./docs/Doxyfile 31 | -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | 1_1 2 | 1_2 3 | 1_3 4 | cache_benchmark 5 | iterator_benchmark 6 | outstanding_requests 7 | throughput 8 | -------------------------------------------------------------------------------- /benchmarks/1_2.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "object_store/FullBladeObjectStore.h" 17 | #include "tests/object_store/object_store_internal.h" 18 | #include "utils/CirrusTime.h" 19 | #include "utils/Stats.h" 20 | #include "client/TCPClient.h" 21 | 22 | // TODO(Tyler): Remove hardcoded IP and PORT 23 | static const uint64_t GB = (1024*1024*1024); 24 | const char PORT[] = "12345"; 25 | const char IP[] = "127.0.0.1"; 26 | static const uint32_t SIZE = 128; 27 | 28 | /** 29 | * This benchmark measures the distribution of times necessary for 30 | * N asynchronous puts. 31 | */ 32 | void test_async(int N) { 33 | cirrus::TCPClient client; 34 | cirrus::serializer_simple> serializer; 35 | cirrus::ostore::FullBladeObjectStoreTempl> 36 | store(IP, PORT, &client, 37 | serializer, 38 | cirrus::deserializer_simple, 39 | sizeof(cirrus::Dummy)>); 40 | cirrus::Stats stats; 41 | 42 | struct cirrus::Dummy d(42); 43 | cirrus::TimerFunction tfs[N]; 44 | 45 | cirrus::ObjectStore>::ObjectStorePutFuture futures[N]; 46 | std::cout << "Warming up." << std::endl; 47 | // warm up 48 | for (int i = 0; i < N; ++i) { 49 | store.put(i, d); 50 | } 51 | 52 | std::cout << "Warm up done" << std::endl; 53 | 54 | bool done[N]; 55 | std::memset(done, 0, sizeof(done)); 56 | int total_done = 0; 57 | 58 | for (int i = 0; i < N; ++i) { 59 | tfs[i].reset(); 60 | futures[i] = store.put_async(i, d); 61 | } 62 | std::cout << "N puts started." << std::endl; 63 | while (total_done != N) { 64 | for (int i = 0; i < N; ++i) { 65 | if (!done[i]) { 66 | bool ret = futures[i].try_wait(); 67 | if (ret) { 68 | done[i] = true; 69 | total_done++; 70 | auto elapsed = tfs[i].getUsElapsed(); 71 | stats.add(elapsed); 72 | } 73 | } 74 | } 75 | } 76 | std::ofstream outfile; 77 | outfile.open("1_2.log"); 78 | 79 | outfile << "count: " << stats.getCount() << std::endl; 80 | outfile << "min: " << stats.min() << std::endl; 81 | outfile << "avg: " << stats.avg() << std::endl; 82 | outfile << "max: " << stats.max() << std::endl; 83 | outfile << "sd: " << stats.sd() << std::endl; 84 | outfile << "99%: " << stats.getPercentile(0.99) << std::endl; 85 | outfile.close(); 86 | 87 | std::cout << "count: " << stats.getCount() << std::endl; 88 | std::cout << "min: " << stats.min() << std::endl; 89 | std::cout << "avg: " << stats.avg() << std::endl; 90 | std::cout << "max: " << stats.max() << std::endl; 91 | std::cout << "sd: " << stats.sd() << std::endl; 92 | std::cout << "99%: " << stats.getPercentile(0.99) << std::endl; 93 | } 94 | 95 | auto main() -> int { 96 | std::cout << "Benchmark 1_2 started." << std::endl; 97 | // test burst of 100 async writes 98 | test_async(100); 99 | 100 | return 0; 101 | } 102 | -------------------------------------------------------------------------------- /benchmarks/1_3.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "object_store/FullBladeObjectStore.h" 16 | #include "tests/object_store/object_store_internal.h" 17 | #include "utils/CirrusTime.h" 18 | #include "client/TCPClient.h" 19 | 20 | // TODO(Tyler): Remove hardcoded IP and PORT 21 | static const uint64_t MILLION = 1000000; 22 | static const int N_THREADS = 10; 23 | const char PORT[] = "12345"; 24 | const char IP[] = "127.0.0.1"; 25 | static const uint32_t SIZE = 128; 26 | static const uint64_t N_MSG = 100; 27 | 28 | uint64_t total_puts = 0; 29 | uint64_t total_time = 0; 30 | 31 | std::atomic count; 32 | 33 | void print_results(std::ostream& out) { 34 | out << "Total time: " << total_time << std::endl; 35 | out << "Average (us) per msg: " 36 | << total_time / N_THREADS / N_MSG << std::endl; 37 | out << "MSG/s: " 38 | << N_MSG * N_THREADS / (total_time * 1.0 / MILLION) << std::endl; 39 | } 40 | 41 | /** 42 | * Test the performance of the system when multiple clients 43 | * are connected to the same store. 44 | * Create ten new threads, 45 | * each of which connects to the store and sends N_MSG puts spread across 46 | * 100 different object ids 47 | */ 48 | void test_multiple_clients() { 49 | std::thread* threads[N_THREADS]; 50 | 51 | for (int i = 0; i < N_THREADS; ++i) { 52 | threads[i] = new std::thread([]() { 53 | cirrus::TCPClient client; 54 | cirrus::serializer_simple> serializer; 55 | cirrus::ostore::FullBladeObjectStoreTempl> 56 | store(IP, PORT, &client, 57 | serializer, 58 | cirrus::deserializer_simple, 59 | sizeof(cirrus::Dummy)>); 60 | std::cout << "New thread connected." << std::endl; 61 | struct cirrus::Dummy d(42); 62 | // warm up 63 | for (uint64_t i = 0; i < 100; ++i) { 64 | store.put(i, d); 65 | } 66 | 67 | // barrier 68 | count++; 69 | while (count != N_THREADS) {} 70 | 71 | cirrus::TimerFunction tf; 72 | for (uint64_t i = 0; i < N_MSG; ++i) { 73 | store.put(i % 100, d); 74 | } 75 | 76 | total_time += tf.getUsElapsed(); 77 | }); 78 | } 79 | 80 | for (int i = 0; i < N_THREADS; ++i) 81 | threads[i]->join(); 82 | 83 | print_results(std::cout); 84 | 85 | std::ofstream outfile; 86 | outfile.open("1_3.log"); 87 | print_results(outfile); 88 | } 89 | 90 | auto main() -> int { 91 | test_multiple_clients(); 92 | 93 | return 0; 94 | } 95 | -------------------------------------------------------------------------------- /benchmarks/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | bin_PROGRAMS = 1_1 1_3 throughput iterator_benchmark \ 6 | cache_benchmark 1_2 outstanding_requests store_bandwidth \ 7 | cache_latency cache_iops iterator_v2 remove_bulk 8 | 9 | LIBS = -lclient -lauthentication -lutils -lcommon -levictionpolicies \ 10 | $(LIBRDMACM) $(LIBIBVERBS) 11 | 12 | LINCLUDES = -L$(top_srcdir)/src/utils/ \ 13 | -L$(top_srcdir)/src/authentication \ 14 | -L$(top_srcdir)/src/common \ 15 | -L$(top_srcdir)/src/cache_manager \ 16 | -L$(top_srcdir)/src/client/ 17 | 18 | LDFLAGS = -pthread 19 | LDADD = $(LINCLUDES) $(LIBS) 20 | 21 | CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/examples/sparsehash/src/ \ 22 | -isystem $(top_srcdir)/third_party/libcuckoo/ \ 23 | -I$(top_srcdir)/src \ 24 | -I$(top_srcdir)/third_party/flatbuffers/include 25 | 26 | 1_1_SOURCES = 1_1.cpp 27 | 28 | 1_2_SOURCES = 1_2.cpp 29 | 30 | 1_3_SOURCES = 1_3.cpp 31 | 32 | throughput_SOURCES = throughput.cpp 33 | 34 | iterator_benchmark_SOURCES = iterator_benchmark.cpp 35 | 36 | iterator_v2_SOURCES = iterator_v2.cpp 37 | 38 | cache_benchmark_SOURCES = cache_benchmark.cpp 39 | 40 | cache_latency_SOURCES = cache_latency.cpp 41 | 42 | cache_iops_SOURCES = cache_iops.cpp 43 | 44 | store_bandwidth_SOURCES = store_bandwidth.cpp 45 | 46 | outstanding_requests_SOURCES = outstanding_requests.cpp 47 | 48 | remove_bulk_SOURCES = remove_bulk.cpp 49 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | Benchmark Results 2 | ================================== 3 | 4 | ~~~~ 5 | ./benchmarks/throughput 6 | 7 | get cache throughput 128 test 8 | msg/s: 10298661.174047 9 | MB/s: 1257.160788 10 | get store throughput 128 test 11 | msg/s: 2644.581939 12 | MB/s: 0.322825 13 | get cache throughput 4096 test 14 | msg/s: 2653927.813163 15 | MB/s: 10366.905520 16 | get store throughput 4096 test 17 | msg/s: 13300.348469 18 | MB/s: 51.954486 19 | get cache throughput 51200 test 20 | msg/s: 403771.223225 21 | MB/s: 19715.391759 22 | get store throughput 51200 test 23 | msg/s: 8822.134736 24 | MB/s: 430.768298 25 | put cache throughput 1048576 test 26 | msg/s: 907.420431 27 | MB/s: 907.420431 28 | put store throughput 1048576 test 29 | msg/s: 1139.105256 30 | MB/s: 1139.105256 31 | put cache throughput 128 test 32 | msg/s: 2996.165358 33 | MB/s: 0.365743 34 | put store throughput 128 test 35 | msg/s: 3558.122464 36 | MB/s: 0.434341 37 | put cache throughput 4096 test 38 | msg/s: 2981.278169 39 | MB/s: 11.645618 40 | put store throughput 4096 test 41 | msg/s: 15045.546631 42 | MB/s: 58.771667 43 | put cache throughput 51200 test 44 | msg/s: 2924.430665 45 | MB/s: 142.794466 46 | put store throughput 51200 test 47 | msg/s: 9354.186934 48 | MB/s: 456.747409 49 | ~~~~ 50 | -------------------------------------------------------------------------------- /benchmarks/cache_latency.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "object_store/FullBladeObjectStore.h" 17 | #include "tests/object_store/object_store_internal.h" 18 | #include "utils/CirrusTime.h" 19 | #include "utils/Stats.h" 20 | #include "client/TCPClient.h" 21 | 22 | #include "cache_manager/PrefetchPolicy.h" 23 | #include "cache_manager/LRAddedEvictionPolicy.h" 24 | #include "cache_manager/CacheManager.h" 25 | 26 | // TODO(Tyler): Remove hardcoded IP and PORT 27 | static const uint64_t GB = (1024*1024*1024); 28 | const char PORT[] = "12345"; 29 | const char IP[] = "127.0.0.1"; 30 | static const uint64_t MILLION = 1000000; 31 | static const uint64_t N_ITER = MILLION; 32 | 33 | void print_stats(std::ostream& out, const cirrus::Stats& stats, 34 | uint64_t size) { 35 | out << "Cache " << size << " byte benchmark" << std::endl; 36 | out << "count: " << stats.getCount() << std::endl; 37 | out << "min: " << stats.min() << " µs" << std::endl; 38 | out << "avg: " << stats.avg() << " µs" << std::endl; 39 | out << "max: " << stats.max() << " µs"<< std::endl; 40 | out << "sd: " << stats.sd() << " µs" << std::endl; 41 | out << "99%: " << stats.getPercentile(0.99) << " µs" << std::endl; 42 | out << std::endl; 43 | } 44 | 45 | /** 46 | * This benchmark has aims to find the latency when getting items from a 47 | * cache of one million objects. 48 | */ 49 | template 50 | void test_get_latency(std::ofstream& outfile) { 51 | cirrus::TCPClient client; 52 | cirrus::serializer_simple> serializer; 53 | cirrus::ostore::FullBladeObjectStoreTempl> 54 | store(IP, PORT, &client, 55 | serializer, 56 | cirrus::deserializer_simple, 57 | sizeof(cirrus::Dummy)>); 58 | 59 | struct cirrus::Dummy d(42); 60 | 61 | // warm up 62 | std::cout << "Warming up" << std::endl; 63 | for (uint64_t i = 0; i < N_ITER; ++i) { 64 | store.put(i, d); 65 | } 66 | 67 | std::cout << "Warm up done" << std::endl; 68 | 69 | 70 | cirrus::LRAddedEvictionPolicy policy(N_ITER); 71 | cirrus::CacheManager> cm(&store, &policy, N_ITER); 72 | 73 | // populate the cache. We may want to change this to prefetch 74 | for (uint64_t i = 0; i < N_ITER; i++) { 75 | cm.get(i); 76 | } 77 | 78 | cirrus::Stats stats; 79 | stats.reserve(N_ITER); 80 | 81 | std::cout << "Measuring latencies.." << std::endl; 82 | for (uint64_t i = 0; i < N_ITER; ++i) { 83 | cirrus::TimerFunction tf; 84 | cm.get(i); 85 | uint64_t elapsed_us = tf.getUsElapsed(); 86 | stats.add(elapsed_us); 87 | } 88 | 89 | print_stats(outfile, stats, SIZE); 90 | 91 | print_stats(std::cout, stats, SIZE); 92 | } 93 | 94 | auto main() -> int { 95 | // test burst of sync writes 96 | std::ofstream outfile; 97 | outfile.open("cache_latency.log"); 98 | test_get_latency<32>(outfile); 99 | test_get_latency<1024>(outfile); 100 | test_get_latency<4 * 1024>(outfile); 101 | // test_get_latency<32 * 1024>(outfile); 102 | // test_get_latency<1024 * 1024>(outfile); 103 | outfile.close(); 104 | 105 | return 0; 106 | } 107 | -------------------------------------------------------------------------------- /benchmarks/remove_bulk.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "object_store/FullBladeObjectStore.h" 8 | #include "tests/object_store/object_store_internal.h" 9 | #include "utils/CirrusTime.h" 10 | #include "utils/Stats.h" 11 | #include "client/TCPClient.h" 12 | 13 | // TODO(Tyler): Remove hardcoded IP and PORT 14 | 15 | static const uint64_t KB = 1024; 16 | static const uint64_t MB = 1024 * KB; 17 | const char PORT[] = "12345"; 18 | const char IP[] = "10.10.49.84"; 19 | 20 | 21 | /** 22 | * This benchmark is used to determine the latency of 23 | * removing a set of objects 24 | */ 25 | template 26 | void remove_bulk_benchmark(uint64_t num_objs) { 27 | cirrus::TCPClient client; 28 | cirrus::serializer_simple> serializer; 29 | cirrus::ostore::FullBladeObjectStoreTempl> 30 | store(IP, PORT, &client, 31 | serializer, 32 | cirrus::deserializer_simple, 33 | sizeof(cirrus::Dummy)>); 34 | 35 | struct cirrus::Dummy d(42); 36 | 37 | // warm up 38 | std::cout << "Inserting " << num_objs << " objects" << std::endl; 39 | for (uint64_t i = 0; i < num_objs; i++) { 40 | store.put(i, d); 41 | } 42 | 43 | cirrus::TimerFunction tf; 44 | store.removeBulk(0, num_objs - 1); 45 | uint64_t elapsed_us = tf.getUsElapsed(); 46 | 47 | std::cout << "Remove bulk took: " << elapsed_us << "us" << std::endl; 48 | } 49 | 50 | 51 | auto main() -> int { 52 | remove_bulk_benchmark<100 * KB>(5000); 53 | 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /benchmarks/run_benchmarks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | 7 | # NOTE: all pathnames start from the top directory where make benchmark is run 8 | 9 | benchmarks = [["./benchmarks/1_3"], ["./benchmarks/1_1"], 10 | ["./benchmarks/iterator_benchmark"], ["./benchmarks/cache_benchmark"], 11 | ["./benchmarks/1_2"], ["./benchmarks/outstanding_requests"], 12 | ["./benchmarks/throughput"], ["./benchmarks/remove_bulk"]] 13 | 14 | server_name = ["./src/server/tcpservermain"] 15 | 16 | for benchmark in benchmarks: 17 | 18 | # Launch the server in the background 19 | print("Starting server.") 20 | server = subprocess.Popen(server_name) 21 | 22 | # Sleep to give server time to start 23 | print("Started server, sleeping.") 24 | time.sleep(2) 25 | print("Sleep finished, launching client.") 26 | 27 | sys.stdout.write("Benchmark " + benchmark[0] + "...") 28 | child = subprocess.Popen(benchmark, stdout=subprocess.PIPE) 29 | 30 | # Uncomment below two lines to print the benchmark's output 31 | 32 | # for line in child.stdout: 33 | # print(line.decode(), end='') 34 | 35 | streamdata = child.communicate()[0] 36 | rc = child.returncode 37 | server.kill() 38 | 39 | #give server time to die 40 | time.sleep(1) 41 | if (rc != 0): 42 | print(" Failed") 43 | sys.exit(rc) 44 | else: 45 | print(" Success") 46 | -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | # get third party libs 2 | git submodule init 3 | git submodule update 4 | 5 | # compile libcuckoo 6 | cd third_party/libcuckoo 7 | cmake . 8 | make -j 10 9 | 10 | # compile sparsehash 11 | cd ../sparsehash 12 | ./configure 13 | make -j 10 14 | 15 | # compile flatbuffers 16 | cd ../flatbuffers 17 | cmake -G "Unix Makefiles" 18 | make -j 10 19 | 20 | # rocksdb 21 | cd ../rocksdb 22 | make -j 10 static_lib 23 | 24 | # download eigen 25 | cd ../ 26 | if [ ! -d "eigen_source" ]; then 27 | sh get_eigen.sh 28 | fi 29 | cd .. 30 | 31 | # download test file for benchmark 32 | cd ./benchmarks 33 | wget http://norvig.com/big.txt 34 | for i in {1..16};do cat big.txt >> very_big.txt; done 35 | rm ./big.txt 36 | cd .. 37 | 38 | # main compilation 39 | touch config.rpath 40 | autoreconf 41 | automake --add-missing 42 | autoreconf 43 | ./configure 44 | -------------------------------------------------------------------------------- /common.mk: -------------------------------------------------------------------------------- 1 | AM_CXXFLAGS = -Wall -Wextra -O3 2 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT(CIRRUS, 0.1, joao@berkeley.edu) 6 | AM_INIT_AUTOMAKE 7 | AC_CONFIG_SRCDIR([src/server/RDMAServer.h]) 8 | #AC_CONFIG_HEADERS([config.h]) 9 | 10 | # Checks for presence of dependencies 11 | AC_CHECK_PROG(CPPLINT_CHECK, cpplint, yes) 12 | if test x"$CPPLINT_CHECK" != x"yes" ; then 13 | AC_MSG_ERROR([Please install cpplint with 'sudo pip install cpplint' before installing.]) 14 | fi 15 | 16 | # Checks for programs. 17 | AC_PROG_CXX([g++-6]) 18 | AC_PROG_AWK 19 | AC_PROG_CC 20 | AC_PROG_CPP 21 | AC_PROG_INSTALL 22 | AC_PROG_LN_S 23 | AC_PROG_MAKE_SET 24 | AC_PROG_RANLIB 25 | AX_CXX_COMPILE_STDCXX_14 26 | 27 | CXXFLAGS="-O3 -g -fPIC -std=c++1z -Werror" 28 | 29 | # Check for libraries 30 | AC_LIB_HAVE_LINKFLAGS(rdmacm) 31 | AC_LIB_HAVE_LINKFLAGS(ibverbs) 32 | AC_LIB_HAVE_LINKFLAGS(mpi) 33 | 34 | AM_CONDITIONAL([USE_RDMA], [test "x${HAVE_LIBRDMACM}" = "xyes"]) 35 | AM_CONDITIONAL([USE_MPI], [test "x${HAVE_LIBMPI}" = "xyes"]) 36 | 37 | # Checks for header files. 38 | AC_CHECK_HEADERS([ arpa/inet.h netdb.h string.h stdint.h\ 39 | stdlib.h sys/types.h sys/socket.h sys/epoll.h\ 40 | sys/time.h syslog.h unistd.h google/dense_hash_map]) 41 | 42 | # Checks for typedefs, structures, and compiler characteristics. 43 | AC_CHECK_HEADER_STDBOOL 44 | AC_C_INLINE 45 | AC_TYPE_SIZE_T 46 | AC_TYPE_UINT32_T 47 | AC_TYPE_UINT64_T 48 | 49 | # Checks for library functions. 50 | AC_FUNC_MALLOC 51 | AC_FUNC_STRCOLL 52 | AC_CHECK_FUNCS([gettimeofday localtime_r memchr memmove memset mkdir stpcpy strchr strcspn strdup strerror strpbrk strrchr strspn strstr]) 53 | AC_CONFIG_FILES([Makefile 54 | src/Makefile 55 | src/authentication/Makefile 56 | src/utils/Makefile 57 | src/common/Makefile 58 | src/object_store/Makefile 59 | src/iterator/Makefile 60 | src/client/Makefile 61 | src/server/Makefile 62 | src/cache_manager/Makefile 63 | src/common/schemas/Makefile 64 | tests/object_store/Makefile 65 | tests/client/Makefile 66 | tests/Makefile 67 | examples/Makefile 68 | examples/graphs/Makefile 69 | examples/graphs/page_rank/Makefile 70 | examples/graphs/wcc/Makefile 71 | benchmarks/Makefile]) 72 | 73 | 74 | AC_CONFIG_SUBDIRS([tests/mpi]) 75 | AC_CONFIG_SUBDIRS([examples/ml]) 76 | 77 | AC_OUTPUT 78 | 79 | #examples/ml/Makefile 80 | -------------------------------------------------------------------------------- /docs/addtl_documentation.dox: -------------------------------------------------------------------------------- 1 | /** @file */ 2 | /** @file AllocatorMessage.fbs 3 | * A schema for AllocatorMessages. 4 | * Parsed by flatc to generate header files then used for sending messages. 5 | */ 6 | 7 | /** @file BladeMessage.fbs 8 | * A schema for BladeMessages. 9 | * Parsed by flatc to generate header files then used for sending messages. 10 | */ 11 | 12 | /** @file BladeObjectStoreMessage.fbs 13 | * A schema for BladeObjectStoreMessages. 14 | * Parsed by flatc to generate header files then used for sending messages. 15 | */ 16 | 17 | /** \namespace cirrus 18 | * The cirrus project 19 | */ 20 | 21 | /** \namespace cirrus::ostore 22 | * Object stores 23 | */ 24 | -------------------------------------------------------------------------------- /docs/mainpage.dox: -------------------------------------------------------------------------------- 1 | /** @mainpage Cirrus Documentation 2 | * @section intro_sec Introduction 3 | * This is the documentation for the cirrus project. 4 | * 5 | */ 6 | -------------------------------------------------------------------------------- /examples/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | SUBDIRS = graphs 5 | 6 | if USE_MPI 7 | SUBDIRS += ml 8 | endif 9 | -------------------------------------------------------------------------------- /examples/graphs/.gitignore: -------------------------------------------------------------------------------- 1 | pr 2 | -------------------------------------------------------------------------------- /examples/graphs/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | SUBDIRS = page_rank wcc 5 | 6 | -------------------------------------------------------------------------------- /examples/graphs/data/graph_1: -------------------------------------------------------------------------------- 1 | 4 2 | 4 3 | 1 4 | 1 5 | 1 6 | 2 7 | 1 8 | 3 9 | 1 10 | 0 11 | -------------------------------------------------------------------------------- /examples/graphs/data/graph_2: -------------------------------------------------------------------------------- 1 | 4 2 | 5 3 | 2 4 | 1 5 | 2 6 | 1 7 | 2 8 | 1 9 | 0 10 | 1 11 | 2 12 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/.gitignore: -------------------------------------------------------------------------------- 1 | pr 2 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/Input.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace graphs { 5 | 6 | /** Graphs must be in the format: 7 | number of vertices 8 | number of edges 9 | number of edges per vertex 10 | edge for vertex 0 11 | ... 12 | edge for vertex n - 1 13 | **/ 14 | std::vector readGraph(const std::string& fname) { 15 | std::ifstream infile(fname.c_str()); 16 | std::vector vertices; 17 | 18 | if (!infile.is_open()) { 19 | throw std::runtime_error("Error opening file: " + fname); 20 | } 21 | 22 | int num_vertices = -1; 23 | int num_edges = -1; 24 | infile >> num_vertices; 25 | infile >> num_edges; 26 | 27 | vertices.reserve(num_vertices); 28 | 29 | for (int j = 0; j < num_vertices; ++j) { 30 | int num_edges_per_vertex = -1; 31 | infile >> num_edges_per_vertex; 32 | 33 | Vertex v(j); // new vertex 34 | 35 | // read neighbors for new vertex 36 | for (int i = 0; i < num_edges_per_vertex; ++i) { 37 | int neigh_id; 38 | infile >> neigh_id; 39 | v.addNeighbor(neigh_id); 40 | } 41 | vertices.push_back(v); 42 | } 43 | return vertices; 44 | } 45 | 46 | } // namespace graphs 47 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/Input.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_PAGE_RANK_INPUT_H_ 2 | #define EXAMPLES_GRAPHS_PAGE_RANK_INPUT_H_ 3 | 4 | #include 5 | #include 6 | #include "Vertex.h" 7 | 8 | namespace graphs { 9 | 10 | std::vector readGraph(const std::string& fname); 11 | 12 | } // graphs 13 | 14 | #endif // EXAMPLES_GRAPHS_PAGE_RANK_INPUT_H_ 15 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | bin_PROGRAMS = pr 6 | 7 | LIBS = -lclient \ 8 | -lauthentication -lutils -lcommon -levictionpolicies \ 9 | $(LIBRDMACM) $(LIBIBVERBS) 10 | 11 | LINCLUDES = -L$(top_srcdir)/src/utils/ \ 12 | -L$(top_srcdir)/src/client/ \ 13 | -L$(top_srcdir)/src/authentication \ 14 | -L$(top_srcdir)/src/common \ 15 | -L$(top_srcdir)/src/cache_manager 16 | 17 | CPPFLAGS = -ggdb -O3 -I$(top_srcdir) -I$(top_srcdir)/examples/sparsehash/src/ \ 18 | -isystem $(top_srcdir)/third_party/libcuckoo/ \ 19 | -I$(top_srcdir)/src \ 20 | -I$(top_srcdir)/third_party/flatbuffers/include 21 | SOURCES = PageRank.cpp Input.cpp Vertex.cpp 22 | 23 | pr_SOURCES = pr.cpp $(SOURCES) 24 | pr_LDFLAGS = -pthread 25 | pr_LDADD = $(LINCLUDES) $(LIBS) 26 | 27 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/PageRank.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "cache_manager/CacheManager.h" 6 | #include "iterator/CirrusIterable.h" 7 | 8 | #define EPS (1e-05) 9 | #define EQUAL(a, b) (fabs(a - b) < EPS) 10 | 11 | namespace graphs { 12 | 13 | /** 14 | * Compute PageRank on graph stored in a Cirrus store 15 | * Preconditions: 16 | * a) vertices are stored in the range 0..num_vertices 17 | * b) Current probability of vertices is initialized to 1.0 and next to 0.0 18 | * @param cm CacheManager to be used to access the graph 19 | * @param num_vertices Number of vertices in the graph 20 | * @param gamma PageRanks's algorithm gamma 21 | * @param num_iterations Number of iterations 22 | */ 23 | void pageRank(cirrus::CacheManager& cm, 24 | unsigned int num_vertices, 25 | double gamma, uint64_t num_iterations) { 26 | double error = 0; 27 | 28 | for (uint64_t it = 0; it < num_iterations; ++it) { 29 | double prev_error = error; 30 | 31 | cirrus::CirrusIterable iter(&cm, 40, 0, num_vertices - 1); 32 | for (const auto& curr : iter) { 33 | auto neighbors = curr.getNeighbors(); 34 | for (auto it = neighbors.begin(); 35 | it != neighbors.end(); 36 | ++it) { 37 | Vertex n = cm.get(*it); 38 | 39 | n.setNextProb(n.getNextProb() + 40 | curr.getCurrProb() / neighbors.size()); 41 | cm.put(*it, n); 42 | } 43 | } 44 | 45 | // use iterator here 46 | iter = cirrus::CirrusIterable(&cm, 40, 0, num_vertices - 1); 47 | uint64_t i = 0; 48 | for (auto curr : iter) { 49 | prev_error = error; 50 | 51 | // update p_curr for next round 52 | curr.setCurrProb(curr.getNextProb() * gamma + (1.0 - gamma) / 53 | static_cast(num_vertices)); 54 | 55 | error += std::abs(curr.getCurrProb() - curr.getNextProb()); 56 | 57 | // set p_next to 0.0 58 | curr.setNextProb(0.0); 59 | cm.put(i++, curr); 60 | } 61 | 62 | if (EQUAL(error, prev_error)) { 63 | break; 64 | } 65 | } 66 | } 67 | 68 | } // namespace graphs 69 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/PageRank.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_PAGE_RANK_PAGERANK_H_ 2 | #define EXAMPLES_GRAPHS_PAGE_RANK_PAGERANK_H_ 3 | 4 | #include 5 | #include "Vertex.h" 6 | #include "cache_manager/CacheManager.h" 7 | 8 | namespace graphs { 9 | 10 | /** 11 | * Compute the PageRank probs. of a set of nodes stored in a Cirrus store 12 | * @param cm Cache manager used to access nodes 13 | * @param num_vertices Number of vertices (0..num_vertices - 1) 14 | * @param gamma Value of PageRank's gamma (e.g., 0.85) 15 | * @param num_iterations Number of pageRank iterations 16 | */ 17 | void pageRank(cirrus::CacheManager& cm, 18 | unsigned int num_vertices, 19 | double gamma, uint64_t num_iterations); 20 | 21 | } // namespace graphs 22 | 23 | #endif // EXAMPLES_GRAPHS_PAGE_RANK_PAGERANK_H_ 24 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/Vertex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace graphs { 5 | 6 | Vertex::Vertex(int id) : 7 | id(id), p_curr(1.0), p_next(0.0) { 8 | } 9 | 10 | Vertex::Vertex(int id, const std::vector& neighbors) : 11 | id(id), p_curr(1.0), p_next(0.0) { 12 | setNeighbors(neighbors); 13 | } 14 | 15 | void Vertex::setNeighbors(const std::vector& v) { 16 | for (const auto& i : v) { 17 | addNeighbor(i); 18 | } 19 | } 20 | 21 | void Vertex::addNeighbor(int id) { 22 | neighbors.insert(id); 23 | } 24 | 25 | std::set Vertex::getNeighbors() const { 26 | return neighbors; 27 | } 28 | 29 | uint64_t Vertex::getNeighborsSize() const { 30 | return neighbors.size(); 31 | } 32 | 33 | int Vertex::getId() const { 34 | return id; 35 | } 36 | 37 | void Vertex::setId(int i) { 38 | id = i; 39 | } 40 | 41 | bool Vertex::hasNeighbor(int id) const { 42 | return neighbors.find(id) != neighbors.end(); 43 | } 44 | 45 | Vertex Vertex::deserializer(const void* data, unsigned int size) { 46 | const double* double_ptr = reinterpret_cast(data); 47 | 48 | Vertex v; 49 | v.setCurrProb(*double_ptr++); 50 | v.setNextProb(*double_ptr++); 51 | 52 | std::cout << "deserialized with currProb: " << v.getCurrProb() 53 | << " nextProb: " << v.getNextProb() 54 | << std::endl; 55 | 56 | const uint32_t* ptr = reinterpret_cast(double_ptr); 57 | v.setId(ntohl(*ptr++)); 58 | uint32_t n = ntohl(*ptr++); 59 | 60 | uint32_t expected_size = 2 * sizeof(double) + sizeof(uint32_t) * (2 + n); 61 | 62 | if (size != expected_size) { 63 | throw std::runtime_error("Incorrect size: " 64 | + std::to_string(size) + 65 | + " expected: " + std::to_string(expected_size)); 66 | } 67 | 68 | for (uint32_t i = 0; i < n; ++i) { 69 | v.addNeighbor(ntohl(*ptr++)); 70 | } 71 | return v; 72 | } 73 | 74 | double Vertex::getCurrProb() const { 75 | return p_curr; 76 | } 77 | 78 | void Vertex::setCurrProb(double prob) { 79 | p_curr = prob; 80 | } 81 | 82 | double Vertex::getNextProb() const { 83 | return p_next; 84 | } 85 | 86 | void Vertex::setNextProb(double prob) { 87 | p_next = prob; 88 | } 89 | 90 | void Vertex::print() const { 91 | std::cout 92 | << "Print vertex id: " << id 93 | << " p_next: " << p_next 94 | << " p_curr: " << p_curr 95 | << std::endl; 96 | } 97 | 98 | } // namespace graphs 99 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/Vertex.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_PAGE_RANK_VERTEX_H_ 2 | #define EXAMPLES_GRAPHS_PAGE_RANK_VERTEX_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "common/Serializer.h" 12 | 13 | namespace graphs { 14 | 15 | class Vertex { 16 | public: 17 | explicit Vertex(int id = 0); 18 | Vertex(int id, const std::vector& neighbors); 19 | 20 | /** 21 | * Add and set neighbrs of this vertex 22 | */ 23 | void addNeighbor(int id); 24 | void setNeighbors(const std::vector& neighbors); 25 | 26 | /** 27 | * Get set of neighbors and neighbors size 28 | */ 29 | std::set getNeighbors() const; 30 | uint64_t getNeighborsSize() const; 31 | 32 | /** 33 | * Check if vertex is a neighbor 34 | */ 35 | bool hasNeighbor(int id) const; 36 | 37 | /** 38 | * get and set the id of this vertex 39 | */ 40 | int getId() const; 41 | void setId(int id); 42 | 43 | /** 44 | * Deserializer routines 45 | */ 46 | static Vertex deserializer(const void* data, unsigned int size); 47 | 48 | /** 49 | * Get and set current and next probabilities 50 | */ 51 | double getCurrProb() const; 52 | void setCurrProb(double prob); 53 | double getNextProb() const; 54 | void setNextProb(double prob); 55 | 56 | /** 57 | * Print some info about this vertex 58 | */ 59 | void print() const; 60 | 61 | private: 62 | std::set neighbors; //< set of neighboring vetrices 63 | int id; //< id of the vertex 64 | double p_curr; //< current probability of this vertex 65 | double p_next; //< probability of this vertex in the next round 66 | }; 67 | 68 | /** Format: 69 | * p_curr (double) 70 | * p_next (double) 71 | * id (uint32_t) 72 | * n neighbors (uint32_t) 73 | * neighbors list (n * uint32_t) 74 | */ 75 | class VertexSerializer : public cirrus::Serializer { 76 | public: 77 | uint64_t size(const Vertex& v) const override { 78 | uint64_t size = sizeof(uint32_t) * 2 + 79 | sizeof(double) * 2 + 80 | sizeof(uint32_t) * v.getNeighborsSize(); // neighbors 81 | return size; 82 | } 83 | void serialize(const Vertex& v, void* mem) const override { 84 | double* double_ptr = reinterpret_cast(mem); 85 | *double_ptr++ = v.getCurrProb(); 86 | *double_ptr++ = v.getNextProb(); 87 | 88 | uint32_t* ptr = reinterpret_cast(double_ptr); 89 | *ptr++ = htonl(v.getId()); 90 | *ptr++ = htonl(v.getNeighborsSize()); 91 | 92 | for (const auto& n : v.getNeighbors()) { 93 | *ptr++ = htonl(n); 94 | } 95 | } 96 | private: 97 | }; 98 | 99 | } // namespace graphs 100 | 101 | #endif // EXAMPLES_GRAPHS_PAGE_RANK_VERTEX_H_ 102 | -------------------------------------------------------------------------------- /examples/graphs/page_rank/pr.cpp: -------------------------------------------------------------------------------- 1 | #include "PageRank.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "Input.h" 8 | #include "Vertex.h" 9 | #include "utils/Log.h" 10 | #include "object_store/FullBladeObjectStore.h" 11 | #include "client/TCPClient.h" 12 | #include "cache_manager/CacheManager.h" 13 | #include "cache_manager/LRAddedEvictionPolicy.h" 14 | 15 | static const uint64_t GB = (1024*1024*1024); 16 | const char PORT[] = "12345"; 17 | const char IP[] = "127.0.0.1"; 18 | static const uint32_t SIZE = 128; 19 | static const uint64_t MILLION = 1000000; 20 | static const uint64_t N_ITER = 100; 21 | const int cache_size = 200; 22 | 23 | int main(int argc, char** argv) { 24 | if (argc != 2) { 25 | throw std::runtime_error("./pr "); 26 | } 27 | 28 | // read vertices from file 29 | std::vector vertices = graphs::readGraph(argv[1]); 30 | cirrus::LOG("Read ", vertices.size(), " vertices"); 31 | 32 | graphs::VertexSerializer vs; 33 | cirrus::TCPClient client; 34 | cirrus::ostore::FullBladeObjectStoreTempl 35 | vertex_store(IP, PORT, &client, 36 | vs, graphs::Vertex::deserializer); 37 | 38 | cirrus::LRAddedEvictionPolicy policy(cache_size); 39 | cirrus::CacheManager cm(&vertex_store, &policy, cache_size); 40 | 41 | std::cout << "Adding to cm" << std::endl; 42 | for (uint64_t i = 0; i < vertices.size(); i++) { 43 | std::cout << "Adding vertex: " << i << std::endl; 44 | cm.put(vertices[i].getId(), vertices[i]); 45 | } 46 | 47 | std::cout << "Running PageRank" << std::endl; 48 | graphs::pageRank(cm, vertices.size(), 0.85, 100); 49 | 50 | std::cout << "PageRank completed" << std::endl; 51 | 52 | std::cout << "PageRank probabilities size: " 53 | << vertices.size() << std::endl; 54 | for (unsigned int i = 0; i < vertices.size(); i++) { 55 | graphs::Vertex curr = cm.get(i); 56 | curr.print(); 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /examples/graphs/wcc/.gitignore: -------------------------------------------------------------------------------- 1 | main 2 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Input.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace graphs { 5 | 6 | /** Graphs must be in the format: 7 | number of vertices 8 | number of edges 9 | number of edges for vertex 0 (n) 10 | edge 0 for vertex 0 11 | edge 1 for vertex 0 12 | ... 13 | edge (n-1) for vertex 0 14 | number of edges for vertex 1 (m) 15 | ... 16 | **/ 17 | std::vector readGraph(const std::string& fname) { 18 | std::ifstream infile(fname.c_str()); 19 | std::vector vertices; 20 | 21 | if (!infile.is_open()) { 22 | throw std::runtime_error("Error opening file: " + fname); 23 | } 24 | 25 | int num_vertices = -1; 26 | int num_edges = -1; 27 | infile >> num_vertices; 28 | infile >> num_edges; 29 | 30 | if (num_vertices < 0 || num_edges < 0) { 31 | throw std::runtime_error("Wrong input values"); 32 | } 33 | 34 | vertices.reserve(num_vertices); 35 | 36 | for (int j = 0; j < num_vertices; ++j) { 37 | int num_edges_per_vertex = -1; 38 | infile >> num_edges_per_vertex; 39 | 40 | Vertex v(j); // new vertex 41 | 42 | // read neighbors for new vertex 43 | for (int i = 0; i < num_edges_per_vertex; ++i) { 44 | int neigh_id; 45 | infile >> neigh_id; 46 | v.addNeighbor(neigh_id); 47 | } 48 | vertices.push_back(v); 49 | } 50 | return vertices; 51 | } 52 | 53 | } // namespace graphs 54 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Input.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_WCC_INPUT_H_ 2 | #define EXAMPLES_GRAPHS_WCC_INPUT_H_ 3 | 4 | #include 5 | #include 6 | #include "Vertex.h" 7 | 8 | namespace graphs { 9 | 10 | std::vector readGraph(const std::string& fname); 11 | 12 | } // namespace graphs 13 | 14 | #endif // EXAMPLES_GRAPHS_WCC_INPUT_H_ 15 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | bin_PROGRAMS = main 6 | 7 | LIBS = -lclient -lauthentication -lutils -lcommon -levictionpolicies \ 8 | $(LIBRDMACM) $(LIBIBVERBS) 9 | 10 | LINCLUDES = -L$(top_srcdir)/src/utils/ \ 11 | -L$(top_srcdir)/src/client/ \ 12 | -L$(top_srcdir)/src/authentication \ 13 | -L$(top_srcdir)/src/common \ 14 | -L$(top_srcdir)/src/cache_manager 15 | 16 | CPPFLAGS = -ggdb -O3 -I$(top_srcdir) -I$(top_srcdir)/examples/sparsehash/src/ \ 17 | -isystem $(top_srcdir)/third_party/libcuckoo/ \ 18 | -I$(top_srcdir)/src \ 19 | -I$(top_srcdir)/third_party/flatbuffers/include 20 | SOURCES = Wcc.cpp Input.cpp Vertex.cpp 21 | 22 | main_SOURCES = main.cpp $(SOURCES) 23 | main_LDFLAGS = -pthread 24 | main_LDADD = $(LINCLUDES) $(LIBS) 25 | 26 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Vertex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace graphs { 5 | 6 | Vertex::Vertex(int id) : 7 | seen(false), cc(-1), id(id) { 8 | } 9 | 10 | Vertex::Vertex(int id, const std::vector& neighbors) : 11 | seen(false), cc(-1), id(id) { 12 | setNeighbors(neighbors); 13 | } 14 | 15 | void Vertex::setNeighbors(const std::vector& v) { 16 | for (const auto& i : v) { 17 | addNeighbor(i); 18 | } 19 | } 20 | 21 | void Vertex::addNeighbor(int id) { 22 | neighbors.insert(id); 23 | } 24 | 25 | std::set Vertex::getNeighbors() const { 26 | return neighbors; 27 | } 28 | 29 | uint64_t Vertex::getNeighborsSize() const { 30 | return neighbors.size(); 31 | } 32 | 33 | int Vertex::getId() const { 34 | return id; 35 | } 36 | 37 | void Vertex::setId(int i) { 38 | id = i; 39 | } 40 | 41 | bool Vertex::hasNeighbor(int id) const { 42 | return neighbors.find(id) != neighbors.end(); 43 | } 44 | 45 | Vertex Vertex::deserializer(const void* data, unsigned int size) { 46 | Vertex v; 47 | 48 | const uint32_t* ptr = reinterpret_cast(data); 49 | v.setId(ntohl(*ptr++)); 50 | uint32_t n = ntohl(*ptr++); 51 | 52 | uint32_t expected_size = sizeof(uint32_t) * (2 + n); 53 | 54 | if (size != expected_size) { 55 | throw std::runtime_error("Incorrect size: " 56 | + std::to_string(size)); 57 | } 58 | 59 | for (uint32_t i = 0; i < n; ++i) { 60 | v.addNeighbor(ntohl(*ptr++)); 61 | } 62 | return v; 63 | } 64 | 65 | void Vertex::print() const { 66 | std::cout 67 | << "Print vertex id: " << id 68 | << std::endl; 69 | } 70 | 71 | } // namespace graphs 72 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Vertex.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_WCC_VERTEX_H_ 2 | #define EXAMPLES_GRAPHS_WCC_VERTEX_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "common/Serializer.h" 13 | 14 | namespace graphs { 15 | 16 | class Vertex { 17 | public: 18 | explicit Vertex(int id = 0); 19 | Vertex(int id, const std::vector& neighbors); 20 | 21 | void addNeighbor(int id); 22 | void setNeighbors(const std::vector& neighbors); 23 | 24 | std::set getNeighbors() const; 25 | uint64_t getNeighborsSize() const; 26 | 27 | int getId() const; 28 | void setId(int id); 29 | 30 | bool hasNeighbor(int id) const; 31 | 32 | static Vertex deserializer(const void* data, unsigned int size); 33 | 34 | void print() const; 35 | 36 | public: 37 | bool seen; //< flag to indicate whether node has been traversed yet 38 | int cc; //< weakly connected component id (-1 if not assigned yet) 39 | 40 | private: 41 | std::set neighbors; //< set of the neighbors of this node 42 | 43 | int id; //< id of this vertex 44 | }; 45 | 46 | /** Format: 47 | * id (uint32_t) 48 | * n neighbors (uint32_t) 49 | * neighbors list (n * uint32_t) 50 | */ 51 | class VertexSerializer : public cirrus::Serializer { 52 | public: 53 | uint64_t size(const Vertex& v) const override { 54 | uint64_t size = sizeof(uint32_t) * 2 + 55 | sizeof(uint32_t) * v.getNeighborsSize(); // neighbors 56 | return size; 57 | } 58 | void serialize(const Vertex& v, void* mem) const override { 59 | std::cout << "Serializing vertex: " 60 | << std::endl; 61 | 62 | uint32_t* ptr = reinterpret_cast(mem); 63 | *ptr++ = htonl(v.getId()); 64 | *ptr++ = htonl(v.getNeighborsSize()); 65 | 66 | for (const auto& n : v.getNeighbors()) { 67 | *ptr++ = htonl(n); 68 | } 69 | } 70 | private: 71 | }; 72 | 73 | } // namespace graphs 74 | 75 | #endif // EXAMPLES_GRAPHS_WCC_VERTEX_H_ 76 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Wcc.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace graphs { 4 | 5 | /** Traverse all the vertices that are weakly connected 6 | * XXX We should use DFS instead 7 | */ 8 | static void bfs(int cc, cirrus::CacheManager& cm, 9 | std::list &fringe, 10 | const std::unique_ptr& ccs) { 11 | while (!fringe.empty()) { 12 | int ID = fringe.front(); 13 | fringe.pop_front(); 14 | 15 | Vertex curr = cm.get(ID); 16 | curr.seen = true; 17 | cm.put(ID, curr); // update vertex to indicate it has been seen 18 | 19 | curr.cc = cc; 20 | ccs[ID] = cc; 21 | 22 | if (!curr.getNeighbors().empty()) { 23 | for (const auto& v : curr.getNeighbors()) { 24 | auto nei = cm.get(v); 25 | if (nei.seen == false) { 26 | fringe.push_back(v); 27 | } 28 | } 29 | } 30 | } 31 | } 32 | 33 | void make_undirected(std::vector& vertices) { 34 | // make directed edges undirected 35 | for (auto& curr : vertices) { 36 | for (auto& v : curr.getNeighbors()) { 37 | Vertex n = vertices[v]; 38 | 39 | // check if node v's neighbour doesn't have v as neighbour 40 | if (!n.hasNeighbor(curr.getId())) { 41 | n.addNeighbor(curr.getId()); 42 | } 43 | } 44 | } 45 | } 46 | 47 | std::unique_ptr weakly_cc(cirrus::CacheManager& cm, 48 | unsigned int num_vertices) { 49 | // vertices have to be 0-indexed, maybe can use a map instead 50 | 51 | std::list fringe; 52 | 53 | std::unique_ptr ccs(new int[num_vertices]); 54 | std::fill(ccs.get(), ccs.get() + num_vertices, -1); 55 | 56 | int cc = 1; 57 | for (unsigned int i = 0; i < num_vertices; i ++) { 58 | Vertex curr = cm.get(i); 59 | if (!curr.seen) { 60 | fringe.push_back(i); 61 | bfs(cc, cm, fringe, ccs); 62 | cc++; 63 | } 64 | } 65 | 66 | return ccs; 67 | } 68 | 69 | 70 | } // namespace graphs 71 | -------------------------------------------------------------------------------- /examples/graphs/wcc/Wcc.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_GRAPHS_WCC_WCC_H_ 2 | #define EXAMPLES_GRAPHS_WCC_WCC_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "Vertex.h" 11 | #include "cache_manager/CacheManager.h" 12 | 13 | namespace graphs { 14 | 15 | /** 16 | * Compute the weakly connected components of the graph stored in this object store 17 | * A weakly connected component is a subset of a graph for which all vertices 18 | * are weakly connected, i.e., there is a path ignoring the edges direction 19 | * @param cm Object store where input graph is stored 20 | * @param size Number of vertices of the graph (vertices in ids 0..size-1) 21 | * @return Vector of ints containing assignment of each vertex to a CC 22 | */ 23 | std::unique_ptr weakly_cc( 24 | cirrus::CacheManager& cm, unsigned int size); 25 | 26 | /** 27 | * Transform a directed graph into an undirected one 28 | * @param Set of vertices 29 | */ 30 | void make_undirected(std::vector& vertices); 31 | 32 | } // namespace graphs 33 | 34 | #endif // EXAMPLES_GRAPHS_WCC_WCC_H_ 35 | -------------------------------------------------------------------------------- /examples/graphs/wcc/main.cpp: -------------------------------------------------------------------------------- 1 | #include "Wcc.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "Input.h" 8 | #include "Vertex.h" 9 | #include "object_store/FullBladeObjectStore.h" 10 | #include "client/TCPClient.h" 11 | #include "cache_manager/CacheManager.h" 12 | #include "cache_manager/LRAddedEvictionPolicy.h" 13 | 14 | const char PORT[] = "12345"; 15 | const char IP[] = "127.0.0.1"; 16 | const int cache_size = 200; 17 | 18 | int main(int argc, char** argv) { 19 | if (argc != 2) { 20 | throw std::runtime_error("./test_wcc "); 21 | } 22 | 23 | std::vector vertices = graphs::readGraph(argv[1]); 24 | 25 | graphs::make_undirected(vertices); 26 | 27 | graphs::VertexSerializer vs; 28 | cirrus::TCPClient client; 29 | cirrus::ostore::FullBladeObjectStoreTempl 30 | vertex_store(IP, PORT, &client, 31 | vs, graphs::Vertex::deserializer); 32 | 33 | cirrus::LRAddedEvictionPolicy policy(cache_size); 34 | cirrus::CacheManager cm(&vertex_store, &policy, cache_size); 35 | 36 | std::cout << "Adding to cm" << std::endl; 37 | for (uint64_t i = 0; i < vertices.size(); i++) { 38 | cm.put(vertices[i].getId(), vertices[i]); 39 | } 40 | 41 | auto output = graphs::weakly_cc(cm, vertices.size()); 42 | 43 | for (unsigned int i = 0; i < vertices.size(); i++) { 44 | std::cout << "Vertex ID "; 45 | std::cout << i; 46 | std::cout << ": "; 47 | std::cout << output[i] << std::endl; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /examples/ml/.gitignore: -------------------------------------------------------------------------------- 1 | parameter_server 2 | bug_regression 3 | -------------------------------------------------------------------------------- /examples/ml/Checksum.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_CHECKSUM_H_ 2 | #define EXAMPLES_ML_CHECKSUM_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | /** 9 | * Compute crc32 checksum of data in buf and with size length 10 | * @return crc32 checksum 11 | */ 12 | uint32_t crc32(const void *buf, size_t size); 13 | 14 | /** 15 | * Compute crc32 checksum for array of doubles with length size 16 | * @return crc32 checksum 17 | */ 18 | double checksum(std::shared_ptr p, uint64_t size); 19 | 20 | #endif // EXAMPLES_ML_CHECKSUM_H_ 21 | -------------------------------------------------------------------------------- /examples/ml/Dataset.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Dataset is a class that is used to manage a dataset 3 | * where each sample is a vector of doubles and the labels are also doubles 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | Dataset::Dataset() { 12 | } 13 | 14 | Dataset::Dataset(const std::vector>& samples, 15 | const std::vector& labels) : 16 | samples_(samples) { 17 | double* l = new double[labels.size()]; 18 | std::copy(labels.data(), labels.data() + labels.size(), l); 19 | labels_.reset(l, array_deleter); 20 | } 21 | 22 | Dataset::Dataset(const double* samples, 23 | const double* labels, 24 | uint64_t n_samples, 25 | uint64_t n_features) : 26 | samples_(samples, n_samples, n_features) { 27 | labels_.reset(labels); 28 | } 29 | 30 | uint64_t Dataset::num_features() const { 31 | return samples_.cols; 32 | } 33 | 34 | uint64_t Dataset::num_samples() const { 35 | return samples_.rows; 36 | } 37 | 38 | void Dataset::check_values() const { 39 | const double* l = labels_.get(); 40 | for (uint64_t i = 0; i < num_samples(); ++i) { 41 | if (std::isnan(l[i]) || std::isinf(l[i])) { 42 | throw std::runtime_error( 43 | "Dataset::check_values nan/inf error in labels"); 44 | } 45 | } 46 | samples_.check_values(); 47 | } 48 | 49 | double Dataset::checksum() const { 50 | return crc32(labels_.get(), num_samples()) + samples_.checksum(); 51 | } 52 | 53 | void Dataset::print() const { 54 | samples_.print(); 55 | } 56 | 57 | -------------------------------------------------------------------------------- /examples/ml/Dataset.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_DATASET_H_ 2 | #define EXAMPLES_ML_DATASET_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /** 10 | * This class is used to hold a dataset (both labels and samples) 11 | */ 12 | class Dataset { 13 | public: 14 | /** 15 | * Construct empty dataset 16 | */ 17 | Dataset(); 18 | 19 | /** 20 | * Construct a dataset given a vector of samples and a vector of labels 21 | * @param samples Vector of samples 22 | * @param labels Vector of labels 23 | */ 24 | Dataset(const std::vector>& samples, 25 | const std::vector& labels); 26 | /** 27 | * Construct a dataset given a vector of samples and a vector of labels 28 | * @param samples Vector of samples 29 | * @param labels Vector of labels 30 | * @param n_samples Nunber of samples in the dataset 31 | * @param n_features Number of features on each sample 32 | */ 33 | Dataset(const double* samples, 34 | const double* labels, 35 | uint64_t n_samples, 36 | uint64_t n_features); 37 | 38 | /** 39 | * Get the number of samples in this dataset 40 | * @return Number of samples in the dataset 41 | */ 42 | uint64_t num_samples() const; 43 | 44 | /** 45 | * Get the number of features in this dataset 46 | * @return Number of features in the dataset 47 | */ 48 | uint64_t num_features() const; 49 | 50 | /** 51 | * Returns pointer to specific sample in this dataset 52 | * @param sample Sample index 53 | * @returns Pointer to dataset sample 54 | */ 55 | const double* sample(uint64_t sample) const { 56 | return samples_.row(sample); 57 | } 58 | 59 | /** 60 | * Get pointer to label 61 | * @return point to label data index by input label 62 | */ 63 | const double* label(uint64_t label_index) const { 64 | return labels_.get() + label_index; 65 | } 66 | 67 | /** 68 | * Sanity check values in the dataset 69 | */ 70 | void check_values() const; 71 | 72 | /** 73 | * Compute checksum of values in the dataset 74 | * @return crc32 checksum 75 | */ 76 | double checksum() const; 77 | 78 | /** 79 | * Print this dataset 80 | */ 81 | void print() const; 82 | 83 | public: 84 | Matrix samples_; //< dataset in matrix format 85 | std::shared_ptr labels_; //< vector of labels 86 | }; 87 | 88 | #endif // EXAMPLES_ML_DATASET_H_ 89 | -------------------------------------------------------------------------------- /examples/ml/Makefile.am: -------------------------------------------------------------------------------- 1 | TOP_DIR = $(top_srcdir)/../../ 2 | 3 | include $(TOP_DIR)/common.mk 4 | AUTOMAKE_OPTIONS = foreign 5 | 6 | CXX = mpic++ 7 | #CXX = $(MPI_HOME)/bin/mpic++ 8 | 9 | bin_PROGRAMS = parameter_server two_clients_test bug_regression \ 10 | parameter_server_softmax 11 | 12 | LIBS = -lclient -lauthentication -lutils -lmpi \ 13 | -levictionpolicies 14 | 15 | if USE_RDMA 16 | LIBS += -lrdmacm -libverbs 17 | endif 18 | 19 | LINCLUDES = -L$(TOP_DIR)/src/utils/ \ 20 | -L$(TOP_DIR)/src/client/ \ 21 | -L$(TOP_DIR)/src/ \ 22 | -L$(TOP_DIR)/src/authentication \ 23 | -L$(TOP_DIR)/src/common \ 24 | -L$(TOP_DIR)/src/cache_manager 25 | 26 | LDADD = $(LINCLUDES) $(LIBS) 27 | LDFLAGS = -pthread 28 | CPPFLAGS = -ggdb -I$(TOP_DIR) -I$(TOP_DIR)/examples/sparsehash/src/ \ 29 | -I$(TOP_DIR)/src \ 30 | -I$(TOP_DIR)/third_party/flatbuffers/include \ 31 | -I$(TOP_DIR)/third_party/eigen_source \ 32 | -isystem $(TOP_DIR)/third_party/libcuckoo/ \ 33 | -I $(MPI_HOME)/include/ 34 | 35 | CPP_SOURCES = Input.cpp Utils.cpp \ 36 | Dataset.cpp Matrix.cpp Model.cpp LRModel.cpp \ 37 | ModelGradient.cpp MlUtils.cpp Configuration.cpp \ 38 | Checksum.cpp SoftmaxModel.cpp Serializers.cpp \ 39 | Tasks.cpp 40 | 41 | CPP_SOURCES_SOFTMAX = Input.cpp Utils.cpp \ 42 | Dataset.cpp Matrix.cpp Model.cpp LRModel.cpp \ 43 | ModelGradient.cpp MlUtils.cpp Configuration.cpp \ 44 | Checksum.cpp SoftmaxModel.cpp \ 45 | Tasks_softmax.cpp Serializers.cpp 46 | 47 | parameter_server_SOURCES = parameter_server.cpp $(CPP_SOURCES) 48 | parameter_server_softmax_SOURCES = parameter_server_softmax.cpp \ 49 | $(CPP_SOURCES_SOFTMAX) 50 | two_clients_test_SOURCES = two_clients_test.cpp $(CPP_SOURCES) 51 | bug_regression_SOURCES = bug_regression.cpp $(CPP_SOURCES) 52 | -------------------------------------------------------------------------------- /examples/ml/Matrix.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | Matrix::Matrix(std::vector> m) : 6 | rows(0), cols(0), data(0) { 7 | if (!m.size()) { 8 | throw std::runtime_error("Wrong vector size in Matrix"); 9 | } 10 | 11 | rows = m.size(); 12 | cols = m[0].size(); 13 | 14 | double* new_array = new double[rows * cols]; 15 | 16 | uint64_t index = 0; 17 | for (const auto& v : m) { 18 | for (const auto& vv : v) { 19 | new_array[index++] = vv; 20 | } 21 | } 22 | 23 | // bug here 24 | data.reset(const_cast(new_array), 25 | std::default_delete()); 26 | } 27 | 28 | Matrix::Matrix(const double* d, uint64_t r, uint64_t c) { 29 | rows = r; 30 | cols = c; 31 | 32 | // XXX extra copy here 33 | double* copy = new double[rows * cols]; 34 | memcpy(copy, d, rows * cols * sizeof(double)); 35 | 36 | data.reset(copy, std::default_delete()); 37 | } 38 | 39 | const double* Matrix::row(uint64_t l) const { 40 | const double* data_start = reinterpret_cast(data.get()); 41 | return &data_start[l * cols]; 42 | } 43 | 44 | Matrix Matrix::T() const { 45 | if (cached_data.get()) { 46 | return *cached_data; 47 | } 48 | 49 | cached_data.reset(new Matrix(data.get(), cols, rows)); 50 | 51 | return *cached_data; 52 | } 53 | 54 | uint64_t Matrix::sizeBytes() const { 55 | return rows * cols * sizeof(double); 56 | } 57 | 58 | void Matrix::check_values() const { 59 | for (uint64_t i = 0; i < rows; ++i) { 60 | for (uint64_t j = 0; j < cols; ++j) { 61 | double val = data.get()[i * cols + j]; 62 | if (std::isinf(val) || std::isnan(val)) { 63 | throw std::runtime_error("Matrix has nans"); 64 | } 65 | 66 | // this sanity check may break even though things are correct 67 | // though it might help catch bugs 68 | if (val > 300 || val < -300) { 69 | throw std::runtime_error("Matrix::check value: " 70 | + std::to_string(val) + " badly normalized"); 71 | } 72 | } 73 | } 74 | } 75 | double Matrix::checksum() const { 76 | return crc32(data.get(), rows * cols * sizeof(double)); 77 | } 78 | 79 | void Matrix::print() const { 80 | for (uint64_t i = 0; i < rows; ++i) { 81 | for (uint64_t j = 0; j < cols; ++j) { 82 | double val = data.get()[i * cols + j]; 83 | std::cout << val << " "; 84 | } 85 | } 86 | std::cout << std::endl; 87 | } 88 | 89 | -------------------------------------------------------------------------------- /examples/ml/Matrix.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_MATRIX_H_ 2 | #define EXAMPLES_ML_MATRIX_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | class Matrix { 10 | public: 11 | /** 12 | * Build matrix 13 | * @param m Contents of matrix in a vector of doubles format 14 | */ 15 | Matrix(std::vector> m = 16 | std::vector>()); 17 | 18 | /** 19 | * Build matrix 20 | * @param data Data in row major order format 21 | * @param rows Number of rows of matrix 22 | * @param cols Number of columns of matrix 23 | */ 24 | Matrix(const double* data, uint64_t rows, uint64_t cols); 25 | 26 | /** 27 | * Returns constant pointer to a row of the matrix 28 | * @param l Index to the row 29 | * @returns Returns const pointer to contents of the row 30 | */ 31 | const double* row(uint64_t l) const; 32 | 33 | /** 34 | * Computes and returns transpose of the matrix 35 | * @returns Transpose of matrix 36 | */ 37 | Matrix T() const; 38 | 39 | /** 40 | * Returns size (in bytes) of the matrix contents 41 | * @returns Size (bytes) of the matrix contents 42 | */ 43 | uint64_t sizeBytes() const; 44 | 45 | /** 46 | * Sanity check of values in this matrix 47 | */ 48 | void check_values() const; 49 | 50 | /** 51 | * Compute checksum of values in this matrix 52 | * @return Matrix checksum 53 | */ 54 | double checksum() const; 55 | 56 | /** 57 | * Print matrix values 58 | */ 59 | void print() const; 60 | 61 | public: 62 | uint64_t rows; //< number of rows of matrix 63 | uint64_t cols; //< number of columns of matrix 64 | std::shared_ptr data; //< pointer to matrix contents 65 | mutable std::shared_ptr cached_data; //< cache for the transpose 66 | }; 67 | 68 | #endif // EXAMPLES_ML_MATRIX_H_ 69 | -------------------------------------------------------------------------------- /examples/ml/MlUtils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace mlutils { 8 | 9 | double s_1(double x) { 10 | double res = 1.0 / (1.0 + exp(-x)); 11 | if (std::isnan(res) || std::isinf(res)) { 12 | throw std::runtime_error( 13 | std::string("s_1 generated nan/inf x: " + std::to_string(x) 14 | + " res: " + std::to_string(res))); 15 | } 16 | 17 | return res; 18 | } 19 | 20 | double log_aux(double v) { 21 | if (std::abs(v) < 10e-9) { 22 | return 10e-9; 23 | } 24 | 25 | double res = log(v); 26 | if (std::isnan(res) || std::isinf(res)) { 27 | throw std::runtime_error( 28 | std::string("log_aux generated nan/inf v: ") + 29 | to_string(v)); 30 | } 31 | 32 | return res; 33 | } 34 | 35 | } // namespace mlutils 36 | -------------------------------------------------------------------------------- /examples/ml/MlUtils.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_MLUTILS_H_ 2 | #define EXAMPLES_ML_MLUTILS_H_ 3 | 4 | namespace mlutils { 5 | 6 | /** 7 | * Computes safe sigmoid of value x 8 | * @param x Input value 9 | * @return Sigmoid of x 10 | */ 11 | double s_1(double x); 12 | 13 | /** 14 | * Computes logarithm 15 | * Check for NaN and Inf values 16 | * Clip values if they are too small (can lead to problems) 17 | * @param x Input value 18 | * @return Logarithm of x 19 | */ 20 | double log_aux(double x); 21 | 22 | } // namespace mlutils 23 | 24 | #endif // EXAMPLES_ML_MLUTILS_H_ 25 | -------------------------------------------------------------------------------- /examples/ml/Model.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | -------------------------------------------------------------------------------- /examples/ml/ModelGradient.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_MODELGRADIENT_H_ 2 | #define EXAMPLES_ML_MODELGRADIENT_H_ 3 | 4 | #include 5 | #include 6 | 7 | /** 8 | * This class is a base class for a model's gradient 9 | */ 10 | class ModelGradient { 11 | public: 12 | ModelGradient() : version(0) {} 13 | virtual ~ModelGradient() = default; 14 | 15 | /** 16 | * Serialize gradient 17 | * @param mem Pointer where to serialize the gradient 18 | */ 19 | virtual void serialize(void* mem) const = 0; 20 | 21 | /** 22 | * Get the size of the gradient when serialized 23 | * @returns Size of serialized gradient 24 | */ 25 | virtual uint64_t getSerializedSize() const = 0; 26 | 27 | /** 28 | * Load gradient from serialized memory 29 | * @param mem Pointer to memory where the gradient lives serialized 30 | */ 31 | virtual void loadSerialized(const void* mem) = 0; 32 | 33 | /** 34 | * Print gradient 35 | */ 36 | virtual void print() const = 0; 37 | 38 | /** 39 | * Set gradient version 40 | */ 41 | virtual void setVersion(uint64_t v) { 42 | version = v; 43 | } 44 | 45 | /** 46 | * Get version of gradient 47 | */ 48 | virtual uint64_t getVersion() const { 49 | return version; 50 | } 51 | 52 | /** 53 | * Sanity check gradient values 54 | */ 55 | virtual void check_values() const = 0; 56 | 57 | protected: 58 | uint64_t version = 0; //< this gradient's version 59 | }; 60 | 61 | class LRGradient : public ModelGradient { 62 | public: 63 | friend class LRModel; 64 | 65 | virtual ~LRGradient() = default; 66 | 67 | LRGradient(LRGradient&& data); 68 | explicit LRGradient(const std::vector& data); 69 | explicit LRGradient(int d); 70 | 71 | LRGradient& operator=(LRGradient&& other); 72 | 73 | void loadSerialized(const void*); 74 | void serialize(void*) const override; 75 | uint64_t getSerializedSize() const override; 76 | 77 | void print() const override; 78 | void check_values() const override; 79 | protected: 80 | std::vector weights; //< gradients of the LR gradient 81 | }; 82 | 83 | class SoftmaxGradient : public ModelGradient { 84 | public: 85 | friend class SoftmaxModel; 86 | 87 | virtual ~SoftmaxGradient() = default; 88 | 89 | SoftmaxGradient(uint64_t nclasses, uint64_t d); 90 | explicit SoftmaxGradient(const std::vector>&); 91 | 92 | void loadSerialized(const void*); 93 | void serialize(void*) const override; 94 | uint64_t getSerializedSize() const override; 95 | 96 | void print() const override; 97 | void check_values() const override; 98 | protected: 99 | // [D * K] 100 | std::vector> weights; //< weights for softmax gradient 101 | }; 102 | 103 | #endif // EXAMPLES_ML_MODELGRADIENT_H_ 104 | -------------------------------------------------------------------------------- /examples/ml/Serializers.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | uint64_t lr_model_serializer::size(const LRModel& model) const { 5 | return model.getSerializedSize(); 6 | } 7 | 8 | void lr_model_serializer::serialize(const LRModel& model, void* mem) const { 9 | model.serializeTo(mem); 10 | } 11 | 12 | LRModel 13 | lr_model_deserializer::operator()(const void* data, unsigned int des_size) { 14 | LRModel model(n); 15 | if (des_size != model.getSerializedSize()) { 16 | throw std::runtime_error( 17 | "Wrong deserializer size at lr_model_deserializer"); 18 | } 19 | model.loadSerialized(data); 20 | 21 | return model; 22 | } 23 | 24 | uint64_t lr_gradient_serializer::size(const LRGradient& g) const { 25 | return g.getSerializedSize(); 26 | } 27 | 28 | void lr_gradient_serializer::serialize(const LRGradient& g, void* mem) const { 29 | g.serialize(mem); 30 | } 31 | 32 | LRGradient 33 | lr_gradient_deserializer::operator()(const void* data, unsigned int des_size) { 34 | LRGradient gradient(n); 35 | if (des_size != gradient.getSerializedSize()) { 36 | throw std::runtime_error( 37 | "Wrong deserializer size at lr_gradient_deserializer"); 38 | } 39 | 40 | gradient.loadSerialized(data); 41 | return gradient; 42 | } 43 | 44 | 45 | /************************************************************************* 46 | ************************************************************************ 47 | * Softmax Serializers 48 | ************************************************************************ 49 | ************************************************************************ 50 | */ 51 | uint64_t sm_gradient_serializer::size(const SoftmaxGradient& g) const { 52 | return g.getSerializedSize(); 53 | } 54 | 55 | void sm_gradient_serializer::serialize(const SoftmaxGradient& g, void* mem) const { 56 | g.serialize(mem); 57 | } 58 | 59 | SoftmaxGradient 60 | sm_gradient_deserializer::operator()(const void* data, unsigned int des_size) { 61 | SoftmaxGradient gradient(nclasses, d); 62 | if (des_size != gradient.getSerializedSize()) { 63 | throw std::runtime_error( 64 | "Wrong deserializer size at sm_gradient_deserializer"); 65 | } 66 | 67 | gradient.loadSerialized(data); 68 | return gradient; 69 | } 70 | 71 | uint64_t sm_model_serializer::size(const SoftmaxModel& model) const { 72 | return model.getSerializedSize(); 73 | } 74 | 75 | void sm_model_serializer::serialize(const SoftmaxModel& model, void* mem) const { 76 | model.serializeTo(mem); 77 | } 78 | 79 | SoftmaxModel 80 | sm_model_deserializer::operator()(const void* data, unsigned int des_size) { 81 | SoftmaxModel model(nclasses, d); 82 | if (des_size != model.getSerializedSize()) { 83 | throw std::runtime_error( 84 | "Wrong deserializer size at lr_model_deserializer"); 85 | } 86 | model.loadSerialized(data); 87 | 88 | return model; 89 | } 90 | -------------------------------------------------------------------------------- /examples/ml/Utils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void check_mpi_error(int err, std::string error) { 5 | if (err != MPI_SUCCESS) { 6 | if (error == "") { 7 | throw std::runtime_error("Unknown error in MPI."); 8 | } else { 9 | throw std::runtime_error(error); 10 | } 11 | } 12 | } 13 | 14 | uint64_t get_time_us() { 15 | struct timeval tv; 16 | gettimeofday(&tv, NULL); 17 | return 1000000UL * tv.tv_sec + tv.tv_usec; 18 | } 19 | 20 | uint64_t get_time_ms() { 21 | struct timeval tv; 22 | gettimeofday(&tv, NULL); 23 | return (1000000UL * tv.tv_sec + tv.tv_usec) / 1000; 24 | } 25 | 26 | uint64_t get_time_ns() { 27 | struct timespec ts; 28 | clock_gettime(CLOCK_REALTIME, &ts); 29 | return ts.tv_nsec + ts.tv_sec * 1000000000UL; 30 | } 31 | 32 | std::ifstream::pos_type filesize(const std::string& filename) { 33 | std::ifstream in(filename.c_str(), 34 | std::ifstream::ate | std::ifstream::binary); 35 | 36 | if (!in) 37 | throw std::runtime_error("Error opening: " + filename); 38 | 39 | return in.tellg(); 40 | } 41 | 42 | void init_mpi(int argc, char**argv) { 43 | int provided; 44 | MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); 45 | if (provided != MPI_THREAD_MULTIPLE) { 46 | std::cerr 47 | << "MPI implementation does not support multiple threads" 48 | << std::endl; 49 | MPI_Abort(MPI_COMM_WORLD, 1); 50 | } 51 | } 52 | 53 | std::string hostname() { 54 | char name[200] = {0}; 55 | gethostname(name, sizeof(name)); 56 | return name; 57 | } 58 | 59 | unsigned int get_rand() { 60 | static std::mt19937 mt_rand(42); 61 | 62 | auto res = mt_rand(); 63 | 64 | return res; 65 | } 66 | 67 | double get_rand_between_0_1() { 68 | static std::mt19937 mt_rand(42); 69 | 70 | return 1.0 * mt_rand() / mt_rand.max(); 71 | } 72 | 73 | -------------------------------------------------------------------------------- /examples/ml/Utils.h: -------------------------------------------------------------------------------- 1 | #ifndef EXAMPLES_ML_UTILS_H_ 2 | #define EXAMPLES_ML_UTILS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #define LOG2(X) ((unsigned) (8*sizeof (uint64_t) - \ 15 | __builtin_clzll((X)) - 1) 16 | 17 | /** 18 | * Check output of an MPI return code 19 | * Throws exception with error message in case of MPI error 20 | */ 21 | void check_mpi_error(int err, std::string error = ""); 22 | 23 | /** 24 | * Get current time/epoch in ns 25 | */ 26 | uint64_t get_time_ns(); 27 | 28 | /** 29 | * Get current time/epoch in microseconds 30 | */ 31 | uint64_t get_time_us(); 32 | 33 | /** 34 | * Get current time/epoch in milliseconds 35 | */ 36 | uint64_t get_time_ms(); 37 | 38 | /** 39 | * Convert string to arbitrary type 40 | */ 41 | template 42 | C string_to(const std::string& s) { 43 | if (s == "") { 44 | return 0; 45 | } else { 46 | std::istringstream iss(s); 47 | C c; 48 | iss >> c; 49 | return c; 50 | } 51 | } 52 | 53 | /** 54 | * Convert arbitrary type to string 55 | */ 56 | template 57 | std::string to_string(const C& s) { 58 | std::ostringstream oss; 59 | oss << s; 60 | return oss.str(); 61 | } 62 | 63 | /** 64 | * Return the size of a file 65 | */ 66 | std::ifstream::pos_type filesize(const std::string& filename); 67 | 68 | /** 69 | * Initializes mpi 70 | */ 71 | void init_mpi(int argc, char**argv); 72 | 73 | /** 74 | * Returns the name of the host where the process is running 75 | */ 76 | std::string hostname(); 77 | 78 | /** 79 | * Get a random number 80 | */ 81 | unsigned int get_rand(); 82 | 83 | /** 84 | * Get a random number between 0 and 1 85 | */ 86 | double get_rand_between_0_1(); 87 | 88 | /** 89 | * Used to delete arrays of arbitrary teypes 90 | */ 91 | template 92 | void array_deleter(T* t) { 93 | delete[] t; 94 | } 95 | 96 | /** 97 | * Print statistical metrics of values that can be iterated on 98 | */ 99 | template 100 | void print_statistics(const T& begin, const T& end) { 101 | double max_val = DBL_MIN; 102 | double min_val = DBL_MAX; 103 | double avg = 0; 104 | 105 | for (T it = begin; it != end; ++it) { 106 | avg += *it; 107 | if (*it > max_val) 108 | max_val = *it; 109 | if (*it < min_val) 110 | min_val = *it; 111 | } 112 | 113 | std::cout << std::endl 114 | << "Min: " << min_val << std::endl 115 | << "Max: " << max_val << std::endl 116 | << "avg: " << avg / std::distance(begin, end) << std::endl 117 | << "distance: " << std::distance(begin, end) << std::endl 118 | << std::endl; 119 | } 120 | 121 | #endif // EXAMPLES_ML_UTILS_H_ 122 | -------------------------------------------------------------------------------- /examples/ml/configs/criteo_fbox.cfg: -------------------------------------------------------------------------------- 1 | input_path: /nscratch/joao/criteo_data/day_0_test_1K.csv 2 | input_type: csv_comma 3 | minibatch_size: 100 4 | learning_rate: 1e-3 5 | epsilon: 0.0001 6 | prefetching: 1 7 | model_type: LogisticRegression 8 | num_classes: 2 9 | limit_cols: 13 10 | normalize: 1 11 | num_samples: 1000 12 | -------------------------------------------------------------------------------- /examples/ml/configs/hosts: -------------------------------------------------------------------------------- 1 | f5 slots=1 2 | f6 slots=1 3 | f7 slots=1 4 | f9 slots=1 5 | -------------------------------------------------------------------------------- /examples/ml/configs/mnist_fbox.cfg: -------------------------------------------------------------------------------- 1 | input_path: /nscratch/joao/mnist.data 2 | input_type: csv_space 3 | minibatch_size: 40 4 | learning_rate: 1e-5 5 | epsilon: 0.001 6 | prefetching: 1 7 | model_type: Softmax 8 | num_classes: 10 9 | normalize: 1 10 | num_samples: 60000 11 | -------------------------------------------------------------------------------- /examples/ml/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT(CIRRUS, 0.1, joao@berkeley.edu) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_PROG_CXX([mpic++]) 10 | AC_PROG_AWK 11 | AC_PROG_CC 12 | AC_PROG_CPP 13 | AC_PROG_INSTALL 14 | AC_PROG_LN_S 15 | AC_PROG_MAKE_SET 16 | AC_PROG_RANLIB 17 | AX_CXX_COMPILE_STDCXX_14 18 | 19 | CXXFLAGS="-O3 -fPIC -std=c++1z -Werror" 20 | 21 | AM_CONDITIONAL([USE_RDMA], [test "x${HAVE_LIBRDMACM}" = "xyes"]) 22 | 23 | # Checks for header files. 24 | AC_CHECK_HEADERS([ arpa/inet.h netdb.h string.h stdint.h\ 25 | stdlib.h sys/types.h sys/socket.h sys/epoll.h\ 26 | sys/time.h syslog.h unistd.h google/dense_hash_map]) 27 | 28 | # Checks for typedefs, structures, and compiler characteristics. 29 | AC_CHECK_HEADER_STDBOOL 30 | AC_C_INLINE 31 | AC_TYPE_SIZE_T 32 | AC_TYPE_UINT32_T 33 | AC_TYPE_UINT64_T 34 | 35 | # Checks for library functions. 36 | AC_FUNC_MALLOC 37 | AC_FUNC_STRCOLL 38 | AC_CHECK_FUNCS([gettimeofday localtime_r memchr memmove memset mkdir stpcpy strchr strcspn strdup strerror strpbrk strrchr strspn strstr]) 39 | AC_CONFIG_FILES([Makefile]) 40 | 41 | AC_OUTPUT 42 | -------------------------------------------------------------------------------- /examples/ml/configure.gnu: -------------------------------------------------------------------------------- 1 | autoconf 2 | ./configure 3 | -------------------------------------------------------------------------------- /examples/ml/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | echo "Make sure server is running" 4 | 5 | cp parameter_server ~/parameter_server 6 | cp configs/criteo_fbox.cfg ~/ 7 | 8 | 9 | # valgrind 10 | #mpirun -hostfile ~/hosts -np 5 valgrind --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no --gen-suppressions=all ~/parameter_server ~/criteo_fbox.cfg 11 | #mpirun -hostfile ~/hosts -np 5 valgrind --log-file=valgrind_output --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no ~/parameter_server ~/criteo_fbox.cfg 12 | 13 | # plain 14 | 15 | SUPPORT_WORKERS=4 16 | ML_WORKERS=2 17 | TOTAL_WORKERS=$((${SUPPORT_WORKERS} + ${ML_WORKERS})) 18 | 19 | echo "Running MPI Support workers:${SUPPORT_WORKERS} Ml_workers:${ML_WORKERS}" 20 | 21 | mpirun -output-filename ~/ps_output -hostfile ~/hosts -np ${TOTAL_WORKERS} ~/parameter_server ~/criteo_fbox.cfg ${ML_WORKERS} 22 | 23 | # gdb 24 | #mpirun -hostfile ~/hosts -np ${TOTAL_WORKERS} gdb -ex run -ex bt --args ~/parameter_server ~/criteo_fbox.cfg ${ML_WORKERS} 25 | -------------------------------------------------------------------------------- /examples/ml/run_regression.sh: -------------------------------------------------------------------------------- 1 | # copy executable to the home directory 2 | cp bug_regression ~/ 3 | #mpirun -hostfile ~/hosts -np 1 valgrind --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no ~/bug_regression ~/criteo_fbox.cfg 4 | #mpirun -hostfile ~/hosts -np 1 gdb -ex run -ex bt --args ~/bug_regression ~/criteo_fbox.cfg 5 | #mpirun -hostfile ~/hosts -np 1 ~/bug_regression ~/criteo_fbox.cfg 6 | 7 | valgrind --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no ~/bug_regression ~/criteo_fbox_bug.cfg 8 | #~/bug_regression ~/criteo_fbox_bug.cfg 9 | #gdb -ex run -ex bt --args ~/bug_regression ~/criteo_fbox_bug.cfg 10 | -------------------------------------------------------------------------------- /examples/ml/run_softmax.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | echo "Make sure server is running" 4 | 5 | EXEC=parameter_server_softmax 6 | 7 | cp ${EXEC} ~/${EXEC} 8 | cp configs/mnist_fbox.cfg ~/ 9 | 10 | 11 | # valgrind 12 | #mpirun -hostfile ~/hosts -np 5 valgrind --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no --gen-suppressions=all ~/${EXEC} ~/mnist_fbox.cfg 13 | #mpirun -hostfile ~/hosts -np 5 valgrind --log-file=valgrind_output --suppressions=/home/eecs/joao/vg_suppressions_cirrus --error-limit=no ~/${EXEC} ~/mnist_fbox.cfg 14 | 15 | # plain 16 | 17 | SUPPORT_WORKERS=4 18 | ML_WORKERS=2 19 | TOTAL_WORKERS=$((${SUPPORT_WORKERS} + ${ML_WORKERS})) 20 | 21 | echo "Running ${EXEC}. MPI Support workers:${SUPPORT_WORKERS} Ml_workers:${ML_WORKERS}" 22 | 23 | mpirun -output-filename ~/ps_output -hostfile ~/hosts -np ${TOTAL_WORKERS} ~/${EXEC} ~/mnist_fbox.cfg ${ML_WORKERS} 24 | 25 | # gdb 26 | #mpirun -hostfile ~/hosts -np ${TOTAL_WORKERS} gdb -ex run -ex 'set height 0' -ex bt --args ~/${EXEC} ~/mnist_fbox.cfg ${ML_WORKERS} 27 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | Setup Docker 2 | ============= 3 | 4 | To install docker follow the instructions [1]: 5 | 6 | ~~~ 7 | sudo apt-get update 8 | sudo apt-get install \ 9 | apt-transport-https \ 10 | ca-certificates \ 11 | curl \ 12 | software-properties-common 13 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 14 | sudo add-apt-repository \ 15 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 16 | $(lsb_release -cs) \ 17 | stable" 18 | sudo apt-get install docker-ce 19 | sudo docker run hello-world 20 | ~~~ 21 | 22 | To run an empty docker container to verify RDMA function (make sure either eth2 or eth3, the RoCE interfaces, is up): 23 | 24 | ~~~ 25 | sudo docker run --net=host --ulimit memlock=-1 --device=/dev/infiniband/uverbs0 --device=/dev/infiniband/uverbs1 --device=/dev/infiniband/rdma_cm -t -i ubuntu /bin/bash 26 | ~~~ 27 | 28 | run runs the specified command (/bin/bash) inside of a container it creates 29 | ~~~ 30 | —net=host allows the container to use the host’s network stack 31 | --ulimit memlock=-1 I’m not fully sure, but it allows ibv_reg_mr to succeed 32 | —device adds an entry to /dev/infiniband/uverbs0 inside of the container 33 | -t tty mode, allows you to disconnect from a running container with CTRL-p CTRL-q 34 | -i specifies interactive mode, not sure if this is actually necessary 35 | ubuntu the image to be run 36 | /bin/bash the command being run 37 | ~~~ 38 | 39 | To run cirrus (tcpservermain specifically) inside of a container: 40 | 1. Create a directory called “docker” (name doesn’t really matter) 41 | 2. Place the attached ‘Dockerfile’ in the directory 42 | 3. git clone cirrus inside this directory (docker/cirrus) 43 | 4. switch to the docker branch (it comments out some parts of compilation. Later this could be changed to compile only the server or only the client/tests to allow for more lean containers) 44 | 5. run `sudo docker build -t cirrus_tcp_server` 45 | 46 | at this point, the docker image is created, and the tcp server can be launched via: 47 | 48 | ~~~ 49 | sudo docker run --net=host -t -i cirrus_tcp_server 50 | ~~~ 51 | 52 | the server can then be stopped via `sudo docker stop CONTAINER_NAME` 53 | 54 | With this setup the container shares an ip with the host, so multiple servers on one host would need to be differentiated via port. 55 | 56 | 57 | ~~~ 58 | FROM ubuntu 59 | 60 | WORKDIR /app 61 | ADD . /app 62 | 63 | WORKDIR ./cirrus 64 | 65 | RUN apt-get update && apt-get install -y build-essential git wget autoconf libtool g++-5 libboost-all-dev cmake gnulib python-pip && pip install cpplint 66 | RUN ./bootstrap.sh 67 | RUN make -j 10 68 | CMD ["./src/server/tcpservermain"] 69 | ~~~ 70 | 71 | 72 | [1] instructions from here https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-repository 73 | -------------------------------------------------------------------------------- /scripts/disable_swapping.sh: -------------------------------------------------------------------------------- 1 | sudo swapoff -a 2 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = foreign 2 | SUBDIRS = authentication common utils server client object_store cache_manager \ 3 | iterator 4 | -------------------------------------------------------------------------------- /src/authentication/ApplicationKey.cpp: -------------------------------------------------------------------------------- 1 | #include "authentication/ApplicationKey.h" 2 | 3 | namespace cirrus { 4 | 5 | ApplicationKey::ApplicationKey() { 6 | } 7 | 8 | } // namespace cirrus 9 | -------------------------------------------------------------------------------- /src/authentication/ApplicationKey.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_AUTHENTICATION_APPLICATIONKEY_H_ 2 | #define SRC_AUTHENTICATION_APPLICATIONKEY_H_ 3 | 4 | namespace cirrus { 5 | 6 | class ApplicationKey { 7 | public: 8 | ApplicationKey(); 9 | private: 10 | }; 11 | 12 | } // namespace cirrus 13 | 14 | #endif // SRC_AUTHENTICATION_APPLICATIONKEY_H_ 15 | -------------------------------------------------------------------------------- /src/authentication/AuthenticationToken.cpp: -------------------------------------------------------------------------------- 1 | #include "authentication/AuthenticationToken.h" 2 | 3 | namespace cirrus { 4 | 5 | AuthenticationToken::AuthenticationToken(bool allow) : 6 | allow(allow) { 7 | } 8 | 9 | AuthenticationToken AuthenticationToken::create_default_allow_token() { 10 | return AuthenticationToken(true); 11 | } 12 | 13 | AuthenticationToken AuthenticationToken::create_default_deny_token() { 14 | return AuthenticationToken(false); 15 | } 16 | 17 | } // namespace cirrus 18 | -------------------------------------------------------------------------------- /src/authentication/AuthenticationToken.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_AUTHENTICATION_AUTHENTICATIONTOKEN_H_ 2 | #define SRC_AUTHENTICATION_AUTHENTICATIONTOKEN_H_ 3 | 4 | #include "GrantingKey.h" 5 | 6 | namespace cirrus { 7 | 8 | class AuthenticationToken { 9 | public: 10 | explicit AuthenticationToken(bool allow); 11 | 12 | static AuthenticationToken create_default_deny_token(); 13 | static AuthenticationToken create_default_allow_token(); 14 | 15 | // crypto key 16 | GrantingKey gkey; 17 | // let application know if 18 | // access was granted 19 | char allow; 20 | }; 21 | 22 | }; // namespace cirrus 23 | 24 | #endif // SRC_AUTHENTICATION_AUTHENTICATIONTOKEN_H_ 25 | -------------------------------------------------------------------------------- /src/authentication/Authenticator.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_AUTHENTICATION_AUTHENTICATOR_H_ 2 | #define SRC_AUTHENTICATION_AUTHENTICATOR_H_ 3 | 4 | #include "authentication/ApplicationKey.h" 5 | #include "common/AllocatorMessage.h" 6 | 7 | namespace cirrus { 8 | 9 | class Authenticator { 10 | public: 11 | virtual bool allowApplication(const AppId& app_id) = 0; 12 | private: 13 | }; 14 | 15 | } // namespace cirrus 16 | 17 | #endif // SRC_AUTHENTICATION_AUTHENTICATOR_H_ 18 | -------------------------------------------------------------------------------- /src/authentication/GrantingKey.cpp: -------------------------------------------------------------------------------- 1 | #include "authentication/GrantingKey.h" 2 | 3 | #include 4 | 5 | namespace cirrus { 6 | 7 | GrantingKey::GrantingKey() { 8 | std::memset(data_, 0, sizeof(data_)); 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/authentication/GrantingKey.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_AUTHENTICATION_GRANTINGKEY_H_ 2 | #define SRC_AUTHENTICATION_GRANTINGKEY_H_ 3 | 4 | namespace cirrus { 5 | 6 | class GrantingKey { 7 | public: 8 | GrantingKey(); 9 | 10 | static constexpr int KEY_SIZE = 10; 11 | 12 | // if access granted access key goes 13 | // here. To be used when contacting 14 | // memory blades 15 | char data_[KEY_SIZE]; 16 | }; 17 | 18 | } // namespace cirrus 19 | 20 | #endif // SRC_AUTHENTICATION_GRANTINGKEY_H_ 21 | -------------------------------------------------------------------------------- /src/authentication/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | noinst_LIBRARIES = libauthentication.a 6 | libauthentication_a_SOURCES = ApplicationKey.cpp AuthenticationToken.cpp GrantingKey.cpp 7 | libauthentication_a_CPPFLAGS = -ggdb -I$(top_srcdir) -I$(top_srcdir)/src 8 | -------------------------------------------------------------------------------- /src/cache_manager/EvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_manager/EvictionPolicy.h" 2 | 3 | namespace cirrus { 4 | 5 | EvictionPolicy::EvictionPolicy() { 6 | } 7 | 8 | } // namespace cirrus 9 | -------------------------------------------------------------------------------- /src/cache_manager/EvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CACHE_MANAGER_EVICTIONPOLICY_H_ 2 | #define SRC_CACHE_MANAGER_EVICTIONPOLICY_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | using ObjectID = uint64_t; 9 | 10 | /** 11 | * A class that defines the interface for eviction policies. The methods 12 | * inside the class are called when their counterparts are called in the 13 | * cache manager. Each method returns a list of ObjectIDs to be removed. 14 | * If these IDs are not in the cache, an error will be thrown. 15 | */ 16 | class EvictionPolicy { 17 | public: 18 | EvictionPolicy() : max_size(0) {} 19 | /** 20 | * Counterpart to the get method. Returns a list of the ObjectIDs to be 21 | * purged from the cache. 22 | */ 23 | virtual std::vector get(ObjectID oid) = 0; 24 | 25 | /** 26 | * Counterpart to the put method. Returns a vector of the ObjectIDs to be 27 | * purged from the cache. 28 | */ 29 | virtual std::vector put(ObjectID oid) = 0; 30 | 31 | /** 32 | * Counterpart to the prefetch method. Returns a vector of the ObjectIDs 33 | * to be purged from the cache. 34 | */ 35 | virtual std::vector prefetch(ObjectID oid) = 0; 36 | 37 | /** 38 | * Counterpart to the remove method. If an oid is passed in that is not 39 | * currently in the cache, should return immediately with no error. 40 | */ 41 | virtual void remove(ObjectID oid) = 0; 42 | 43 | private: 44 | /** 45 | * The maximum capacity of the cache. Will never be exceeded. Set 46 | * at time of instantiation. 47 | */ 48 | uint64_t max_size; 49 | }; 50 | 51 | } // namespace cirrus 52 | 53 | #endif // SRC_CACHE_MANAGER_EVICTIONPOLICY_H_ 54 | -------------------------------------------------------------------------------- /src/cache_manager/LRAddedEvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_manager/LRAddedEvictionPolicy.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "common/Exception.h" 9 | 10 | namespace cirrus { 11 | 12 | /** 13 | * Constructor for the eviction policy. 14 | */ 15 | LRAddedEvictionPolicy::LRAddedEvictionPolicy(uint64_t max_size) : 16 | max_size(max_size) {} 17 | 18 | 19 | /** 20 | * Get method for the eviction policy. If the cache is at capacity when get 21 | * is called, the oldest item is returned to be removed from the cache. 22 | */ 23 | std::vector LRAddedEvictionPolicy::get(ObjectID oid) { 24 | return process_oid(oid); 25 | } 26 | 27 | /** 28 | * Put method for the eviction policy. 29 | */ 30 | std::vector LRAddedEvictionPolicy::put(ObjectID oid) { 31 | return process_oid(oid); 32 | } 33 | 34 | /** 35 | * Prefetch method for the eviction policy. If the cache is at capacity when get 36 | * is called, the oldest item is returned to be removed from the cache. 37 | */ 38 | std::vector LRAddedEvictionPolicy::prefetch(ObjectID oid) { 39 | return process_oid(oid); 40 | } 41 | 42 | /** 43 | * Remove method for the eviction policy. Simply records internally that the 44 | * object is no longer in the cache. 45 | */ 46 | void LRAddedEvictionPolicy::remove(ObjectID oid) { 47 | auto it = object_set.find(oid); 48 | if (it != object_set.end()) { 49 | // remove item from set 50 | object_set.erase(it); 51 | // remove item from deque 52 | // This is a linear search and is thus O(n), which will severely impact 53 | // performance when the store is very full. 54 | auto deque_iterator = std::find(object_deque.begin(), 55 | object_deque.end(), 56 | oid); 57 | object_deque.erase(deque_iterator); 58 | } 59 | } 60 | /** 61 | * Method that processes an item being added to the cache. If the cache 62 | * is at capacity when it is called, the oldest item is returned to be 63 | * removed from the cache. 64 | */ 65 | std::vector LRAddedEvictionPolicy::process_oid(ObjectID oid) { 66 | auto it = object_set.find(oid); 67 | if (it == object_set.end()) { 68 | object_set.insert(oid); 69 | object_deque.push_back(oid); 70 | if (object_deque.size() > max_size) { 71 | // remove oldest from deque 72 | ObjectID to_remove = object_deque.front(); 73 | 74 | // remove item from deque 75 | object_deque.pop_front(); 76 | // remove oldest from set 77 | auto set_iterator = object_set.find(to_remove); 78 | object_set.erase(set_iterator); 79 | 80 | std::vector return_vec = {to_remove}; 81 | return return_vec; 82 | } else { 83 | return std::vector(); 84 | } 85 | } else { 86 | return std::vector(); 87 | } 88 | } 89 | } // namespace cirrus 90 | -------------------------------------------------------------------------------- /src/cache_manager/LRAddedEvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CACHE_MANAGER_LRADDEDEVICTIONPOLICY_H_ 2 | #define SRC_CACHE_MANAGER_LRADDEDEVICTIONPOLICY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "cache_manager/EvictionPolicy.h" 9 | 10 | namespace cirrus { 11 | 12 | using ObjectID = uint64_t; 13 | 14 | /** 15 | * A class that inherits from EvictionPolicy. When it needs to evict an item, 16 | * evicts the single oldest item. 17 | */ 18 | class LRAddedEvictionPolicy : public EvictionPolicy { 19 | public: 20 | explicit LRAddedEvictionPolicy(uint64_t max_num_objs); 21 | 22 | std::vector get(ObjectID oid) override; 23 | 24 | std::vector put(ObjectID oid) override; 25 | 26 | std::vector prefetch(ObjectID oid) override; 27 | 28 | void remove(ObjectID oid) override; 29 | 30 | private: 31 | std::vector process_oid(ObjectID oid); 32 | 33 | /** 34 | * The maximum capacity of the cache. Will never be exceeded. Set 35 | * at time of instantiation. 36 | */ 37 | uint64_t max_size; 38 | 39 | /** 40 | * Queue to store the order that objects were inserted. 41 | */ 42 | std::deque object_deque; 43 | 44 | /** 45 | * Set to keep track of which objects are present. 46 | */ 47 | std::set object_set; 48 | }; 49 | 50 | } // namespace cirrus 51 | 52 | #endif // SRC_CACHE_MANAGER_LRADDEDEVICTIONPOLICY_H_ 53 | -------------------------------------------------------------------------------- /src/cache_manager/LRUEvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_manager/LRUEvictionPolicy.h" 2 | #include 3 | #include 4 | #include 5 | 6 | #include "common/Exception.h" 7 | 8 | namespace cirrus { 9 | 10 | /** 11 | * Constructor for the eviction policy. 12 | */ 13 | LRUEvictionPolicy::LRUEvictionPolicy(uint64_t max_size) : 14 | max_size(max_size) {} 15 | 16 | 17 | /** 18 | * Get method for the eviction policy. If the cache is at capacity when get 19 | * is called, the oldest item is returned to be removed from the cache. 20 | */ 21 | std::vector LRUEvictionPolicy::get(ObjectID oid) { 22 | return process_oid(oid); 23 | } 24 | 25 | /** 26 | * Put method for the eviction policy. Returns empty list as a put will never 27 | * require additional cache space. 28 | */ 29 | std::vector LRUEvictionPolicy::put(ObjectID /* oid */) { 30 | return std::vector(); 31 | } 32 | 33 | /** 34 | * Prefetch method for the eviction policy. If the cache is at capacity when get 35 | * is called, the oldest item is returned to be removed from the cache. 36 | */ 37 | std::vector LRUEvictionPolicy::prefetch(ObjectID oid) { 38 | return process_oid(oid); 39 | } 40 | 41 | /** 42 | * Remove method for the eviction policy. Simply records internally that the 43 | * object is no longer in the cache. 44 | */ 45 | void LRUEvictionPolicy::remove(ObjectID oid) { 46 | auto it = object_map.find(oid); 47 | if (it != object_map.end()) { 48 | // remove item from set 49 | object_list.erase(object_map[oid]); 50 | object_map.erase(it); 51 | } 52 | } 53 | /** 54 | * Method that processes an item being added to the cache. If the cache 55 | * is at capacity when it is called, the least recently used item is removed. 56 | */ 57 | std::vector LRUEvictionPolicy::process_oid(ObjectID oid) { 58 | auto it = object_map.find(oid); 59 | if (it == object_map.end()) { 60 | object_map[oid] = object_list.insert(object_list.begin(), oid); 61 | if (object_map.size() > max_size) { 62 | // remove oldest from deque 63 | ObjectID to_remove = object_list.back(); 64 | object_list.pop_back(); 65 | 66 | auto map_iterator = object_map.find(to_remove); 67 | object_map.erase(map_iterator); 68 | 69 | std::vector return_vec = {to_remove}; 70 | return return_vec; 71 | } else { 72 | return std::vector(); 73 | } 74 | } else { 75 | // Move an object back to the front if it has been accessed. 76 | object_list.splice(object_list.begin(), object_list, it->second); 77 | return std::vector(); 78 | } 79 | } 80 | } // namespace cirrus 81 | -------------------------------------------------------------------------------- /src/cache_manager/LRUEvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CACHE_MANAGER_LRUEVICTIONPOLICY_H_ 2 | #define SRC_CACHE_MANAGER_LRUEVICTIONPOLICY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "cache_manager/EvictionPolicy.h" 10 | 11 | namespace cirrus { 12 | 13 | using ObjectID = uint64_t; 14 | 15 | /** 16 | * A class that inherits from EvictionPolicy. When it needs to evict an item, 17 | * evicts least recently used item. 18 | */ 19 | class LRUEvictionPolicy : public EvictionPolicy { 20 | public: 21 | explicit LRUEvictionPolicy(uint64_t max_num_objs); 22 | 23 | std::vector get(ObjectID oid) override; 24 | 25 | std::vector put(ObjectID oid) override; 26 | 27 | std::vector prefetch(ObjectID oid) override; 28 | 29 | void remove(ObjectID oid) override; 30 | 31 | private: 32 | std::vector process_oid(ObjectID oid); 33 | 34 | /** 35 | * The maximum capacity of the cache. Will never be exceeded. Set 36 | * at time of instantiation. 37 | */ 38 | uint64_t max_size; 39 | 40 | /** 41 | * Unordered map to track iterators to inside of list. 42 | */ 43 | std::unordered_map::iterator> object_map; 44 | 45 | /** 46 | * List to keep track of ordering. 47 | */ 48 | std::list object_list; 49 | }; 50 | 51 | } // namespace cirrus 52 | 53 | #endif // SRC_CACHE_MANAGER_LRUEVICTIONPOLICY_H_ 54 | -------------------------------------------------------------------------------- /src/cache_manager/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | noinst_LIBRARIES = libevictionpolicies.a 6 | libevictionpolicies_a_SOURCES = LRAddedEvictionPolicy.cpp LRUEvictionPolicy.cpp 7 | 8 | libevictionpolicies_a_CPPFLAGS = -ggdb -I$(top_srcdir) -I$(top_srcdir)/src 9 | -------------------------------------------------------------------------------- /src/cache_manager/PrefetchPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CACHE_MANAGER_PREFETCHPOLICY_H_ 2 | #define SRC_CACHE_MANAGER_PREFETCHPOLICY_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | using ObjectID = uint64_t; 9 | 10 | /** 11 | * A class that defines the interface for prefetch policies. The get method 12 | * inside the class is called when its counterpart is called inside the 13 | * cache manager. A list of ObjectIDs to prefetch is then return 14 | * If these IDs are not in the cache, an error will be thrown. 15 | */ 16 | template 17 | class PrefetchPolicy { 18 | public: 19 | PrefetchPolicy() = default; 20 | /** 21 | * Counterpart to the get method. Returns a list of the ObjectIDs to be 22 | * prefetched from the cache. 23 | * @param id the objectID that was obtained. 24 | * @param obj a const reference to the object retrieved, potentially 25 | * useful in determining the next objects to prefetch. 26 | * @return a vector of ObjectIDs to prefetch. 27 | */ 28 | virtual std::vector get(const ObjectID& id, const T& obj) = 0; 29 | }; 30 | 31 | } // namespace cirrus 32 | 33 | #endif // SRC_CACHE_MANAGER_PREFETCHPOLICY_H_ 34 | -------------------------------------------------------------------------------- /src/client/.gitignore: -------------------------------------------------------------------------------- 1 | clientmain 2 | scalability_main 3 | -------------------------------------------------------------------------------- /src/client/BladeClient.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CLIENT_BLADECLIENT_H_ 2 | #define SRC_CLIENT_BLADECLIENT_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "common/Serializer.h" 10 | #include "common/Synchronization.h" 11 | #include "common/Exception.h" 12 | #include "utils/Log.h" 13 | 14 | namespace cirrus { 15 | 16 | using ObjectID = uint64_t; 17 | 18 | struct FutureData { 19 | FutureData( 20 | bool result = false, 21 | bool result_available = false, 22 | cirrus::ErrorCodes error_code = cirrus::ErrorCodes(), 23 | std::shared_ptr data_ptr = nullptr, 24 | uint64_t data_size = 0); 25 | 26 | /** Pointer to the result. */ 27 | bool result; 28 | /** Boolean monitoring result state. */ 29 | bool result_available; 30 | /** lock for the result. */ 31 | std::shared_ptr sem; 32 | /** Any errors thrown. */ 33 | cirrus::ErrorCodes error_code; 34 | /** Pointer to any mem for a read, if any. */ 35 | std::shared_ptr data_ptr; 36 | /** Size of the memory block for a read. */ 37 | uint64_t data_size; 38 | }; 39 | 40 | /** 41 | * A class that all clients inherit from. Outlines the interface that the 42 | * store will use to interface with the network level. 43 | */ 44 | class BladeClient { 45 | public: 46 | class ClientFuture { 47 | public: 48 | explicit ClientFuture(std::shared_ptr fd); 49 | ClientFuture() = default; 50 | void wait(); 51 | 52 | bool try_wait(); 53 | 54 | bool get(); 55 | 56 | std::pair, unsigned int> getDataPair(); 57 | 58 | protected: 59 | std::shared_ptr fd; 60 | }; 61 | 62 | virtual ~BladeClient() = default; 63 | 64 | virtual void connect(const std::string& address, 65 | const std::string& port) = 0; 66 | 67 | // Read 68 | virtual std::pair, unsigned int> read_sync( 69 | ObjectID id) = 0; 70 | virtual std::pair, unsigned int> read_sync_bulk( 71 | const std::vector& ids) = 0; 72 | 73 | virtual BladeClient::ClientFuture read_async(ObjectID oid) = 0; 74 | virtual BladeClient::ClientFuture read_async_bulk( 75 | const std::vector& oids) = 0; 76 | 77 | // Write 78 | virtual bool write_sync(ObjectID id, const WriteUnit& w) = 0; 79 | virtual bool write_sync_bulk( 80 | const std::vector& oids, 81 | const WriteUnits& w) = 0; 82 | 83 | virtual BladeClient::ClientFuture write_async(ObjectID oid, 84 | const WriteUnit& w) = 0; 85 | virtual BladeClient::ClientFuture write_async_bulk( 86 | const std::vector& oids, 87 | const WriteUnits& w) = 0; 88 | 89 | virtual bool remove(ObjectID id) = 0; 90 | }; 91 | 92 | } // namespace cirrus 93 | 94 | #endif // SRC_CLIENT_BLADECLIENT_H_ 95 | -------------------------------------------------------------------------------- /src/client/DataPointer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CLIENT_DATAPOINTER_H_ 2 | #define SRC_CLIENT_DATAPOINTER_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | /** A class that allows us to share access to data. 9 | * Apps can pass around DataPointers instead of 10 | * shipping data 11 | */ 12 | class DataPointer { 13 | DataPointer(std::string blade_addr, 14 | std::string blade_port, 15 | uint64_t remote_addr); 16 | // FIX: we also need a key here 17 | 18 | private: 19 | std::string blade_address_; 20 | std::string blade_port_; 21 | uint64_t remote_addr_; 22 | }; 23 | 24 | } // namespace cirrus 25 | 26 | #endif // SRC_CLIENT_DATAPOINTER_H_ 27 | -------------------------------------------------------------------------------- /src/client/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | SOURCES = TCPClient.cpp BladeClient.cpp 6 | 7 | LIBS = -lclient -L../utils/ -lutils -L../authentication/ -lauthentication \ 8 | -L../common/ -lcommon -L. $(LIBRDMACM) $(LIBIBVERBS) 9 | 10 | if USE_RDMA 11 | SOURCES += RDMAClient.cpp 12 | endif 13 | 14 | noinst_LIBRARIES = libclient.a 15 | 16 | libclient_a_SOURCES = $(SOURCES) 17 | libclient_a_CPPFLAGS = -ggdb -I$(top_srcdir) -I$(top_srcdir)/src \ 18 | -I$(top_srcdir)/third_party/flatbuffers/include \ 19 | -isystem $(top_srcdir)/third_party/libcuckoo 20 | -------------------------------------------------------------------------------- /src/common/AllocationRecord.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_ALLOCATIONRECORD_H_ 2 | #define SRC_COMMON_ALLOCATIONRECORD_H_ 3 | 4 | namespace cirrus { 5 | 6 | /** 7 | * Contains information tracking an allocation. 8 | */ 9 | struct AllocationRecord { 10 | int alloc_id; 11 | uint64_t remote_addr; 12 | uint64_t peer_rkey; 13 | 14 | AllocationRecord( 15 | int alloc_id_ = 0, 16 | uint64_t remote_addr_ = 0, 17 | uint64_t peer_rkey_ = 0) : 18 | alloc_id(alloc_id_), remote_addr(remote_addr_), peer_rkey(peer_rkey_) 19 | {} 20 | }; 21 | 22 | } // namespace cirrus 23 | 24 | #endif // SRC_COMMON_ALLOCATIONRECORD_H_ 25 | -------------------------------------------------------------------------------- /src/common/AllocatorMessage.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_ALLOCATORMESSAGE_H_ 2 | #define SRC_COMMON_ALLOCATORMESSAGE_H_ 3 | 4 | #include 5 | #include "authentication/AuthenticationToken.h" 6 | 7 | namespace cirrus { 8 | 9 | enum auth_msg_type { 10 | AUTH1, AUTH_ACK1, AUTH2, AUTH_ACK2, 11 | STATS, STATS_ACK, 12 | ALLOC, ALLOC_ACK, 13 | }; 14 | 15 | using AppId = uint64_t; 16 | 17 | struct AllocatorMessage { 18 | int id; 19 | auth_msg_type type; 20 | union { 21 | // applications can ask 22 | // to be authenticated and get 23 | // a granting key 24 | struct { 25 | AppId app_id; 26 | } auth1; 27 | struct { 28 | char allow; 29 | uint64_t challenge; 30 | } auth_ack1; 31 | struct { 32 | uint64_t response; 33 | } auth2; 34 | struct { 35 | char allow; 36 | AuthenticationToken auth_token; 37 | } auth_ack2; 38 | struct { 39 | } stats; 40 | struct { 41 | uint64_t total_mem_allocated; 42 | } stats_ack; 43 | struct { 44 | } alloc; 45 | struct { 46 | } alloc_ack; 47 | } data; 48 | }; 49 | 50 | } // namespace cirrus 51 | 52 | #endif // SRC_COMMON_ALLOCATORMESSAGE_H_ 53 | -------------------------------------------------------------------------------- /src/common/AllocatorMessageGenerator.cpp: -------------------------------------------------------------------------------- 1 | #include "common/AllocatorMessageGenerator.h" 2 | 3 | #include 4 | 5 | namespace cirrus { 6 | 7 | void AllocatorMessageGenerator::auth1(void *data, AppId app_id) { 8 | AllocatorMessage* msg = 9 | reinterpret_cast(data); 10 | 11 | msg->type = AUTH1; 12 | msg->data.auth1.app_id = app_id; 13 | } 14 | 15 | void AllocatorMessageGenerator::auth_ack1(void *data, 16 | char allow, 17 | uint64_t challenge) { 18 | AllocatorMessage* msg = 19 | reinterpret_cast(data); 20 | 21 | msg->type = AUTH_ACK1; 22 | msg->data.auth_ack1.allow = allow; 23 | msg->data.auth_ack1.challenge = challenge; 24 | } 25 | 26 | void AllocatorMessageGenerator::auth2(void *data, uint64_t response) { 27 | AllocatorMessage* msg = 28 | reinterpret_cast(data); 29 | 30 | msg->type = AUTH2; 31 | msg->data.auth2.response = response; 32 | } 33 | 34 | void AllocatorMessageGenerator::auth_ack2(void *data, 35 | char allow, 36 | AuthenticationToken auth_token) { 37 | AllocatorMessage* msg = 38 | reinterpret_cast(data); 39 | 40 | msg->type = AUTH_ACK2; 41 | msg->data.auth_ack2.allow = allow; 42 | msg->data.auth_ack2.auth_token = auth_token; 43 | } 44 | 45 | void AllocatorMessageGenerator::stats(void *data) { 46 | AllocatorMessage* msg = 47 | reinterpret_cast(data); 48 | 49 | msg->type = STATS; 50 | } 51 | 52 | void AllocatorMessageGenerator::stats_ack(void *data, 53 | uint64_t total_mem_allocated) { 54 | AllocatorMessage* msg = 55 | reinterpret_cast(data); 56 | 57 | msg->type = STATS_ACK; 58 | msg->data.stats_ack.total_mem_allocated = total_mem_allocated; 59 | } 60 | 61 | void AllocatorMessageGenerator::alloc(void *data) { 62 | AllocatorMessage* msg = 63 | reinterpret_cast(data); 64 | 65 | msg->type = ALLOC; 66 | } 67 | 68 | void AllocatorMessageGenerator::alloc_ack(void *data) { 69 | AllocatorMessage* msg = 70 | reinterpret_cast(data); 71 | 72 | msg->type = ALLOC_ACK; 73 | } 74 | 75 | } // namespace cirrus 76 | -------------------------------------------------------------------------------- /src/common/AllocatorMessageGenerator.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_ALLOCATORMESSAGEGENERATOR_H_ 2 | #define SRC_COMMON_ALLOCATORMESSAGEGENERATOR_H_ 3 | 4 | #include 5 | #include "common/AllocatorMessage.h" 6 | #include "authentication/ApplicationKey.h" 7 | 8 | namespace cirrus { 9 | 10 | class AllocatorMessageGenerator { 11 | public: 12 | static void auth1(void *data, AppId app_id); 13 | static void auth_ack1(void *data, char allow, uint64_t challenge); 14 | static void auth2(void *data, uint64_t challenge); 15 | static void auth_ack2(void *data, char allow, 16 | AuthenticationToken auth_token); 17 | 18 | static void stats(void *data); 19 | static void stats_ack(void *data, uint64_t total_mem_allocated); 20 | 21 | static void alloc(void *data); 22 | static void alloc_ack(void *data); 23 | }; 24 | 25 | } // namespace cirrus 26 | 27 | #endif // SRC_COMMON_ALLOCATORMESSAGEGENERATOR_H_ 28 | -------------------------------------------------------------------------------- /src/common/Decls.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_DECLS_H_ 2 | #define SRC_COMMON_DECLS_H_ 3 | 4 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 5 | TypeName(const TypeName&) = delete; \ 6 | void operator=(const TypeName&) = delete 7 | 8 | #endif // SRC_COMMON_DECLS_H_ 9 | -------------------------------------------------------------------------------- /src/common/Exception.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_EXCEPTION_H_ 2 | #define SRC_COMMON_EXCEPTION_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace cirrus { 8 | 9 | // TODO(Tyler): Add in enum for exceptions 10 | 11 | enum ErrorCodes { 12 | kOk = 0, 13 | kException, 14 | kServerMemoryErrorException, 15 | kNoSuchIDException, 16 | }; 17 | 18 | /** 19 | * A class for cirrus generated exceptions. 20 | */ 21 | class Exception : public std::exception { 22 | public: 23 | explicit Exception(std::string msg): msg(msg) {} 24 | const char* what() const throw() { 25 | return msg.c_str(); 26 | } 27 | private: 28 | std::string msg; 29 | }; 30 | 31 | /** 32 | * An exception generated when the remote server experience an issue 33 | * during memory allocation. This could be due to the server running out 34 | * of memory. 35 | */ 36 | class ServerMemoryErrorException : public cirrus::Exception { 37 | public: 38 | explicit ServerMemoryErrorException(std::string msg): 39 | cirrus::Exception(msg) {} 40 | }; 41 | 42 | /** 43 | * An exception generated when the local cache is completely filled and 44 | * the user attempts to perform a get operation, either directly or through 45 | * the use of an iterator. Also thrown if user sets max size of the 46 | * cache to be zero. 47 | */ 48 | class CacheCapacityException : public cirrus::Exception { 49 | public: 50 | explicit CacheCapacityException(std::string msg): 51 | cirrus::Exception(msg) {} 52 | }; 53 | 54 | /** 55 | * An exception generated when the user requests a get operation on an 56 | * ObjectID that was either never pushed to the remote store or was removed 57 | * from the remote store. 58 | */ 59 | class NoSuchIDException : public cirrus::Exception { 60 | public: 61 | explicit NoSuchIDException(std::string msg): 62 | cirrus::Exception(msg) {} 63 | }; 64 | 65 | /** 66 | * An exception generated when the client or server fail to make a connection 67 | * with the other. 68 | */ 69 | class ConnectionException : public cirrus::Exception { 70 | public: 71 | explicit ConnectionException(std::string msg): 72 | cirrus::Exception(msg) {} 73 | }; 74 | 75 | /** 76 | * An exception generated when a read on the server fails. 77 | */ 78 | class ServerReadException : public cirrus::Exception { 79 | public: 80 | explicit ServerReadException(std::string msg): 81 | cirrus::Exception(msg) {} 82 | }; 83 | 84 | } // namespace cirrus 85 | 86 | #endif // SRC_COMMON_EXCEPTION_H_ 87 | -------------------------------------------------------------------------------- /src/common/FileAllocationRecord.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_FILEALLOCATIONRECORD_H_ 2 | #define SRC_COMMON_FILEALLOCATIONRECORD_H_ 3 | 4 | namespace cirrus { 5 | 6 | /** 7 | * A struct that contains information on where to find an allocated file. 8 | */ 9 | struct FileAllocationRecord { 10 | /** 11 | * The remote address where this file is stored. 12 | */ 13 | uint64_t remote_addr; /** The remote address where this file is stored. */ 14 | /** 15 | * The peer_rkey for this file. 16 | */ 17 | uint64_t peer_rkey; 18 | 19 | FileAllocationRecord(uint64_t remote_addr_, uint64_t peer_rkey_) : 20 | remote_addr(remote_addr_), peer_rkey(peer_rkey_) 21 | {} 22 | }; 23 | 24 | } // namespace cirrus 25 | 26 | #endif // SRC_COMMON_FILEALLOCATIONRECORD_H_ 27 | -------------------------------------------------------------------------------- /src/common/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | SUBDIRS = schemas 5 | noinst_LIBRARIES = libcommon.a 6 | libcommon_a_SOURCES = AllocatorMessageGenerator.cpp Synchronization.cpp 7 | 8 | libcommon_a_CPPFLAGS = -ggdb -I$(top_srcdir) -I$(top_srcdir)/src 9 | -------------------------------------------------------------------------------- /src/common/Serializer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_SERIALIZER_H_ 2 | #define SRC_COMMON_SERIALIZER_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace cirrus { 8 | 9 | /** 10 | * A template class outlining the interface that all serializers should have. 11 | */ 12 | template 13 | class Serializer { 14 | public: 15 | /** 16 | * Purely virtual function. Should return size needed to serialize when 17 | * object is passed in. 18 | */ 19 | virtual uint64_t size(const T& object) const = 0; 20 | /** 21 | * Purely virtual function. Should serialize the object passed in. 22 | * @param object the object to be serialized. 23 | * @param mem the memory the object should be serialized into. 24 | */ 25 | virtual void serialize(const T& object, void *mem) const = 0; 26 | }; 27 | 28 | class WriteUnit { 29 | public: 30 | virtual uint64_t size() const = 0; 31 | virtual void serialize(void *mem) const = 0; 32 | }; 33 | 34 | template 35 | class WriteUnitTemplate : public WriteUnit { 36 | public: 37 | WriteUnitTemplate(const cirrus::Serializer& serializer, const T& obj) : 38 | serializer(serializer), obj(obj) {} 39 | 40 | void serialize(void *mem) const override { 41 | serializer.serialize(obj, mem); 42 | return; 43 | } 44 | 45 | uint64_t size() const override { 46 | return serializer.size(obj); 47 | } 48 | 49 | private: 50 | const cirrus::Serializer& serializer; 51 | const T& obj; 52 | }; 53 | 54 | class WriteUnits { 55 | public: 56 | virtual uint64_t size() const = 0; 57 | virtual void serialize(void *mem) const = 0; 58 | }; 59 | 60 | template 61 | class WriteUnitsTemplate : public WriteUnits { 62 | public: 63 | WriteUnitsTemplate(const cirrus::Serializer& serializer, 64 | const std::vector& objs) : 65 | serializer(serializer), objs(objs) {} 66 | 67 | void serialize(void *mem) const override { 68 | char* ptr = reinterpret_cast(mem); 69 | for (uint64_t i = 0; i < objs.size(); ++i) { 70 | uint64_t object_size = serializer.size(*objs[i]); 71 | *reinterpret_cast(ptr) = htonl(object_size); 72 | ptr += sizeof(uint64_t); 73 | serializer.serialize(*objs[i], ptr); 74 | ptr += object_size; 75 | } 76 | } 77 | 78 | uint64_t size() const override { 79 | uint64_t total_size = 0; 80 | for (uint64_t i = 0; i < objs.size(); ++i) { 81 | total_size += serializer.size(*objs[i]) + sizeof(uint64_t); 82 | } 83 | return total_size; 84 | } 85 | 86 | private: 87 | const cirrus::Serializer& serializer; 88 | const std::vector objs; 89 | }; 90 | 91 | } // namespace cirrus 92 | 93 | #endif // SRC_COMMON_SERIALIZER_H_ 94 | -------------------------------------------------------------------------------- /src/common/Synchronization.cpp: -------------------------------------------------------------------------------- 1 | #include "common/Synchronization.h" 2 | 3 | namespace cirrus { 4 | 5 | SpinLock sl; 6 | 7 | } // namespace cirrus 8 | -------------------------------------------------------------------------------- /src/common/ThreadPinning.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_THREADPINNING_H_ 2 | #define SRC_COMMON_THREADPINNING_H_ 3 | 4 | #include 5 | #include 6 | #include "utils/logging.h" 7 | 8 | namespace cirrus { 9 | 10 | class ThreadPinning { 11 | public: 12 | static void pinThread( 13 | std::thread::native_handle_type thread_handle, int cpu) { 14 | cpu_set_t cpuset; 15 | CPU_ZERO(&cpuset); 16 | CPU_SET(cpu, &cpuset); 17 | 18 | int ret = pthread_setaffinity_np(thread_handle, 19 | sizeof(cpu_set_t), &cpuset); 20 | if (ret != 0) { 21 | LOG("Error calling pthread_setaffinity_np: ", ret); 22 | } 23 | } 24 | }; 25 | 26 | } // namespace cirrus 27 | 28 | #endif // SRC_COMMON_THREADPINNING_H_ 29 | -------------------------------------------------------------------------------- /src/common/config.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COMMON_CONFIG_H_ 2 | #define SRC_COMMON_CONFIG_H_ 3 | 4 | #include "utils/logging.h" 5 | 6 | #endif // SRC_COMMON_CONFIG_H_ 7 | -------------------------------------------------------------------------------- /src/common/schemas/.gitignore: -------------------------------------------------------------------------------- 1 | *_generated.h 2 | -------------------------------------------------------------------------------- /src/common/schemas/AllocatorMessage.fbs: -------------------------------------------------------------------------------- 1 | namespace cirrus.message.AllocatorMessage; 2 | 3 | union Data { Auth1, AuthAck1, Auth2, AuthAck2, Stats, StatsAck, Alloc, AllocAck } 4 | 5 | table Auth1{ 6 | app_id:ulong; 7 | } 8 | 9 | table AuthAck1{ 10 | allow:byte; 11 | challenge:long; 12 | } 13 | 14 | table Auth2{ 15 | response:ulong; 16 | } 17 | 18 | table AuthAck2{ 19 | allow:byte; 20 | } 21 | 22 | table Stats{ 23 | } 24 | 25 | table StatsAck{ 26 | total_mem_allocated:ulong; 27 | } 28 | 29 | table Alloc{ 30 | } 31 | 32 | table AllocAck{ 33 | } 34 | 35 | table AllocatorMessage { 36 | id:int; 37 | data:Data; 38 | } 39 | 40 | root_type AllocatorMessage; 41 | -------------------------------------------------------------------------------- /src/common/schemas/BladeMessage.fbs: -------------------------------------------------------------------------------- 1 | namespace cirrus.message.BladeMessage; 2 | 3 | union Data { Alloc, AllocAck, Dealloc, DeallocAck } 4 | 5 | table Alloc{ 6 | size:ulong; 7 | } 8 | 9 | table AllocAck{ 10 | mr_id:ulong; 11 | remote_addr:ulong; 12 | peer_rkey:ulong; 13 | } 14 | 15 | table Dealloc{ 16 | addr:ulong; 17 | } 18 | 19 | table DeallocAck{ 20 | result:byte; 21 | } 22 | 23 | table BladeMessage { 24 | data:Data; 25 | } 26 | 27 | root_type BladeMessage; 28 | -------------------------------------------------------------------------------- /src/common/schemas/BladeObjectStoreMessage.fbs: -------------------------------------------------------------------------------- 1 | namespace cirrus.message.BladeObjectStoreMessage; 2 | 3 | union Data { Alloc, AllocAck, Dealloc, DeallocAck, KeepAlive, KeepAliveAck, Sub, SubAck, Flush, FlushAck, Lock, LockAck } 4 | 5 | table Alloc{ 6 | size:ulong; 7 | } 8 | 9 | table AllocAck{ 10 | mr_id:ulong; 11 | remote_addr:ulong; 12 | peer_rkey:ulong; 13 | } 14 | 15 | table Dealloc{ 16 | addr:ulong; 17 | } 18 | 19 | table DeallocAck{ 20 | result:byte; 21 | } 22 | 23 | table KeepAlive{ 24 | rand:ulong; 25 | } 26 | 27 | table KeepAliveAck{ 28 | rand:ulong; 29 | } 30 | 31 | table Sub{ 32 | oid:ulong; 33 | addr:string; //20 char vector? 34 | } 35 | 36 | table SubAck{ 37 | oid:ulong; 38 | } 39 | 40 | table Flush{ 41 | oid:ulong; 42 | } 43 | 44 | table FlushAck{ 45 | oid:ulong; 46 | } 47 | 48 | table Lock{ 49 | id:ulong; 50 | } 51 | 52 | table LockAck{ 53 | id:ulong; 54 | } 55 | 56 | table BladeObjectStoreMessage { 57 | data:Data; 58 | } 59 | 60 | root_type BladeObjectStoreMessage; 61 | -------------------------------------------------------------------------------- /src/common/schemas/Makefile.am: -------------------------------------------------------------------------------- 1 | flatc = $(top_srcdir)/third_party/flatbuffers/flatc 2 | 3 | all-local: 4 | $(flatc) --cpp *.fbs 5 | clean-local: 6 | rm -f *.h 7 | -------------------------------------------------------------------------------- /src/common/schemas/TCPBladeMessage.fbs: -------------------------------------------------------------------------------- 1 | namespace cirrus.message.TCPBladeMessage; 2 | 3 | union Message { Write, WriteAck, WriteBulk, WriteBulkAck, Read, ReadAck, ReadBulk, ReadBulkAck, Remove, RemoveAck } 4 | 5 | table Write{ 6 | oid:ulong; 7 | data:[byte]; 8 | } 9 | 10 | table WriteAck{ 11 | oid:ulong; 12 | success:byte; 13 | } 14 | 15 | table WriteBulk{ 16 | num_oids:ulong; 17 | oids:[ulong]; 18 | data:[byte]; 19 | } 20 | 21 | table WriteBulkAck{ 22 | success:byte; 23 | } 24 | 25 | table Read{ 26 | oid:ulong; 27 | } 28 | 29 | table ReadAck{ 30 | oid:ulong; 31 | success:byte; 32 | data:[byte]; 33 | } 34 | 35 | table ReadBulk{ 36 | num_oids:ulong; 37 | oids:[ulong]; 38 | } 39 | 40 | table ReadBulkAck{ 41 | success:byte; 42 | data:[byte]; 43 | } 44 | 45 | table Remove{ 46 | oid:ulong; 47 | } 48 | 49 | table RemoveAck{ 50 | oid:ulong; 51 | success:byte; 52 | } 53 | 54 | table TCPBladeMessage { 55 | txnid:ulong; 56 | error_code:long; 57 | message:Message (required); 58 | } 59 | 60 | root_type TCPBladeMessage; 61 | -------------------------------------------------------------------------------- /src/coordination/LockManager.cpp: -------------------------------------------------------------------------------- 1 | #include "coordination/LockManager.h" 2 | 3 | namespace cirrus { 4 | 5 | LockManager::LockManager() { 6 | } 7 | 8 | } // namespace cirrus 9 | -------------------------------------------------------------------------------- /src/coordination/LockManager.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_COORDINATION_LOCKMANAGER_H_ 2 | #define SRC_COORDINATION_LOCKMANAGER_H_ 3 | 4 | #include "coordination/LockManager.h" 5 | 6 | namespace cirrus { 7 | 8 | class LockManager { 9 | LockManager::LockManager(); 10 | }; 11 | 12 | } // namespace cirrus 13 | 14 | #endif // SRC_COORDINATION_LOCKMANAGER_H_ 15 | -------------------------------------------------------------------------------- /src/iterator/IteratorPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_ITERATOR_ITERATORPOLICY_H_ 2 | #define SRC_ITERATOR_ITERATORPOLICY_H_ 3 | 4 | #include 5 | #include 6 | #include "iterator/CirrusIterable.h" 7 | namespace cirrus { 8 | 9 | using ObjectID = uint64_t; 10 | 11 | /** 12 | * A class that outlines the interface that PrefetchPolicies for the iterator 13 | * should use. 14 | */ 15 | class IteratorPolicy { 16 | public: 17 | enum Position { 18 | kBegin = 0, 19 | kEnd 20 | }; 21 | 22 | virtual void setState(ObjectID first, ObjectID last, uint64_t read_ahead, 23 | Position position) = 0; 24 | /** 25 | * Called by the Iterator whenever it dereferences an object. 26 | * @param oid the ObjectID being dereferenced. 27 | * @param it a reference to the iterator that is making the call. 28 | * @return a std::vector of ObjectIDs that should be prefetched. 29 | */ 30 | virtual std::vector getPrefetchList() = 0; 31 | 32 | virtual ObjectID dereference() = 0; 33 | 34 | virtual void increment() = 0; 35 | 36 | virtual uint64_t getState() const = 0; 37 | 38 | virtual std::unique_ptr clone() const = 0; 39 | }; 40 | 41 | } // namespace cirrus 42 | 43 | #endif // SRC_ITERATOR_ITERATORPOLICY_H_ 44 | -------------------------------------------------------------------------------- /src/iterator/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | -------------------------------------------------------------------------------- /src/memories/3DXPoint.cpp: -------------------------------------------------------------------------------- 1 | #include "memories/3DXPoint.h" 2 | 3 | namespace cirrus { 4 | 5 | 3DXPoint::getWriteBandwidth() { 6 | return write_bandwidth_; 7 | } 8 | 9 | 3DXPoint::getReadBandwidth() { 10 | return read_bandwidth_; 11 | } 12 | 13 | 3DXPoint::getReadLatency() { 14 | return read_latency_us_; 15 | } 16 | 17 | 3DXPoint::getWriteLatency() { 18 | return write_latency_us_; 19 | } 20 | 21 | } // namespace cirrus 22 | -------------------------------------------------------------------------------- /src/memories/3DXPoint.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_MEMORIES_3DXPOINT_H_ 2 | #define SRC_MEMORIES_3DXPOINT_H_ 3 | 4 | namespace cirrus { 5 | 6 | class 3DXPoint : public Memory { 7 | public: 8 | uint64_t getReadLatency() const {} 9 | uint64_t getWriteLatency() const {} 10 | uint64_t getReadBandwidth() const {} 11 | uint64_t getWriteBandwidth() const {} 12 | private: 13 | const uint64_t GB = 1024*1024*1024; 14 | const uint64_t read_latency_us_ = 10; 15 | const uint64_t write_latency_us_ = 10; 16 | const uint64_t read_bandwidth_ = 10 * GB; 17 | const uint64_t write_bandwidth_ = 10 GB; 18 | }; 19 | 20 | } // namespace cirrus 21 | 22 | #endif // SRC_MEMORIES_3DXPOINT_H_ 23 | -------------------------------------------------------------------------------- /src/memories/Memory.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_MEMORIES_MEMORY_H_ 2 | #define SRC_MEMORIES_MEMORY_H_ 3 | 4 | namespace cirrus { 5 | 6 | struct Memory { 7 | virtual uint64_t getReadLatency() const = 0; 8 | virtual uint64_t getWriteLatency() const = 0; 9 | virtual uint64_t getReadBandwidth() const = 0; 10 | virtual uint64_t getWriteBandwidth() const = 0; 11 | }; 12 | 13 | } // namespace cirrus 14 | 15 | #endif // SRC_MEMORIES_MEMORY_H_ 16 | -------------------------------------------------------------------------------- /src/object_store/EvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "object_store/EvictionPolicy.h" 2 | 3 | namespace cirrus { 4 | 5 | EvictionPolicy::EvictionPolicy(size_t max_num_objs) : 6 | max_num_objs_(max_num_objs) { 7 | } 8 | 9 | } // namespace cirrus 10 | -------------------------------------------------------------------------------- /src/object_store/EvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_OBJECT_STORE_EVICTIONPOLICY_H_ 2 | #define SRC_OBJECT_STORE_EVICTIONPOLICY_H_ 3 | 4 | #include 5 | 6 | #include "object_store/FullCacheStore.h" 7 | 8 | 9 | namespace cirrus { 10 | 11 | class EvictionPolicy { 12 | public: 13 | explicit EvictionPolicy(size_t max_num_objs); 14 | virtual ~EvictionPolicy() = default; 15 | 16 | virtual bool evictIfNeeded(FullCacheStore *fc) = 0; 17 | 18 | protected: 19 | size_t max_num_objs_; /** Max number of objects in the store */ 20 | }; 21 | 22 | } // namespace cirrus 23 | 24 | #endif // SRC_OBJECT_STORE_EVICTIONPOLICY_H_ 25 | -------------------------------------------------------------------------------- /src/object_store/FullCacheStore.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "object_store/FullCacheStore.h" 4 | #include "utils/utils.h" 5 | 6 | namespace cirrus { 7 | 8 | FullCacheStore::FullCacheStore() : 9 | ObjectStore() { 10 | #ifdef GOOGLE 11 | objects_.set_empty_key(ObjectID()); 12 | sem_init(&hash_sem, 0, 1); 13 | #elif !defined(CUCKOO) 14 | objects_ = new void*[CACHE_SIZE]; 15 | std::memset(objects_, 0, CACHE_SIZE * sizeof(void*)); 16 | #endif 17 | } 18 | 19 | Object FullCacheStore::get(ObjectID name) { 20 | #if !defined(GOOGLE) && !defined(CUCKOO) 21 | return objects_[name]; 22 | #else 23 | 24 | exit(-1); 25 | 26 | Object ret; 27 | #ifdef GOOGLE 28 | google::dense_hash_map::iterator it; 29 | sem_wait(&hash_sem); 30 | it = objects_.find(name); 31 | if (it == objects_.end()) { 32 | sem_post(&hash_sem); 33 | #elif defined(CUCKOO) 34 | bool found = objects_.find(name, ret); 35 | if (!found) { 36 | #endif 37 | return 0; 38 | } else { 39 | #ifdef GOOGLE 40 | ret = it->second; 41 | sem_post(&hash_sem); 42 | #endif 43 | return ret; 44 | } 45 | 46 | #endif 47 | } 48 | 49 | bool FullCacheStore::put(Object obj, uint64_t size, ObjectID name) { 50 | // right now we use C++ allocator 51 | // later I may use my own thing 52 | void *mem = 0; 53 | 54 | #ifdef GOOGLE 55 | sem_wait(&hash_sem); 56 | google::dense_hash_map::iterator it; 57 | it = objects_.find(name); 58 | if (it == objects_.end()) { 59 | #elif defined(CUCKOO) 60 | bool found = objects_.find(name, mem); 61 | if (!found) { 62 | #else 63 | if (!(mem = objects_[name])) { 64 | #endif 65 | mem = new char[size]; 66 | if (!mem) 67 | DIE("Error allocating cache memory"); 68 | objects_[name] = mem; 69 | } else { 70 | #ifdef GOOGLE 71 | mem = it->second; 72 | #else 73 | #endif 74 | } 75 | 76 | #ifdef GOOGLE 77 | sem_post(&hash_sem); 78 | #endif 79 | 80 | std::memcpy(mem, obj, size); 81 | 82 | return true; 83 | } 84 | 85 | void FullCacheStore::printStats() { 86 | #if defined(GOOGLE) || defined(CUCKOO) 87 | std::cout << "Cache size: " 88 | << objects_.size() << std::endl; 89 | #endif 90 | 91 | #ifdef GOOGLE 92 | std::cout << "Max size: " 93 | << objects_.max_size() << std::endl; 94 | std::cout << "Max bucket count: " 95 | << objects_.max_bucket_count() << std::endl; 96 | #endif 97 | } 98 | 99 | } // namespace cirrus 100 | -------------------------------------------------------------------------------- /src/object_store/LRUEvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "object_store/LRUEvictionPolicy.h" 2 | #include "object_store/FullCacheStore.h" 3 | 4 | namespace cirrus { 5 | 6 | LRUEvictionPolicy::LRUEvictionPolicy(size_t num_objs) : 7 | EvictionPolicy(num_objs) { 8 | } 9 | 10 | LRUEvictionPolicy::~LRUEvictionPolicy() { 11 | } 12 | 13 | bool LRUEvictionPolicy::evictIfNeeded(FullCacheStore& fc) { 14 | if (fc->getNumObjs() > max_num_objs_) { 15 | fc->dropLRUObj(); 16 | return true; 17 | } 18 | return false; 19 | } 20 | 21 | } // namespace cirrus 22 | -------------------------------------------------------------------------------- /src/object_store/LRUEvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_OBJECT_STORE_LRUEVICTIONPOLICY_H_ 2 | #define SRC_OBJECT_STORE_LRUEVICTIONPOLICY_H_ 3 | 4 | #include "object_store/EvictionPolicy.h" 5 | 6 | namespace cirrus { 7 | 8 | class LRUEvictionPolicy : public EvictionPolicy { 9 | public: 10 | explicit LRUEvictionPolicy(size_t num_objs = DEFAULT_SIZE); 11 | virtual ~LRUEvictionPolicy(); 12 | 13 | virtual bool evictIfNeeded(FullCacheStore *fc); 14 | 15 | private: 16 | static const size_t DEFAULT_SIZE = 1000; 17 | }; 18 | 19 | } // namespace cirrus 20 | 21 | #endif // SRC_OBJECT_STORE_LRUEVICTIONPOLICY_H_ 22 | -------------------------------------------------------------------------------- /src/object_store/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | -------------------------------------------------------------------------------- /src/object_store/ObjectStore.cpp: -------------------------------------------------------------------------------- 1 | #include "object_store/ObjectStore.h" 2 | 3 | namespace cirrus { 4 | 5 | ObjectStore::ObjectStore() { 6 | } 7 | 8 | } // namespace cirrus 9 | -------------------------------------------------------------------------------- /src/object_store/RDMAObjectStore.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "object_store/RDMAObjectStore.h" 4 | #include "object_store/RandomEvictionPolicy.h" 5 | #include "utils/utils.h" 6 | 7 | namespace cirrus { 8 | 9 | template 10 | RDMAObjectStore::RDMAObjectStore(const std::string& blade_addr, 11 | const std::string& port, 12 | EvictionPolicy* ev = new RandomEvictionPolicy()) : 13 | ObjectStore(), 14 | ep(ev) { 15 | client.connect(blade_addr, port); 16 | } 17 | 18 | template 19 | Object RDMAObjectStore::get(ObjectID name) { 20 | Object obj = cache_.get(name); 21 | if (obj) { 22 | return obj; 23 | } else { 24 | std::cout << "Object not found. name: " << name << std::endl; 25 | DIE("DIE"); 26 | return 0; 27 | } 28 | } 29 | 30 | bool RDMAObjectStore::put(Object obj, uint64_t size, ObjectID name) { 31 | ep->evictIfNeeded(cache_); 32 | return cache_.put(obj, size, name); 33 | } 34 | 35 | template 36 | typename RDMAObjectStore::ObjectStoreGetFuture ObjectStore::get_async( 37 | const ObjectID& id) { 38 | throw std::runtime_error("Function not implemented."); 39 | } 40 | 41 | template 42 | typename RDMAObjectStore::ObjectStorePutFuture ObjectStore::put_async( 43 | const ObjectID& id, const T& obj) { 44 | throw std::runtime_error("Function not implemented."); 45 | } 46 | 47 | template 48 | void RDMAObjectStore::put_bulk(ObjectID /* start */, 49 | ObjectID /* last */, T* /* data */) { 50 | throw std::runtime_error("Function not implemented."); 51 | } 52 | 53 | template 54 | void RDMAObjectStore::get_bulk(ObjectID /* start */, 55 | ObjectID /* last */, T* /* data */) { 56 | throw std::runtime_error("Function not implemented."); 57 | } 58 | 59 | 60 | void RDMAObjectStore::printStats() { 61 | std::cout << "ObjectStore stats" << std::endl; 62 | std::cout << "Cache stats:" << std::endl; 63 | cache_.printStats(); 64 | } 65 | 66 | } // namespace cirrus 67 | 68 | -------------------------------------------------------------------------------- /src/object_store/RDMAObjectStore.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_OBJECT_STORE_RDMAOBJECTSTORE_H_ 2 | #define SRC_OBJECT_STORE_RDMAOBJECTSTORE_H_ 3 | 4 | #include 5 | 6 | #include "object_store/ObjectStore.h" 7 | #include "object_store/FullCacheStore.h" 8 | #include "object_store/EvictionPolicy.h" 9 | #include "client/BladeClient.h" 10 | 11 | namespace cirrus { 12 | 13 | template 14 | class RDMAObjectStore : public ObjectStore { 15 | public: 16 | RDMAObjectStore(const std::string& blade_addr, 17 | const std::string& port, EvictionPolicy* ev); 18 | virtual ~RDMAObjectStore() = default; 19 | 20 | Object get(ObjectID) override; 21 | bool put(Object, uint64_t, ObjectID) override; 22 | ObjectStoreGetFuture get_async(const ObjectID& id) override; 23 | ObjectStorePutFuture put_async(const ObjectID& id, const T& obj) override; 24 | 25 | void get_bulk(ObjectID start, ObjectID last, T* data) override; 26 | void put_bulk(ObjectID start, ObjectID last, T* data) override; 27 | void printStats() override; 28 | 29 | private: 30 | std::unique_ptr ep; 31 | // cache of objects 32 | mutable FullCacheStore cache_; 33 | BladeClient client; 34 | }; 35 | 36 | } // namespace cirrus 37 | 38 | #endif // SRC_OBJECT_STORE_RDMAOBJECTSTORE_H_ 39 | -------------------------------------------------------------------------------- /src/object_store/RandomEvictionPolicy.cpp: -------------------------------------------------------------------------------- 1 | #include "object_store/RandomEvictionPolicy.h" 2 | #include "object_store/FullCacheStore.h" 3 | 4 | namespace cirrus { 5 | 6 | RandomEvictionPolicy::RandomEvictionPolicy(size_t num_objs) : 7 | EvictionPolicy(num_objs) { 8 | } 9 | 10 | bool RandomEvictionPolicy::evictIfNeeded(FullCacheStore& fc) { 11 | if (fc->getNumObjs() > max_num_objs_) { 12 | fc->dropRandomObj(); 13 | return true; 14 | } 15 | return false; 16 | } 17 | 18 | } // namespace cirrus 19 | -------------------------------------------------------------------------------- /src/object_store/RandomEvictionPolicy.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_OBJECT_STORE_RANDOMEVICTIONPOLICY_H_ 2 | #define SRC_OBJECT_STORE_RANDOMEVICTIONPOLICY_H_ 3 | 4 | #include "object_store/EvictionPolicy.h" 5 | 6 | namespace cirrus { 7 | 8 | class RandomEvictionPolicy : public EvictionPolicy { 9 | public: 10 | explicit RandomEvictionPolicy(size_t num_objs = DEFAULT_SIZE); 11 | virtual ~RandomEvictionPolicy() = default; 12 | 13 | virtual bool evictIfNeeded(FullCacheStore *fc); 14 | 15 | private: 16 | static const size_t DEFAULT_SIZE = 1000; 17 | }; 18 | 19 | } // namespace cirrus 20 | 21 | #endif // SRC_OBJECT_STORE_RANDOMEVICTIONPOLICY_H_ 22 | -------------------------------------------------------------------------------- /src/server/.gitignore: -------------------------------------------------------------------------------- 1 | allocmain 2 | bladeallocmain 3 | blademain 4 | bladepoolmain 5 | controllermain 6 | bladefilemain 7 | objectstoremain 8 | tcpservermain 9 | -------------------------------------------------------------------------------- /src/server/Allocation.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_ALLOCATION_H_ 2 | #define SRC_SERVER_ALLOCATION_H_ 3 | 4 | 5 | 6 | namespace cirrus { 7 | /** 8 | * This base class describes a resource 9 | * reserved by one app. 10 | */ 11 | class Allocation { 12 | public: 13 | Allocation() {} 14 | virtual ~Allocation() {} 15 | private: 16 | }; 17 | 18 | } // namespace cirrus 19 | 20 | #endif // SRC_SERVER_ALLOCATION_H_ 21 | -------------------------------------------------------------------------------- /src/server/ApplicationRecord.cpp: -------------------------------------------------------------------------------- 1 | #include "server/ApplicationRecord.h" 2 | 3 | namespace cirrus { 4 | 5 | ApplicationRecord::ApplicationRecord(uint64_t app_id) : 6 | app_id_(app_id) { 7 | } 8 | 9 | ApplicationRecord::~ApplicationRecord() { 10 | } 11 | 12 | void ApplicationRecord::addAllocation(const Allocation& alloc) { 13 | allocations_.push_back(alloc); 14 | } 15 | 16 | } // namespace cirrus 17 | -------------------------------------------------------------------------------- /src/server/ApplicationRecord.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_APPLICATIONRECORD_H_ 2 | #define SRC_SERVER_APPLICATIONRECORD_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | class ApplicationRecord { 9 | public: 10 | ApplicationRecord(); 11 | virtual ~ApplicationRecord(); 12 | 13 | void addAllocation(const Allocation& alloc); 14 | 15 | private: 16 | uint64_t app_id_; 17 | std::vector allocations_; 18 | }; 19 | 20 | } // namespace cirrus 21 | 22 | #endif // SRC_SERVER_APPLICATIONRECORD_H_ 23 | -------------------------------------------------------------------------------- /src/server/BladeAllocServer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_BLADEALLOCSERVER_H_ 2 | #define SRC_SERVER_BLADEALLOCSERVER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "utils/utils.h" 11 | #include "RDMAServer.h" 12 | 13 | #include 14 | 15 | namespace cirrus { 16 | 17 | struct BladeAllocation { 18 | explicit BladeAllocation(void* p = 0) : ptr(p) {} 19 | void* ptr; 20 | }; 21 | 22 | /** 23 | * This server supports allocations on top of a big mem pool. 24 | * Allocations are isolated through the use of mem. windows 25 | * We use a mem. allocator to manage mem. 26 | * @param port the port the server will listen on 27 | * @param pool_size the number of bytes in the memory pool 28 | */ 29 | class BladeAllocServer : public RDMAServer { 30 | public: 31 | BladeAllocServer(int port, uint64_t pool_size, 32 | int timeout_ms = 500); 33 | virtual ~BladeAllocServer() = default; 34 | void init() final; 35 | 36 | private: 37 | void process_message(rdma_cm_id*, void* msg) final; 38 | void handle_connection(struct rdma_cm_id* id) final; 39 | void handle_disconnection(struct rdma_cm_id* id) final; 40 | 41 | bool create_pool(uint64_t size); 42 | 43 | void allocate_mem(rdma_cm_id* id, uint64_t size, const void* ptr, 44 | const uint32_t& rkey); 45 | void* find_free_data(uint64_t size); 46 | 47 | // mr and pointer to big pool 48 | ibv_mr* big_pool_mr_; 49 | void* big_pool_data_; 50 | uint64_t big_pool_size_; 51 | std::once_flag pool_flag_; 52 | 53 | // mrs and pointers to allocations 54 | // from clients 55 | // std::set mrs_; 56 | std::set mrs_data_; 57 | 58 | std::map id_to_alloc_; 59 | uint64_t num_allocs_; 60 | uint64_t mem_allocated; 61 | 62 | std::unique_ptr allocator; 63 | }; 64 | 65 | } // namespace cirrus 66 | 67 | #endif // SRC_SERVER_BLADEALLOCSERVER_H_ 68 | -------------------------------------------------------------------------------- /src/server/BladeObjectStore.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_BLADEOBJECTSTORE_H_ 2 | #define SRC_SERVER_BLADEOBJECTSTORE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "utils/utils.h" 11 | #include "RDMAServer.h" 12 | #include 13 | 14 | namespace cirrus { 15 | 16 | struct BladeAllocation { 17 | explicit BladeAllocation(void* p = 0) : ptr(p) {} 18 | void* ptr; 19 | }; 20 | 21 | class BladeObjectStore : public RDMAServer { 22 | public: 23 | BladeObjectStore(int port, uint64_t pool_size, 24 | int timeout_ms = 500); 25 | virtual ~BladeObjectStore() = default; 26 | void init() final; 27 | 28 | private: 29 | void process_message(rdma_cm_id*, void* msg) final; 30 | void handle_connection(struct rdma_cm_id* id) final; 31 | void handle_disconnection(struct rdma_cm_id* id) final; 32 | 33 | bool create_pool(uint64_t size); 34 | 35 | void allocate_mem(rdma_cm_id* id, uint64_t size, const void* ptr, 36 | const uint32_t& rkey); 37 | void* find_free_data(uint64_t size); 38 | 39 | void checkpoint(); 40 | 41 | // mr and pointer to big pool 42 | ibv_mr* big_pool_mr_; 43 | void* big_pool_data_; 44 | uint64_t big_pool_size_; 45 | std::once_flag pool_flag_; 46 | 47 | std::set mrs_data_; 48 | 49 | std::map id_to_alloc_; 50 | uint64_t num_allocs_; 51 | uint64_t mem_allocated; 52 | 53 | std::unique_ptr allocator; 54 | }; 55 | 56 | } // namespace cirrus 57 | 58 | #endif // SRC_SERVER_BLADEOBJECTSTORE_H_ 59 | -------------------------------------------------------------------------------- /src/server/BladePoolServer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_BLADEPOOLSERVER_H_ 2 | #define SRC_SERVER_BLADEPOOLSERVER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils/utils.h" 9 | #include "RDMAServer.h" 10 | 11 | /* 12 | * This server only creates one big pool 13 | * Allocations here always return a pointer 14 | * to that big pool 15 | */ 16 | 17 | namespace cirrus { 18 | 19 | class BladePoolServer : public RDMAServer { 20 | public: 21 | BladePoolServer(int port, uint64_t pool_size, 22 | int timeout_ms = 500); 23 | virtual ~BladePoolServer() = default; 24 | void init() final; 25 | 26 | private: 27 | void process_message(rdma_cm_id*, void* msg) final; 28 | void handle_connection(struct rdma_cm_id* id) final; 29 | void handle_disconnection(struct rdma_cm_id* id) final; 30 | 31 | uint32_t create_pool(uint64_t size, struct rdma_cm_id*); 32 | 33 | std::vector mr_data_; 34 | std::vector mr_pool_; 35 | uint64_t pool_size_; 36 | 37 | std::once_flag pool_flag_; 38 | }; 39 | 40 | } // namespace cirrus 41 | 42 | #endif // SRC_SERVER_BLADEPOOLSERVER_H_ 43 | -------------------------------------------------------------------------------- /src/server/ConnectionContext.cpp: -------------------------------------------------------------------------------- 1 | #include "server/ConnectionContext.h" 2 | #include "utils/utils.h" 3 | 4 | namespace cirrus { 5 | 6 | uint64_t ConnectionContext::id_count_ = 0; 7 | 8 | ConnectionContext::ConnectionContext() : 9 | recv_msg(nullptr), 10 | recv_msg_mr(nullptr), 11 | send_msg(nullptr), 12 | send_msg_mr(nullptr), 13 | server(nullptr), 14 | info(0), 15 | context_id_(id_count_++) { 16 | } 17 | 18 | } // namespace cirrus 19 | -------------------------------------------------------------------------------- /src/server/ConnectionContext.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_CONNECTIONCONTEXT_H_ 2 | #define SRC_SERVER_CONNECTIONCONTEXT_H_ 3 | 4 | #include 5 | #include 6 | #include "server/ConnectionInfo.h" 7 | 8 | namespace cirrus { 9 | 10 | class RDMAServer; 11 | 12 | class ConnectionContext { 13 | public: 14 | ConnectionContext(); 15 | virtual ~ConnectionContext() = default; 16 | 17 | // buffer and memory region for receiving messages 18 | void* recv_msg; 19 | ibv_mr* recv_msg_mr; 20 | 21 | // buffer and memory region to send messages 22 | void* send_msg; 23 | ibv_mr* send_msg_mr; 24 | 25 | // sem_t ack_sem; 26 | 27 | // pointer to RDMAServer responsible 28 | // for handling events on this connection 29 | RDMAServer* server; 30 | 31 | // info about connection 32 | ConnectionInfo* info; 33 | 34 | // we need a way to distinguish connection contexts 35 | // an integer is handy 36 | int context_id_; 37 | static uint64_t id_count_; 38 | }; 39 | 40 | } // namespace cirrus 41 | 42 | #endif // SRC_SERVER_CONNECTIONCONTEXT_H_ 43 | -------------------------------------------------------------------------------- /src/server/ConnectionInfo.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_CONNECTIONINFO_H_ 2 | #define SRC_SERVER_CONNECTIONINFO_H_ 3 | 4 | namespace cirrus { 5 | 6 | class ConnectionInfo { 7 | public: 8 | ConnectionInfo() {} 9 | 10 | private: 11 | }; 12 | 13 | } // namespace cirrus 14 | 15 | #endif // SRC_SERVER_CONNECTIONINFO_H_ 16 | -------------------------------------------------------------------------------- /src/server/GeneralContext.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_GENERALCONTEXT_H_ 2 | #define SRC_SERVER_GENERALCONTEXT_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | class GeneralContext { 9 | public: 10 | GeneralContext() : 11 | ctx(0), 12 | pd(0), 13 | cq(0), 14 | comp_channel(0), 15 | cq_poller_thread(0) {} 16 | 17 | struct ibv_context *ctx; 18 | struct ibv_pd *pd; 19 | struct ibv_cq *cq; 20 | struct ibv_comp_channel *comp_channel; 21 | std::thread* cq_poller_thread; 22 | }; 23 | 24 | } // namespace cirrus 25 | 26 | #endif // SRC_SERVER_GENERALCONTEXT_H_ 27 | -------------------------------------------------------------------------------- /src/server/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | DEFAULT_LIB = -L. -lserver \ 6 | -L../authentication/ -lauthentication \ 7 | -L../utils/ -lutils \ 8 | -L../../third_party/rocksdb/ -lrocksdb -lsnappy -lbz2 -lz \ 9 | -L../common/ -lcommon $(LIBRDMACM) $(LIBIBVERBS) 10 | 11 | noinst_LIBRARIES = libserver.a 12 | bin_PROGRAMS = tcpservermain 13 | 14 | libserver_a_SOURCES = TCPServer.cpp MemoryBackend.cpp \ 15 | MemoryBackend.cpp NVStorageBackend.cpp 16 | libserver_a_CPPFLAGS = -ggdb -I$(top_srcdir) \ 17 | -I$(top_srcdir)/third_party/flatbuffers/include \ 18 | -isystem $(top_srcdir)/third_party/rocksdb/include \ 19 | -I$(top_srcdir)/src 20 | 21 | if USE_RDMA 22 | bin_PROGRAMS += bladepoolmain bladeallocmain allocmain controllermain \ 23 | objectstoremain 24 | libserver_a_SOURCES += RDMAServer.cpp BladePoolServer.cpp \ 25 | ConnectionContext.cpp \ 26 | ResourceAllocator.cpp BladeAllocServer.cpp \ 27 | BladeObjectStore.cpp 28 | endif 29 | 30 | LDFLAGS = -pthread 31 | LDADD = $(DEFAULT_LIB) 32 | CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src \ 33 | -I$(top_srcdir)/third_party/flatbuffers/include 34 | 35 | bladepoolmain_DEPENDENCIES = libserver.a 36 | bladepoolmain_SOURCES = bladepoolmain.cpp 37 | 38 | bladeallocmain_DEPENDENCIES = libserver.a 39 | bladeallocmain_SOURCES = bladeallocmain.cpp 40 | 41 | allocmain_DEPENDENCIES = libserver.a 42 | allocmain_SOURCES = allocmain.cpp 43 | 44 | controllermain_DEPENDENCIES = libserver.a 45 | controllermain_SOURCES = controllermain.cpp 46 | 47 | objectstoremain_DEPENDENCIES = libserver.a 48 | objectstoremain_SOURCES = objectstoremain.cpp 49 | 50 | tcpservermain_DEPENDENCIES = libserver.a 51 | tcpservermain_SOURCES = TCPServerMain.cpp 52 | -------------------------------------------------------------------------------- /src/server/MemoryBackend.cpp: -------------------------------------------------------------------------------- 1 | #include "MemoryBackend.h" 2 | 3 | #include 4 | #include 5 | #include "server/MemoryBackend.h" 6 | 7 | namespace cirrus { 8 | 9 | MemoryBackend::MemoryBackend(uint64_t bytes) : remove_increment(bytes) {} 10 | 11 | void MemoryBackend::init() {} 12 | 13 | bool MemoryBackend::put(uint64_t oid, const MemSlice& data) { 14 | store[oid] = data.get(); 15 | return true; 16 | } 17 | 18 | bool MemoryBackend::exists(uint64_t oid) const { 19 | return store.find(oid) != store.end(); 20 | } 21 | 22 | MemSlice MemoryBackend::get(uint64_t oid) const { 23 | auto it = store.find(oid); 24 | if (it == store.end()) { 25 | throw std::runtime_error( 26 | "MemoryBackend get() called on nonexistent id"); 27 | } 28 | 29 | return &it->second; 30 | } 31 | 32 | bool MemoryBackend::delet(uint64_t oid) { 33 | // we assume object exists 34 | remove_counter += store[oid].size(); 35 | store.erase(oid); 36 | if (remove_counter >= remove_increment) { 37 | remove_counter %= remove_increment; 38 | malloc_trim(0); 39 | } 40 | return true; 41 | } 42 | 43 | uint64_t MemoryBackend::size(uint64_t oid) const { 44 | auto it = store.find(oid); 45 | if (it == store.end()) { 46 | return 0; 47 | } 48 | 49 | return it->second.size(); 50 | } 51 | 52 | } // namespace cirrus 53 | -------------------------------------------------------------------------------- /src/server/MemoryBackend.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_MEMORYBACKEND_H_ 2 | #define SRC_SERVER_MEMORYBACKEND_H_ 3 | 4 | #include "StorageBackend.h" 5 | #include 6 | #include 7 | 8 | namespace cirrus { 9 | 10 | /** 11 | * This is a key-value store backed by memory 12 | * It's just a wrapper around a std::map or something similar 13 | */ 14 | class MemoryBackend : public StorageBackend { 15 | public: 16 | MemoryBackend() = default; 17 | MemoryBackend(uint64_t bytes); 18 | virtual ~MemoryBackend() = default; 19 | 20 | void init(); 21 | bool put(uint64_t oid, const MemSlice& data) override; 22 | bool exists(uint64_t oid) const override; 23 | MemSlice get(uint64_t oid) const override; 24 | bool delet(uint64_t oid) override; 25 | uint64_t size(uint64_t oid) const override; 26 | 27 | private: 28 | // make this mutable because std::map 29 | // doesn't play well with const 30 | mutable std::unordered_map> store; 31 | mutable uint64_t remove_counter = 0; 32 | mutable uint64_t remove_increment = 1'000'000; 33 | }; 34 | 35 | } // namespace cirrus 36 | 37 | #endif // SRC_SERVER_MEMORYBACKEND_H_ 38 | -------------------------------------------------------------------------------- /src/server/NVStorageBackend.cpp: -------------------------------------------------------------------------------- 1 | #include "src/server/NVStorageBackend.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "utils/logging.h" 7 | 8 | /** 9 | * Interface over RocksDB 10 | */ 11 | namespace cirrus { 12 | 13 | NVStorageBackend::NVStorageBackend(const std::string& path) : 14 | path(path) { 15 | } 16 | 17 | void NVStorageBackend::init() { 18 | options.IncreaseParallelism(); 19 | options.OptimizeLevelStyleCompaction(); 20 | 21 | options.create_if_missing = true; 22 | 23 | // open DB 24 | rocksdb::Status s = rocksdb::DB::Open(options, path, &db); 25 | if (!s.ok()) { 26 | throw std::runtime_error("Error opening rocksdb"); 27 | } 28 | LOG("Opened rocksdb in path: ", path); 29 | } 30 | 31 | bool NVStorageBackend::put(uint64_t oid, const MemSlice& data) { 32 | LOG("Putting in rocksdb. oid: ", oid, 33 | " with size: ", data.size()); 34 | rocksdb::Status s = db->Put(rocksdb::WriteOptions(), 35 | std::to_string(oid), data.toString()); 36 | 37 | if (!s.ok()) { 38 | throw std::runtime_error("Error in put in rocksdb"); 39 | } 40 | return true; 41 | } 42 | 43 | bool NVStorageBackend::exists(uint64_t oid) const { 44 | std::string value; 45 | rocksdb::Status s = db->Get(rocksdb::ReadOptions(), 46 | std::to_string(oid), &value); 47 | 48 | LOG("Object oid: ", oid, " not found: ", s.IsNotFound()); 49 | return !s.IsNotFound(); 50 | } 51 | 52 | MemSlice NVStorageBackend::get(uint64_t oid) const { 53 | std::string value; 54 | rocksdb::Status s = db->Get(rocksdb::ReadOptions(), 55 | std::to_string(oid), &value); 56 | if (!s.ok() || s.IsNotFound()) { 57 | throw std::runtime_error("Error in get in rocksdb"); 58 | } 59 | 60 | LOG("Get in rocksdb. oid: ", oid); 61 | return MemSlice(value); 62 | } 63 | 64 | bool NVStorageBackend::delet(uint64_t oid) { 65 | // we assume object exists 66 | db->Delete(rocksdb::WriteOptions(), std::to_string(oid)); 67 | LOG("Deleted oid: ", oid); 68 | return true; 69 | } 70 | 71 | uint64_t NVStorageBackend::size(uint64_t oid) const { 72 | MemSlice obj = get(oid); 73 | LOG("Size of oid: ", oid, " is: ", obj.size()); 74 | return obj.size(); 75 | } 76 | 77 | } // namespace cirrus 78 | -------------------------------------------------------------------------------- /src/server/NVStorageBackend.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_NVSTORAGEBACKEND_H_ 2 | #define SRC_SERVER_NVSTORAGEBACKEND_H_ 3 | 4 | #include "StorageBackend.h" 5 | #include 6 | 7 | #include "rocksdb/db.h" 8 | #include "rocksdb/slice.h" 9 | #include "rocksdb/options.h" 10 | 11 | namespace cirrus { 12 | 13 | /** 14 | * This class aims to provide an interface for 15 | * non-volatile storage devices such as SSDs and 3DXPoint 16 | */ 17 | class NVStorageBackend : public StorageBackend { 18 | public: 19 | explicit NVStorageBackend(const std::string& path); 20 | 21 | void init(); 22 | bool put(uint64_t oid, const MemSlice& data) override; 23 | bool exists(uint64_t oid) const override; 24 | MemSlice get(uint64_t oid) const override; 25 | bool delet(uint64_t oid) override; 26 | uint64_t size(uint64_t oid) const override; 27 | 28 | private: 29 | std::string path; //< path to raw device 30 | 31 | rocksdb::DB* db = nullptr; //< rocksdb handler 32 | rocksdb::Options options; //< rocksdb options 33 | }; 34 | 35 | } // namespace cirrus 36 | 37 | #endif // SRC_SERVER_NVSTORAGEBACKEND_H_ 38 | -------------------------------------------------------------------------------- /src/server/QuotaManager.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_QUOTAMANAGER_H_ 2 | #define SRC_SERVER_QUOTAMANAGER_H_ 3 | 4 | #include 5 | 6 | namespace cirrus { 7 | 8 | class QuotaManager { 9 | public: 10 | QuotaManager(); 11 | 12 | virtual bool canAllocateMemory(uint64_t app_id, uint64_t memorySize) = 0;; 13 | virtual void allocateMemory(uint64_t app_id, uint64_t memorySize) = 0; 14 | }; 15 | 16 | } // namespace cirrus 17 | 18 | #endif // SRC_SERVER_QUOTAMANAGER_H_ 19 | -------------------------------------------------------------------------------- /src/server/RDMAServer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_RDMASERVER_H_ 2 | #define SRC_SERVER_RDMASERVER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "utils/utils.h" 9 | #include "server/Server.h" 10 | #include "server/GeneralContext.h" 11 | #include "ConnectionContext.h" 12 | 13 | namespace cirrus { 14 | 15 | /** 16 | * Parent class for servers that communicate over RDMA. 17 | */ 18 | class RDMAServer : public Server { 19 | public: 20 | explicit RDMAServer(int port, int timeout_ms = 500); 21 | ~RDMAServer(); 22 | 23 | // init the server 24 | virtual void init(); 25 | 26 | // get into the event loop 27 | virtual void loop(); 28 | 29 | protected: 30 | // RDMA setup functions 31 | void build_connection(struct rdma_cm_id *id); 32 | void build_params(struct rdma_conn_param *params); 33 | void build_gen_context(struct ibv_context *verbs); 34 | void build_qp_attr(struct ibv_qp_init_attr *qp_attr); 35 | 36 | void create_connection_context(struct rdma_cm_id *id); 37 | void handle_disconnected(struct rdma_cm_id *id); 38 | void handle_established(struct rdma_cm_id *id); 39 | void exit_destroy(); 40 | 41 | static void on_completion(struct ibv_wc *wc); 42 | static void* poll_cq(); 43 | static void post_msg_receive(struct rdma_cm_id *id); 44 | static void send_message(struct rdma_cm_id *id, uint64_t size); 45 | 46 | virtual void handle_connection(struct rdma_cm_id*); 47 | virtual void handle_disconnection(struct rdma_cm_id*); 48 | 49 | // leave this to specific server programs 50 | virtual void process_message(rdma_cm_id*, void* msg) = 0; 51 | 52 | // connections 53 | std::vector conns_; 54 | 55 | // structures used to setup connections 56 | struct rdma_cm_id *id_ = nullptr; 57 | struct rdma_event_channel *ec_ = nullptr; 58 | 59 | // port 60 | int port_; 61 | // timeout in miliseconds 62 | int timeout_ms_; 63 | 64 | // RDMA context for this process 65 | static GeneralContext gen_ctx_; 66 | 67 | // listen backlog 68 | int NUM_BACKLOG = 10; 69 | // size of memory pool 70 | int SIZE = 10000; 71 | 72 | static const int RECV_MSG_SIZE = 10000; 73 | static const int SEND_MSG_SIZE = 10000; 74 | // seems to be the maximum inline data 75 | static const int MAX_INLINE_DATA = 912; 76 | 77 | using msg_handler = void (RDMAServer::*)(rdma_cm_id*, void*); 78 | 79 | std::once_flag gen_ctx_flag; 80 | }; 81 | 82 | } // namespace cirrus 83 | 84 | #endif // SRC_SERVER_RDMASERVER_H_ 85 | -------------------------------------------------------------------------------- /src/server/ResourceAllocator.cpp: -------------------------------------------------------------------------------- 1 | #include "server/ResourceAllocator.h" 2 | #include "authentication/ApplicationKey.h" 3 | #include "authentication/AuthenticationToken.h" 4 | #include "common/AllocatorMessageGenerator.h" 5 | 6 | #include "utils/logging.h" 7 | 8 | /* 9 | * Users authenticate in the ResourceManager and get a token 10 | * this token is an encrypted structure with application ID 11 | * This token can be used to communicate with the datacenter controllers 12 | * (e.g., allocate resources) 13 | */ 14 | 15 | namespace cirrus { 16 | 17 | ResourceAllocator::ResourceAllocator(int port, 18 | int timeout_ms) : 19 | RDMAServer(port, timeout_ms), 20 | total_mem_allocated_(0) { 21 | } 22 | 23 | ResourceAllocator::~ResourceAllocator() { 24 | } 25 | 26 | void ResourceAllocator::init() { 27 | RDMAServer::init(); 28 | } 29 | 30 | void ResourceAllocator::send_challenge(rdma_cm_id* id, 31 | const AllocatorMessage& /*msg*/) { 32 | __attribute__((unused)) 33 | auto token = AuthenticationToken::create_default_allow_token(); // allow 34 | auto ctx = reinterpret_cast(id->context); 35 | 36 | AllocatorMessageGenerator::auth_ack1( 37 | ctx->send_msg, 1, 42); 38 | send_message(id, sizeof(AllocatorMessage)); 39 | } 40 | 41 | void ResourceAllocator::send_stats(rdma_cm_id* id, 42 | const AllocatorMessage& /*msg*/) { 43 | auto ctx = reinterpret_cast(id->context); 44 | 45 | AllocatorMessageGenerator::stats_ack( 46 | ctx->send_msg, 47 | total_mem_allocated_); 48 | send_message(id, sizeof(AllocatorMessage)); 49 | } 50 | 51 | void ResourceAllocator::process_message(rdma_cm_id* id, void* message) { 52 | auto msg = reinterpret_cast(message); 53 | 54 | LOG("ResourceAllocator Received message"); 55 | 56 | __attribute__((unused)) 57 | auto ctx = reinterpret_cast(id->context); 58 | 59 | switch (msg->type) { 60 | case AUTH1: 61 | LOG("ResourceAllocator AUTH1"); 62 | // we get the application public key 63 | // we could also get an ID here 64 | // ApplicationKey* ak = &msg->data.auth.application_key; 65 | // AppId app_id = msg->data.auth.app_id; 66 | 67 | send_challenge(id, *msg); 68 | 69 | break; 70 | case AUTH2: 71 | LOG("ResourceAllocator AUTH2"); 72 | // we get the challenge response 73 | break; 74 | case STATS: 75 | send_stats(id, *msg); 76 | break; 77 | case ALLOC: 78 | // client wants to allocate some memory 79 | default: 80 | LOG("Unknown message"); 81 | exit(-1); 82 | break; 83 | } 84 | } 85 | 86 | } // namespace cirrus 87 | -------------------------------------------------------------------------------- /src/server/ResourceAllocator.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_RESOURCEALLOCATOR_H_ 2 | #define SRC_SERVER_RESOURCEALLOCATOR_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "utils/utils.h" 10 | #include "server/RDMAServer.h" 11 | #include "authentication/Authenticator.h" 12 | #include "common/AllocatorMessage.h" 13 | 14 | 15 | namespace cirrus { 16 | 17 | class ResourceAllocator : public RDMAServer { 18 | public: 19 | explicit ResourceAllocator(int port, int timeout_ms = 500); 20 | virtual ~ResourceAllocator(); 21 | 22 | virtual void init(); 23 | protected: 24 | void process_message(rdma_cm_id*, void* msg); 25 | 26 | void send_challenge(rdma_cm_id* id, const AllocatorMessage& msg); 27 | 28 | void send_stats(rdma_cm_id* id, 29 | const AllocatorMessage& msg); 30 | 31 | std::unique_ptr authenticator_; 32 | 33 | // total amount of memory allocated 34 | uint64_t total_mem_allocated_; 35 | }; 36 | 37 | } // namespace cirrus 38 | 39 | #endif // SRC_SERVER_RESOURCEALLOCATOR_H_ 40 | -------------------------------------------------------------------------------- /src/server/Server.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_SERVER_H_ 2 | #define SRC_SERVER_SERVER_H_ 3 | 4 | namespace cirrus { 5 | 6 | /** 7 | * Parent class for all servers. 8 | */ 9 | class Server { 10 | public: 11 | Server() = default; 12 | Server(Server&) = delete; 13 | }; 14 | 15 | } // namespace cirrus 16 | 17 | #endif // SRC_SERVER_SERVER_H_ 18 | -------------------------------------------------------------------------------- /src/server/StorageBackend.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_STORAGEBACKEND_H_ 2 | #define SRC_SERVER_STORAGEBACKEND_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "utils/logging.h" 11 | 12 | #include "flatbuffers/flatbuffers.h" 13 | #include "MemSlice.h" 14 | 15 | namespace cirrus { 16 | 17 | /** 18 | * Base class for datastructures that store key-value data 19 | */ 20 | class StorageBackend { 21 | public: 22 | virtual ~StorageBackend() = default; 23 | 24 | /** 25 | * Initialize backend 26 | */ 27 | virtual void init() = 0; 28 | 29 | /** 30 | * Put object 31 | * @param oid Object ID 32 | * @param data MemSlice to be written 33 | * @return bool Indicates success (true) or failure (false) 34 | */ 35 | virtual bool put(uint64_t oid, const MemSlice& data) = 0; 36 | 37 | /** 38 | * Check if object exists 39 | * @param oid Object ID 40 | * @return bool Indicates whether object exists (true) or not (false) 41 | */ 42 | virtual bool exists(uint64_t) const = 0; 43 | 44 | /** 45 | * Get object 46 | * @param oid Object ID 47 | * @return MemSlice containing the object's data 48 | */ 49 | virtual MemSlice get(uint64_t oid) const = 0; 50 | 51 | /** 52 | * Delete object 53 | * @param oid Object ID 54 | * @return bool Indicates success (true) or failure (false) 55 | */ 56 | virtual bool delet(uint64_t oid) = 0; 57 | 58 | /** 59 | * Get size of object 60 | * @param oid Object ID 61 | * @return Size of the object 62 | */ 63 | virtual uint64_t size(uint64_t oid) const = 0; 64 | }; 65 | 66 | } // namespace cirrus 67 | 68 | #endif // SRC_SERVER_STORAGEBACKEND_H_ 69 | -------------------------------------------------------------------------------- /src/server/TCPServer.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_SERVER_TCPSERVER_H_ 2 | #define SRC_SERVER_TCPSERVER_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "server/Server.h" 10 | #include "server/MemoryBackend.h" 11 | 12 | namespace cirrus { 13 | 14 | using ObjectID = uint64_t; 15 | /** 16 | * This class serves as a remote store that allows connection from 17 | * clients over TCP. 18 | * Clients can make requests for reading and writing data. 19 | */ 20 | class TCPServer : public Server { 21 | public: 22 | explicit TCPServer( 23 | int port, uint64_t pool_size_, 24 | const std::string& backend = "Memory", 25 | const std::string& storage_path = "/tmp/cirrus_storage/", 26 | uint64_t max_fds = 100); 27 | ~TCPServer() = default; 28 | 29 | virtual void init(); 30 | 31 | virtual void loop(); 32 | 33 | private: 34 | bool process(int sock); 35 | 36 | ssize_t send_all(int, const void*, size_t, int); 37 | ssize_t read_all(int sock, void*, size_t len); 38 | bool read_from_client(std::vector&, int, uint64_t&); 39 | 40 | bool testRemove(struct pollfd x); 41 | 42 | /** The port that the server is listening on. */ 43 | int port_; 44 | /** The fd for the socket the server listens for incoming requests on. */ 45 | int server_sock_ = 0; 46 | 47 | /** Maximum number of bytes that can be stored in the pool. */ 48 | uint64_t pool_size; 49 | 50 | /** Number of bytes currently in the pool. */ 51 | uint64_t curr_size = 0; 52 | 53 | /** Max number of sockets open at once. */ 54 | const uint64_t max_fds; 55 | 56 | /** 57 | * Index that the next socket accepted should have in the 58 | * array of struct pollfds. 59 | */ 60 | uint64_t curr_index = 0; 61 | /** 62 | * How long the client will wait during a call to poll() before 63 | * timing out. 64 | */ 65 | int timeout = 60 * 1000 * 3; 66 | 67 | /** 68 | * Vector that serves as a wrapper for a c style array containing 69 | * struct pollfd objects. Used for calls to poll(). New structs are 70 | * always inserted to the end of a continus range, indicated by curr_index. 71 | * When the vector reaches capacity, unused structs are removed and the 72 | * remaining items shifted. 73 | */ 74 | std::vector fds = std::vector(max_fds); 75 | 76 | /** 77 | * Memory interface 78 | */ 79 | std::unique_ptr mem; 80 | }; 81 | 82 | } // namespace cirrus 83 | 84 | #endif // SRC_SERVER_TCPSERVER_H_ 85 | -------------------------------------------------------------------------------- /src/server/TCPServerMain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "server/TCPServer.h" 4 | #include "utils/logging.h" 5 | 6 | const int port = 12345; 7 | const int max_fds = 100; 8 | static const uint64_t MB = (1024 * 1024); 9 | static const uint64_t GB = (1024 * MB); 10 | 11 | void print_arguments() { 12 | std::cout 13 | << "Error: ./tcpservermain" 14 | << " [pool_size=10] [backend_type=Memory]" 15 | << std::endl 16 | << " pool_size in MB" << std::endl 17 | << std::endl; 18 | } 19 | 20 | /** 21 | * Starts a TCP based key value store server. Accepts the pool size as 22 | * a command line argument. This specifies how large a memory pool will be 23 | * available to the server. Pool size defaults to 10 GB if not specified. 24 | */ 25 | auto main(int argc, char *argv[]) -> int { 26 | uint64_t pool_size = 10 * GB; 27 | std::string backend_type = "Memory"; 28 | std::string storage_path = "/tmp/cirrus_storage"; 29 | 30 | switch (argc) { 31 | case 4: 32 | { 33 | storage_path = argv[3]; 34 | #if __GNUC__ >= 7 35 | [[fallthrough]]; 36 | #endif 37 | } 38 | case 3: 39 | { 40 | if (strcmp(argv[2], "Memory") && strcmp(argv[2], "Storage")) { 41 | throw std::runtime_error("Wrong backend type"); 42 | } 43 | backend_type = argv[2]; 44 | #if __GNUC__ >= 7 45 | [[fallthrough]]; 46 | #endif 47 | } 48 | case 2: 49 | { 50 | std::istringstream iss(argv[1]); 51 | if (!(iss >> pool_size)) { 52 | std::cout << "Pool size in invalid format." << std::endl; 53 | return -1; 54 | } 55 | 56 | pool_size *= MB; 57 | #if __GNUC__ >= 7 58 | [[fallthrough]]; 59 | #endif 60 | } 61 | case 1: 62 | break; 63 | default: 64 | print_arguments(); 65 | throw std::runtime_error("Wrong number of arguments"); 66 | } 67 | 68 | // Instantiate the server 69 | cirrus::LOG( 70 | "Starting TCPServer in port: ", port, 71 | " with memory: ", pool_size); 72 | cirrus::TCPServer server(port, pool_size, backend_type, 73 | storage_path, max_fds); 74 | // Initialize the server 75 | server.init(); 76 | // Loop the server and listen for clients. Act on requests 77 | cirrus::LOG("Running server's loop"); 78 | server.loop(); 79 | return 0; 80 | } 81 | -------------------------------------------------------------------------------- /src/server/allocmain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "server/ResourceAllocator.h" 3 | #include "utils/logging.h" 4 | 5 | static const int PORT = 12346; 6 | 7 | void ctrlc_handler(int sig_num) { 8 | cirrus::LOG("Caught CTRL-C. sig_num: ", sig_num); 9 | exit(EXIT_FAILURE); 10 | } 11 | 12 | void set_ctrlc_handler() { 13 | struct sigaction sig_int_handler; 14 | 15 | sig_int_handler.sa_handler = ctrlc_handler; 16 | sigemptyset(&sig_int_handler.sa_mask); 17 | sig_int_handler.sa_flags = 0; 18 | 19 | sigaction(SIGINT, &sig_int_handler, nullptr); 20 | } 21 | 22 | auto main() -> int { 23 | cirrus::LOG("Starting resource allocator in port: ", PORT); 24 | 25 | cirrus::ResourceAllocator resalloc(PORT); 26 | 27 | cirrus::LOG("Running allocator init()"); 28 | resalloc.init(); 29 | 30 | cirrus::LOG("Running resource allocator loop"); 31 | resalloc.loop(); 32 | 33 | return 0; 34 | } 35 | 36 | -------------------------------------------------------------------------------- /src/server/bladeallocmain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "BladeAllocServer.h" 5 | #include "utils/logging.h" 6 | 7 | static const uint64_t MB = (1024 * 1024); 8 | static const uint64_t GB = (1024 * MB); 9 | 10 | static const int PORT = 12345; 11 | 12 | void ctrlc_handler(int sig_num) { 13 | cirrus::LOG("Caught CTRL-C. sig_num: ", sig_num); 14 | exit(EXIT_FAILURE); 15 | } 16 | 17 | void set_ctrlc_handler() { 18 | struct sigaction sig_int_handler; 19 | 20 | sig_int_handler.sa_handler = ctrlc_handler; 21 | sigemptyset(&sig_int_handler.sa_mask); 22 | sig_int_handler.sa_flags = 0; 23 | 24 | sigaction(SIGINT, &sig_int_handler, nullptr); 25 | } 26 | 27 | /** 28 | * Starts an RDMA based key value store server. Accepts the pool size as 29 | * a command line argument. This specifies how large a memory pool will be 30 | * available to the server. Pool size defaults to 10 GB if not specified. 31 | */ 32 | auto main(int argc, char *argv[]) -> int { 33 | uint64_t pool_size; 34 | // Parse arguments 35 | if (argc == 1) { 36 | // Default of 10 gigabytes 37 | pool_size = 10 * GB; 38 | } else if (argc == 2) { 39 | std::istringstream iss(argv[1]); 40 | if (!(iss >> pool_size)) { 41 | std::cout << "Pool size in invalid format." << std::endl; 42 | return -1; 43 | } 44 | pool_size *= MB; 45 | } else { 46 | std::cout << "Error: ./bladeallocmain [pool_size]" << std::endl; 47 | return -1; 48 | } 49 | 50 | cirrus::LOG("Starting BladeAllocServer in port: ", PORT); 51 | 52 | cirrus::BladeAllocServer server(PORT, pool_size); 53 | 54 | server.init(); 55 | 56 | cirrus::LOG("Running server's loop"); 57 | server.loop(); 58 | 59 | return 0; 60 | } 61 | -------------------------------------------------------------------------------- /src/server/bladepoolmain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "BladePoolServer.h" 3 | #include "utils/logging.h" 4 | 5 | static const uint64_t GB = (1024*1024*1024); 6 | static const int PORT = 12345; 7 | static const int initial_buffer_size = 50; 8 | 9 | void ctrlc_handler(int sig_num) { 10 | cirrus::LOG("Caught CTRL-C. sig_num: ", sig_num); 11 | exit(EXIT_FAILURE); 12 | } 13 | 14 | void set_ctrlc_handler() { 15 | struct sigaction sig_int_handler; 16 | 17 | sig_int_handler.sa_handler = ctrlc_handler; 18 | sigemptyset(&sig_int_handler.sa_mask); 19 | sig_int_handler.sa_flags = 0; 20 | 21 | sigaction(SIGINT, &sig_int_handler, nullptr); 22 | } 23 | 24 | auto main() -> int { 25 | cirrus::LOG("Starting RDMA server in port: ", PORT); 26 | 27 | cirrus::BladePoolServer server(PORT, 10 * GB); 28 | 29 | server.init(); 30 | 31 | cirrus::LOG("Running server's loop"); 32 | server.loop(); 33 | 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /src/server/controllermain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "ResourceAllocator.h" 3 | #include "utils/logging.h" 4 | 5 | static const uint64_t GB = (1024*1024*1024); 6 | static const int PORT = 12346; 7 | 8 | void ctrlc_handler(int sig_num) { 9 | cirrus::LOG("Caught CTRL-C. sig_num: ", sig_num); 10 | exit(EXIT_FAILURE); 11 | } 12 | 13 | void set_ctrlc_handler() { 14 | struct sigaction sig_int_handler; 15 | 16 | sig_int_handler.sa_handler = ctrlc_handler; 17 | sigemptyset(&sig_int_handler.sa_mask); 18 | sig_int_handler.sa_flags = 0; 19 | 20 | sigaction(SIGINT, &sig_int_handler, nullptr); 21 | } 22 | 23 | auto main() -> int { 24 | cirrus::LOG("Starting RDMA server in port: ", PORT); 25 | 26 | cirrus::ResourceAllocator server(PORT); 27 | 28 | server.init(); 29 | 30 | cirrus::LOG("Running ResourceAllocator's loop"); 31 | server.loop(); 32 | 33 | return 0; 34 | } 35 | 36 | -------------------------------------------------------------------------------- /src/server/objectstoremain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "BladeObjectStore.h" 3 | #include "utils/logging.h" 4 | 5 | static const uint64_t GB = (1024*1024*1024); 6 | 7 | static const int PORT = 12345; 8 | 9 | void ctrlc_handler(int sig_num) { 10 | cirrus::LOG("Caught CTRL-C. sig_num: ", sig_num); 11 | exit(EXIT_FAILURE); 12 | } 13 | 14 | void set_ctrlc_handler() { 15 | struct sigaction sig_int_handler; 16 | 17 | sig_int_handler.sa_handler = ctrlc_handler; 18 | sigemptyset(&sig_int_handler.sa_mask); 19 | sig_int_handler.sa_flags = 0; 20 | 21 | sigaction(SIGINT, &sig_int_handler, nullptr); 22 | } 23 | 24 | auto main() -> int { 25 | cirrus::LOG("Starting BladeObjectStore in port: ", PORT); 26 | 27 | cirrus::BladeObjectStore server(PORT, 10 * GB); 28 | 29 | server.init(); 30 | 31 | cirrus::LOG("Running server's loop"); 32 | server.loop(); 33 | 34 | return 0; 35 | } 36 | 37 | -------------------------------------------------------------------------------- /src/utils/CirrusTime.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "utils/CirrusTime.h" 3 | #include "utils/Log.h" 4 | 5 | namespace cirrus { 6 | 7 | TimerFunction::~TimerFunction() { 8 | if (!print_) 9 | return; 10 | 11 | auto t1 = Time::now(); 12 | us d_us = std::chrono::duration_cast(t1 - t0); 13 | ms d_ms = std::chrono::duration_cast(t1 - t0); 14 | 15 | LOG(name_, ": ", d_us.count(), "us ", d_ms.count(), "ms"); 16 | } 17 | 18 | void TimerFunction::reset() { 19 | t0 = Time::now(); 20 | } 21 | 22 | uint64_t TimerFunction::getNsElapsed() const { 23 | auto t1 = Time::now(); 24 | ns d_ns = std::chrono::duration_cast(t1 - t0); 25 | return d_ns.count(); 26 | } 27 | 28 | double TimerFunction::getUsElapsed() const { 29 | return getNsElapsed() / 1000.0; 30 | } 31 | 32 | double TimerFunction::getSecElapsed() const { 33 | return getUsElapsed() / 1000000.0; 34 | } 35 | 36 | std::string getTimeNow() { 37 | // std::chrono::time_point start = std::chrono::system_clock::now(); 39 | auto t = std::chrono::system_clock::now(); 40 | auto t2 = std::chrono::time_point_cast(t); 41 | 42 | auto res = t2.time_since_epoch().count(); 43 | return to_string(res); 44 | } 45 | 46 | } // namespace cirrus 47 | -------------------------------------------------------------------------------- /src/utils/CirrusTime.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_CIRRUSTIME_H_ 2 | #define SRC_UTILS_CIRRUSTIME_H_ 3 | 4 | #include 5 | #include 6 | #include "utils/StringUtils.h" 7 | 8 | namespace cirrus { 9 | 10 | std::string getTimeNow(); 11 | 12 | class TimerFunction { 13 | public: 14 | explicit TimerFunction(const std::string& name = "", bool print = false) 15 | : name_(name), 16 | t0(Time::now()), 17 | print_(print) { } 18 | 19 | ~TimerFunction(); 20 | 21 | void reset(); 22 | 23 | uint64_t getNsElapsed() const; 24 | double getUsElapsed() const; 25 | double getSecElapsed() const; 26 | 27 | using Time = std::chrono::high_resolution_clock; 28 | using ns = std::chrono::nanoseconds; 29 | using us = std::chrono::microseconds; 30 | using ms = std::chrono::milliseconds; 31 | 32 | private: 33 | std::string name_; 34 | Time::time_point t0; 35 | bool print_; 36 | }; 37 | 38 | } // namespace cirrus 39 | 40 | #endif // SRC_UTILS_CIRRUSTIME_H_ 41 | -------------------------------------------------------------------------------- /src/utils/InfinibandSupport.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "utils/InfinibandSupport.h" 3 | #include "utils/utils.h" 4 | #include "utils/logging.h" 5 | 6 | namespace cirrus { 7 | 8 | void InfinibandSupport::check_mw_support(ibv_context* ctx) { 9 | struct ibv_exp_device_attr device_attr; 10 | 11 | std::memset(&device_attr, 0, sizeof(device_attr)); 12 | device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_RESERVED - 1; 13 | 14 | ibv_exp_query_device(ctx, &device_attr); 15 | if (device_attr.exp_device_cap_flags & IBV_EXP_DEVICE_MEM_WINDOW || 16 | device_attr.exp_device_cap_flags & IBV_EXP_DEVICE_MW_TYPE_2B) { 17 | LOG("MW supported"); 18 | } else { 19 | DIE("MW not supported"); 20 | } 21 | } 22 | 23 | void InfinibandSupport::check_odp_support(ibv_context* ctx) { 24 | struct ibv_exp_device_attr device_attr; 25 | 26 | std::memset(&device_attr, 0, sizeof(device_attr)); 27 | device_attr.comp_mask = 28 | IBV_EXP_DEVICE_ATTR_ODP | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS; 29 | 30 | ibv_exp_query_device(ctx, &device_attr); 31 | if (device_attr.exp_device_cap_flags & IBV_EXP_DEVICE_ODP) { 32 | LOG("ODP supported"); 33 | } else { 34 | LOG("ODP not supported!"); 35 | } 36 | } 37 | 38 | } // namespace cirrus 39 | -------------------------------------------------------------------------------- /src/utils/InfinibandSupport.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_INFINIBANDSUPPORT_H_ 2 | #define SRC_UTILS_INFINIBANDSUPPORT_H_ 3 | 4 | #include 5 | #include 6 | #include "utils/logging.h" 7 | 8 | namespace cirrus { 9 | 10 | class InfinibandSupport { 11 | public: 12 | InfinibandSupport() = default; 13 | 14 | void check_mw_support(ibv_context* ctx); 15 | void check_odp_support(ibv_context* ctx); 16 | 17 | std::once_flag mw_support_check_; 18 | std::once_flag odp_support_check_; 19 | }; 20 | 21 | } // namespace cirrus 22 | 23 | #endif // SRC_UTILS_INFINIBANDSUPPORT_H_ 24 | -------------------------------------------------------------------------------- /src/utils/Log.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_LOG_H_ 2 | #define SRC_UTILS_LOG_H_ 3 | 4 | #include 5 | #include 6 | #include "utils/CirrusTime.h" 7 | #include "utils/StringUtils.h" 8 | #include "common/Synchronization.h" 9 | 10 | #define INIT_SWITCH (1) 11 | // by default logging is off 12 | #define DEFAULT_SWITCH (0) 13 | 14 | namespace cirrus { 15 | 16 | enum { INFO_TAG = 1, ERROR_TAG = 2, PERF_TAG = 3}; 17 | 18 | struct INFO { 19 | static const int value = 1; 20 | constexpr static const char* prefix = "[INFO]"; 21 | }; 22 | struct ERROR { 23 | static const int value = 2; 24 | constexpr static const char* prefix = "[ERROR]"; 25 | }; 26 | struct PERF { 27 | static const int value = 3; 28 | constexpr static const char* prefix = "[PERF]"; 29 | }; 30 | 31 | #if __GNUC__ >= 7 32 | struct FLUSH { }; 33 | struct NO_FLUSH { }; 34 | struct TIME { }; 35 | struct NO_TIME { }; 36 | 37 | template 41 | #else 42 | template 43 | #endif 44 | bool LOG(Params&& ... param) { 45 | static int INFO_SWITCH = INIT_SWITCH; 46 | static int ERROR_SWITCH = INIT_SWITCH; 47 | static int PERF_SWITCH = INIT_SWITCH; 48 | if (INFO_SWITCH == INIT_SWITCH) { 49 | INFO_SWITCH = DEFAULT_SWITCH; 50 | ERROR_SWITCH = DEFAULT_SWITCH; 51 | PERF_SWITCH = DEFAULT_SWITCH; 52 | if (const char* env = std::getenv("CIRRUS_LOG")) { 53 | int v = string_to(env); 54 | if (v) { 55 | INFO_SWITCH = v; 56 | ERROR_SWITCH = v; 57 | } 58 | } 59 | if (const char* env = std::getenv("CIRRUS_PERF")) { 60 | int v = string_to(env); 61 | if (v) { 62 | PERF_SWITCH = v; 63 | } 64 | } 65 | } 66 | 67 | if ( (T::value == INFO_TAG && INFO_SWITCH <= 0) || 68 | (T::value == ERROR_TAG && ERROR_SWITCH <= 0) || 69 | (T::value == PERF_TAG && PERF_SWITCH <= 0)) { 70 | return true; 71 | } 72 | 73 | std::cout << T::prefix 74 | << " -"; 75 | 76 | #if __GNUC__ >= 7 77 | if constexpr (std::is_same_v) { 78 | std::cout << getTimeNow(); 79 | } 80 | #else 81 | std::cout << getTimeNow(); 82 | #endif 83 | 84 | auto f = [](const auto& arg) { 85 | std::cout << " " << arg; 86 | }; 87 | 88 | __attribute__((unused)) 89 | int dummy[] = { 0, ( (void) f(std::forward(param)), 0) ... }; 90 | 91 | #if __GNUC__ >= 7 92 | if constexpr (std::is_same_v) { 93 | std::cout << std::endl; 94 | } 95 | #else 96 | std::cout << std::endl; 97 | #endif 98 | 99 | return true; // success 100 | } 101 | 102 | } // namespace cirrus 103 | 104 | #endif // SRC_UTILS_LOG_H_ 105 | -------------------------------------------------------------------------------- /src/utils/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | 5 | noinst_LIBRARIES = libutils.a 6 | libutils_a_SOURCES = CirrusTime.cpp Stats.cpp 7 | libutils_a_CPPFLAGS = -ggdb -I$(top_srcdir) -I$(top_srcdir)/src 8 | 9 | if USE_RDMA 10 | libutils_a_SOURCES += InfinibandSupport.cpp 11 | endif 12 | -------------------------------------------------------------------------------- /src/utils/Stats.cpp: -------------------------------------------------------------------------------- 1 | #include "utils/Stats.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace cirrus { 10 | 11 | Stats::Stats() { 12 | } 13 | 14 | void Stats::add(double d) { 15 | sum += d; 16 | sq_sum += d * d; 17 | count++; 18 | data.push_back(d); 19 | } 20 | 21 | double Stats::avg() const { 22 | return sum / count; 23 | } 24 | 25 | double Stats::size() const { 26 | return count; 27 | } 28 | 29 | double Stats::sd() const { 30 | return sqrt(sq_sum / count - avg() * avg()); 31 | } 32 | 33 | double Stats::total() const { 34 | return sum; 35 | } 36 | 37 | int Stats::getCount() const { 38 | return count; 39 | } 40 | 41 | double Stats::getPercentile(double p) const { 42 | if (data.size() == 0) 43 | throw std::runtime_error("No data"); 44 | 45 | std::sort(data.begin(), data.end()); 46 | uint32_t loc = static_cast(p * data.size()); 47 | assert(loc < data.size()); 48 | return data[loc]; 49 | } 50 | 51 | double Stats::min() const { 52 | if (data.size() == 0) 53 | throw std::runtime_error("No data"); 54 | 55 | std::sort(data.begin(), data.end()); 56 | return data[0]; 57 | } 58 | 59 | double Stats::max() const { 60 | if (data.size() == 0) 61 | throw std::runtime_error("No data"); 62 | 63 | std::sort(data.begin(), data.end()); 64 | return data[data.size() - 1]; 65 | } 66 | 67 | void Stats::reserve(uint64_t n) { 68 | data.reserve(n); 69 | } 70 | 71 | } // namespace cirrus 72 | 73 | -------------------------------------------------------------------------------- /src/utils/Stats.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_STATS_H_ 2 | #define SRC_UTILS_STATS_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace cirrus { 8 | 9 | class Stats { 10 | public: 11 | Stats(); 12 | 13 | void add(double d); 14 | 15 | double min() const; 16 | double max() const; 17 | double avg() const; 18 | double size() const; 19 | double sd() const; 20 | double getPercentile(double p) const; 21 | double total() const; 22 | 23 | int getCount() const; 24 | 25 | void reserve(uint64_t); 26 | private: 27 | mutable std::vector data; 28 | 29 | 30 | double sum = 0; 31 | double sq_sum = 0; 32 | double count = 0; 33 | }; 34 | 35 | } // namespace cirrus 36 | 37 | #endif // SRC_UTILS_STATS_H_ 38 | -------------------------------------------------------------------------------- /src/utils/StringUtils.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_STRINGUTILS_H_ 2 | #define SRC_UTILS_STRINGUTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace cirrus { 8 | 9 | template 10 | std::string to_string(Params&& ... param) { 11 | std::ostringstream ss; 12 | auto f = [](std::ostringstream& ss, const auto& arg) { 13 | ss << " " << arg; 14 | }; 15 | 16 | __attribute__((unused)) 17 | int dummy[] = { 0, ( (void) f(ss, std::forward(param)), 0) ... }; 18 | 19 | return ss.str(); 20 | } 21 | 22 | template 23 | T string_to(const std::string& s) { 24 | std::istringstream iss(s); 25 | T v; 26 | iss >> v; 27 | return v; 28 | } 29 | 30 | } // namespace cirrus 31 | 32 | #endif // SRC_UTILS_STRINGUTILS_H_ 33 | -------------------------------------------------------------------------------- /src/utils/logging.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_LOGGING_H_ 2 | #define SRC_UTILS_LOGGING_H_ 3 | 4 | #include "utils/Log.h" 5 | 6 | #endif // SRC_UTILS_LOGGING_H_ 7 | -------------------------------------------------------------------------------- /src/utils/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_UTILS_UTILS_H_ 2 | #define SRC_UTILS_UTILS_H_ 3 | 4 | #include 5 | #include 6 | 7 | #define DIE(s) { \ 8 | fprintf(stderr, s);\ 9 | fflush(stderr); \ 10 | std::cout << "Exiting" << std::endl; \ 11 | exit(-1);\ 12 | } 13 | 14 | // die if not zero 15 | #define TEST_NZ(x) do { \ 16 | if ((x)) { \ 17 | printf("errno: %d\n", errno); \ 18 | DIE("error: " #x " failed (returned non-zero).");}\ 19 | } while (0) 20 | 21 | // template 22 | // void TEST_NZ(const T& t) { 23 | // if (t) { 24 | // DIE("error: " #x " failed (returned non-zero)."); 25 | // } 26 | //} 27 | 28 | // die if zero 29 | #define TEST_Z(x) do { \ 30 | if (!(x)) { \ 31 | printf("errno: %d\n", errno); \ 32 | DIE("error: " #x " failed (returned zero/null).");}\ 33 | } while (0) 34 | 35 | #endif // SRC_UTILS_UTILS_H_ 36 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | test_mt 2 | test_mult_clients 3 | test_fullblade_store 4 | -------------------------------------------------------------------------------- /tests/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | SUBDIRS = object_store client 5 | 6 | if USE_MPI 7 | SUBDIRS += mpi 8 | endif 9 | -------------------------------------------------------------------------------- /tests/client/.gitignore: -------------------------------------------------------------------------------- 1 | tcpclientmain 2 | rdmaclientmain 3 | -------------------------------------------------------------------------------- /tests/client/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | bin_PROGRAMS = tcpclientmain 5 | 6 | LIBS = -lclient -lutils -lauthentication -lcommon 7 | 8 | LINCLUDES = -L$(top_srcdir)/src/utils/ \ 9 | -L$(top_srcdir)/src/client/ \ 10 | -L$(top_srcdir)/src/authentication \ 11 | -L$(top_srcdir)/src/common \ 12 | $(LIBRDMACM) $(LIBIBVERBS) 13 | 14 | if USE_RDMA 15 | bin_PROGRAMS += rdmaclientmain 16 | endif 17 | 18 | CPPFLAGS = -ggdb -I$(top_srcdir) $(DEFINE_LOG) \ 19 | -I$(top_srcdir)/third_party/flatbuffers/include \ 20 | -I$(top_srcdir)/src \ 21 | -isystem $(top_srcdir)/third_party/libcuckoo/ 22 | LDFLAGS = -pthread 23 | 24 | LDADD = $(LIBS) $(LINCLUDES) 25 | 26 | rdmaclientmain_SOURCES = RDMAClientmain.cpp 27 | 28 | tcpclientmain_SOURCES = TCPClientMain.cpp 29 | -------------------------------------------------------------------------------- /tests/client/TCPClientMain.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "client/TCPClient.h" 4 | #include "tests/object_store/object_store_internal.h" 5 | #include "common/Serializer.h" 6 | 7 | // TODO(Tyler): Remove hardcoded port 8 | const char port[] = "12345"; 9 | const char *IP; 10 | 11 | /** 12 | * Simple test verifying that basic put/get works as intended. 13 | */ 14 | void test_sync() { 15 | cirrus::TCPClient client; 16 | cirrus::serializer_simple serializer; 17 | std::cout << "Test Starting." << std::endl; 18 | client.connect(IP, port); 19 | std::cout << "Connected to server." << std::endl; 20 | int message = 42; 21 | std::cout << "message declared." << std::endl; 22 | cirrus::WriteUnitTemplate w(serializer, message); 23 | client.write_sync(1, w); 24 | std::cout << "write sync complete" << std::endl; 25 | 26 | auto ptr_pair = client.read_sync(1); 27 | const int *int_ptr = reinterpret_cast(ptr_pair.first.get()); 28 | std::cout << *int_ptr << " returned from server" << std::endl; 29 | 30 | if (*int_ptr != message) { 31 | throw std::runtime_error("Wrong value returned."); 32 | } 33 | } 34 | 35 | /** 36 | * Simple test verifying that basic asynchronous put/get works as intended. 37 | */ 38 | void test_async() { 39 | cirrus::TCPClient client; 40 | cirrus::serializer_simple serializer; 41 | client.connect(IP, port); 42 | 43 | int message = 42; 44 | cirrus::WriteUnitTemplate w(serializer, message); 45 | auto future = client.write_async(1, w); 46 | std::cout << "write sync complete" << std::endl; 47 | 48 | if (!future.get()) { 49 | throw std::runtime_error("Error during async write."); 50 | } 51 | 52 | auto read_future = client.read_async(1); 53 | 54 | if (!read_future.get()) { 55 | throw std::runtime_error("Error during async write."); 56 | } 57 | 58 | auto ret_ptr = read_future.getDataPair().first; 59 | 60 | const int returned = *(reinterpret_cast(ret_ptr.get())); 61 | 62 | std::cout << returned << " returned from server" << std::endl; 63 | 64 | if (returned != message) { 65 | throw std::runtime_error("Wrong value returned."); 66 | } 67 | } 68 | 69 | auto main(int argc, char *argv[]) -> int { 70 | IP = cirrus::test_internal::ParseIP(argc, argv); 71 | std::cout << "Test Starting." << std::endl; 72 | test_sync(); 73 | test_async(); 74 | std::cout << "Test successful." << std::endl; 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /tests/mpi/.gitignore: -------------------------------------------------------------------------------- 1 | test_mpi 2 | -------------------------------------------------------------------------------- /tests/mpi/Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = foreign 2 | 3 | bin_PROGRAMS = test_mpi 4 | 5 | TOP_DIR = $(top_srcdir)/../../ 6 | 7 | LIBS = -lclient -lauthentication -lutils -lcommon 8 | 9 | LINCLUDES = -L$(TOP_DIR)/src/utils/ \ 10 | -L$(TOP_DIR)/src/client/ \ 11 | -L$(TOP_DIR)/src/authentication \ 12 | -L$(TOP_DIR)/src/common 13 | 14 | test_mpi_SOURCES = test_mpi.cpp 15 | test_mpi_LDFLAGS = -pthread 16 | test_mpi_CPPFLAGS = -ggdb -I$(TOP_DIR) -I$(TOP_DIR)/examples/sparsehash/src/ \ 17 | -I$(TOP_DIR)/src \ 18 | -isystem $(TOP_DIR)/third_party/libcuckoo/ \ 19 | -I$(TOP_DIR)/third_party/flatbuffers/include 20 | test_mpi_LDADD = $(LINCLUDES) $(LIBS) 21 | 22 | 23 | -------------------------------------------------------------------------------- /tests/mpi/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT(CIRRUS, 0.1, joao@berkeley.edu) 6 | AM_INIT_AUTOMAKE 7 | 8 | # Checks for programs. 9 | AC_PROG_CXX([mpic++]) 10 | AC_PROG_AWK 11 | AC_PROG_CC 12 | AC_PROG_CPP 13 | AC_PROG_INSTALL 14 | AC_PROG_LN_S 15 | AC_PROG_MAKE_SET 16 | AC_PROG_RANLIB 17 | AX_CXX_COMPILE_STDCXX_14 18 | 19 | CXXFLAGS="-O3 -fPIC -std=c++1z -Werror" 20 | 21 | # Checks for header files. 22 | AC_CHECK_HEADERS([ arpa/inet.h netdb.h string.h stdint.h\ 23 | stdlib.h sys/types.h sys/socket.h sys/epoll.h\ 24 | sys/time.h syslog.h unistd.h google/dense_hash_map]) 25 | 26 | # Checks for typedefs, structures, and compiler characteristics. 27 | AC_CHECK_HEADER_STDBOOL 28 | AC_C_INLINE 29 | AC_TYPE_SIZE_T 30 | AC_TYPE_UINT32_T 31 | AC_TYPE_UINT64_T 32 | 33 | # Checks for library functions. 34 | AC_FUNC_MALLOC 35 | AC_FUNC_STRCOLL 36 | AC_CHECK_FUNCS([gettimeofday localtime_r memchr memmove memset mkdir stpcpy strchr strcspn strdup strerror strpbrk strrchr strspn strstr]) 37 | AC_CONFIG_FILES([Makefile]) 38 | 39 | AC_OUTPUT 40 | -------------------------------------------------------------------------------- /tests/mpi/configure.gnu: -------------------------------------------------------------------------------- 1 | autoconf 2 | ./configure 3 | -------------------------------------------------------------------------------- /tests/mpi/run.sh: -------------------------------------------------------------------------------- 1 | #export LD_LIBRARY_PATH=~/ 2 | cp test_mpi ~/ 3 | mpirun -np 10 ~/test_mpi 4 | #mpirun -x $LD_LIBRARY_PATH -hostfile ~/hosts2 -np 1 ~/test_mpi 5 | -------------------------------------------------------------------------------- /tests/object_store/.gitignore: -------------------------------------------------------------------------------- 1 | test_store_v2 2 | exhaustion 3 | test_fullblade_store 4 | test_mt 5 | test_mult_clients 6 | test_cache_manager 7 | test_iterator 8 | -------------------------------------------------------------------------------- /tests/object_store/Makefile.am: -------------------------------------------------------------------------------- 1 | include $(top_srcdir)/common.mk 2 | 3 | AUTOMAKE_OPTIONS = foreign 4 | bin_PROGRAMS = exhaustion test_store_v2 test_mt test_mult_clients \ 5 | test_cache_manager test_iterator test_fullblade_store \ 6 | test_store_bulk 7 | 8 | LIBS = -lclient -lauthentication -lutils -lcommon -levictionpolicies \ 9 | $(LIBRDMACM) $(LIBIBVERBS) 10 | 11 | LINCLUDES = -L$(top_srcdir)/src/utils/ \ 12 | -L$(top_srcdir)/src/authentication \ 13 | -L$(top_srcdir)/src/common \ 14 | -L$(top_srcdir)/src/client \ 15 | -L$(top_srcdir)/src/cache_manager 16 | 17 | LDFLAGS = -pthread 18 | LDADD = $(LINCLUDES) $(LIBS) 19 | CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/src \ 20 | -I$(top_srcdir)/examples/sparsehash/src/ \ 21 | -isystem $(top_srcdir)/third_party/libcuckoo/ \ 22 | -I$(top_srcdir)/third_party/flatbuffers/include 23 | 24 | test_fullblade_store_SOURCES = test_fullblade_store.cpp 25 | 26 | test_mt_SOURCES = test_mt.cpp 27 | 28 | test_mult_clients_SOURCES = test_mult_clients.cpp 29 | 30 | test_store_v2_SOURCES = test_store_v2.cpp 31 | 32 | exhaustion_SOURCES = mem_exhaustion.cpp 33 | 34 | test_cache_manager_SOURCES = test_cache_manager.cpp 35 | 36 | test_iterator_SOURCES = test_iterator.cpp 37 | 38 | test_store_bulk_SOURCES = test_store_bulk.cpp 39 | -------------------------------------------------------------------------------- /tests/object_store/mem_exhaustion.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "object_store/FullBladeObjectStore.h" 5 | #include "tests/object_store/object_store_internal.h" 6 | #include "common/Exception.h" 7 | #include "client/BladeClient.h" 8 | 9 | // TODO(Tyler): Remove hardcoded IP and PORT 10 | static const uint64_t GB = (1024*1024*1024); 11 | const char PORT[] = "12345"; 12 | const char *IP; 13 | static const uint32_t SIZE = 1024*1024; // One MB 14 | static const uint64_t MILLION = 1000000; 15 | bool use_rdma_client; 16 | 17 | /** 18 | * This test aims to ensure that when the remote server no longer has 19 | * room to fulfill all allocations it notifies the client, which will 20 | * then throw an error message. 21 | * This test assumes that the server does not have enough room to store 22 | * one million objects of one MB each (1 TB). This test also ensures 23 | * that the allocation error does not crash the server as in order 24 | * for notification of failure to reach the client, the server must 25 | * have been running to send the message. 26 | */ 27 | void test_exhaustion() { 28 | std::unique_ptr client = 29 | cirrus::test_internal::GetClient(use_rdma_client); 30 | cirrus::serializer_simple> serializer; 31 | cirrus::ostore::FullBladeObjectStoreTempl> 32 | store(IP, PORT, client.get(), 33 | serializer, 34 | cirrus::deserializer_simple, 35 | sizeof(cirrus::Dummy)>); 36 | struct cirrus::Dummy d(42); 37 | 38 | std::cout << "Putting one million objects" << std::endl; 39 | for (uint64_t i = 0; i < MILLION; ++i) { 40 | store.put(i, d); 41 | } 42 | } 43 | 44 | /** 45 | * Test to ensure that the remove method works as expected. Ensures that when 46 | * the server is at capacity, an item can be removed to make more room. 47 | */ 48 | void test_exhaustion_remove() { 49 | std::unique_ptr client = 50 | cirrus::test_internal::GetClient(use_rdma_client); 51 | cirrus::serializer_simple> serializer; 52 | cirrus::ostore::FullBladeObjectStoreTempl> 53 | store(IP, PORT, client.get(), 54 | serializer, 55 | cirrus::deserializer_simple, SIZE>); 56 | struct cirrus::Dummy d(42); 57 | 58 | std::cout << "Putting one million objects" << std::endl; 59 | uint64_t i; 60 | for (i = 0; i < MILLION; ++i) { 61 | try { 62 | store.put(i, d); 63 | } catch (const cirrus::ServerMemoryErrorException& e) { 64 | break; 65 | } 66 | } 67 | std::cout << "Removing an object to make room." << std::endl; 68 | // Remove an object to make room 69 | store.remove(0); 70 | std::cout << "Attempting to insert an object in the newly freed space." 71 | << std::endl; 72 | // Attempt to use the newly freed space 73 | store.put(i, d); 74 | } 75 | 76 | auto main(int argc, char *argv[]) -> int { 77 | std::cout << "Runing exhaustion test" << std::endl; 78 | 79 | use_rdma_client = cirrus::test_internal::ParseMode(argc, argv); 80 | IP = cirrus::test_internal::ParseIP(argc, argv); 81 | test_exhaustion_remove(); 82 | try { 83 | test_exhaustion(); 84 | } catch (const cirrus::ServerMemoryErrorException& e) { 85 | std::cout << "Test successful" << std::endl; 86 | return 0; 87 | } 88 | /* Exception should be thrown above and caught */ 89 | return -1; 90 | } 91 | -------------------------------------------------------------------------------- /tests/object_store/test_mt.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "object_store/FullBladeObjectStore.h" 10 | #include "tests/object_store/object_store_internal.h" 11 | #include "utils/CirrusTime.h" 12 | #include "client/BladeClient.h" 13 | 14 | // TODO(Tyler): Remove hardcoded IP and PORT 15 | static const uint64_t GB = (1024*1024*1024); 16 | static const uint64_t MB = (1024*1024); 17 | static const int N_THREADS = 2; 18 | const char PORT[] = "12345"; 19 | const char *IP; 20 | static const uint32_t SIZE = 1024; 21 | bool use_rdma_client; 22 | 23 | /** 24 | * Tests that behavior is as expected when multiple threads make get and put 25 | * requests to the remote store. These threads all use the same instance of 26 | * the store to connect. 27 | */ 28 | void test_mt() { 29 | cirrus::TimerFunction tf("connect time", true); 30 | 31 | std::unique_ptr client = 32 | cirrus::test_internal::GetClient(use_rdma_client); 33 | cirrus::serializer_simple> serializer; 34 | cirrus::ostore::FullBladeObjectStoreTempl> store(IP, 35 | PORT, 36 | client.get(), 37 | serializer, 38 | cirrus::deserializer_simple, 39 | sizeof(cirrus::Dummy)>); 40 | 41 | std::thread* threads[N_THREADS]; 42 | 43 | std::random_device rd; 44 | std::mt19937 gen(rd()); 45 | std::uniform_int_distribution<> dis(1, 10); 46 | int start = 0; 47 | int stop = 10; 48 | for (int i = 0; i < N_THREADS; ++i) { 49 | threads[i] = new std::thread([dis, gen, start, stop, & store]() { 50 | for (int i = start; i < stop; i++) { 51 | int rnd = std::rand(); 52 | struct cirrus::Dummy d(rnd); 53 | 54 | store.put(i, d); 55 | cirrus::Dummy d2 = store.get(i); 56 | 57 | if (d2.id != rnd) 58 | throw std::runtime_error("Incorrect value returned."); 59 | } 60 | }); 61 | start += 10; 62 | stop += 10; 63 | } 64 | 65 | for (int i = 0; i < N_THREADS; ++i) 66 | threads[i]->join(); 67 | } 68 | 69 | auto main(int argc, char *argv[]) -> int { 70 | use_rdma_client = cirrus::test_internal::ParseMode(argc, argv); 71 | IP = cirrus::test_internal::ParseIP(argc, argv); 72 | test_mt(); 73 | std::cout << "Test success" << std::endl; 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /tests/object_store/test_mult_clients.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "object_store/FullBladeObjectStore.h" 12 | #include "tests/object_store/object_store_internal.h" 13 | #include "utils/CirrusTime.h" 14 | #include "common/Synchronization.h" 15 | #include "client/BladeClient.h" 16 | 17 | // TODO(Tyler): Remove hardcoded IP and PORT 18 | static const uint64_t GB = (1024*1024*1024); 19 | static const uint64_t MB = (1024*1024); 20 | static const int N_THREADS = 20; 21 | const char PORT[] = "12345"; 22 | const char *IP; 23 | static const uint32_t SIZE = 1024; 24 | bool use_rdma_client; 25 | 26 | std::atomic total_puts = {0}; 27 | 28 | /** 29 | * Checks that the system works properly when multiple clients get and put. 30 | * These clients each use their own local store and client instances. 31 | */ 32 | void test_multiple_clients() { 33 | std::cout << "Multiple clients test" << std::endl; 34 | 35 | cirrus::TimerFunction tf("connect time", true); 36 | 37 | std::thread* threads[N_THREADS]; 38 | 39 | std::random_device rd; 40 | std::mt19937 gen(rd()); 41 | std::uniform_int_distribution<> dis(1, 10); 42 | int start = 0; 43 | int stop = 10; 44 | for (int i = 0; i < N_THREADS; ++i) { 45 | threads[i] = new std::thread([dis, gen, start, stop]() { 46 | std::unique_ptr client = 47 | cirrus::test_internal::GetClient( 48 | use_rdma_client); 49 | cirrus::serializer_simple> serializer; 50 | cirrus::ostore::FullBladeObjectStoreTempl> 51 | store(IP, PORT, client.get(), 52 | serializer, 53 | cirrus::deserializer_simple, 54 | sizeof(cirrus::Dummy)>); 55 | for (int i = start; i < stop; i++) { 56 | int rnd = std::rand(); 57 | struct cirrus::Dummy d(rnd); 58 | store.put(i, d); 59 | 60 | struct cirrus::Dummy d2 = store.get(i); 61 | 62 | if (d2.id != rnd) { 63 | std::cout << "Expected " << rnd << " but got " << d2.id 64 | << std::endl; 65 | throw std::runtime_error("Wrong value returned."); 66 | } 67 | total_puts++; 68 | } 69 | }); 70 | start += 10; 71 | stop += 10; 72 | } 73 | 74 | for (int i = 0; i < N_THREADS; ++i) 75 | threads[i]->join(); 76 | 77 | std::cout << "Test successful" << std::endl; 78 | } 79 | 80 | auto main(int argc, char *argv[]) -> int { 81 | use_rdma_client = cirrus::test_internal::ParseMode(argc, argv); 82 | IP = cirrus::test_internal::ParseIP(argc, argv); 83 | test_multiple_clients(); 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /tests/object_store/test_shards.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "object_store/FullBladeObjectStore.h" 6 | #include "tests/object_store/object_store_internal.h" 7 | #include "utils/CirrusTime.h" 8 | #include "utils/Stats.h" 9 | #include "client/BladeClient.h" 10 | 11 | // TODO(Tyler): Remove hardcoded IP and PORT 12 | static const uint64_t MILLION = 1000000; 13 | static const uint64_t BILLION = MILLION * 1000; 14 | static const uint64_t GB = 1024 * 1024 * 1024; 15 | bool use_rdma_client; 16 | 17 | /** 18 | * Test the correctness of multiple shards 19 | * We assume all the servers are running in 127.0.01 20 | * with IPs in the range 12345 ... 12345 + num_shards 21 | */ 22 | void test_store_shards(uint64_t num_shards) { 23 | std::unique_ptr client = 24 | cirrus::test_internal::GetClient(use_rdma_client); 25 | cirrus::serializer_simple serializer; 26 | 27 | std::vector ips; 28 | std::vector ports; 29 | 30 | for (uint64_t i = 0; i < num_shards; ++i) { 31 | ips.push_back("127.0.0.1"); 32 | ports.push_back(std::to_string(12345 + i)); 33 | } 34 | 35 | cirrus::ostore::FullBladeObjectStoreTempl store( 36 | ips, ports, 37 | client.get(), 38 | serializer, 39 | cirrus::deserializer_simple); 40 | 41 | srand(time(NULL)); 42 | 43 | std::vector oids; 44 | for (uint64_t i = 0; i < 1000; i++) { 45 | cirrus::ObjectID oid = rand() % BILLION; 46 | oids.push_back(oid); 47 | store.put(oid, oid); 48 | } 49 | 50 | for (const auto& oid : oids) { 51 | cirrus::ObjectID retval = store.get(oid); 52 | if (retval != oid) { 53 | throw std::runtime_error("Wrong value returned."); 54 | } 55 | } 56 | } 57 | 58 | auto main(int argc, char *argv[]) -> int { 59 | use_rdma_client = cirrus::test_internal::ParseMode(argc, argv); 60 | char* num_shards = cirrus::test_internal::ParseNumShards(argc, argv); 61 | std::cout << "Test starting" << std::endl; 62 | 63 | if (num_shards == nullptr) { 64 | throw std::runtime_error("Wrong num shards"); 65 | } 66 | std::cout << "num_shards: " << num_shards << std::endl; 67 | test_store_shards(std::stoi(num_shards)); 68 | std::cout << "Test successful" << std::endl; 69 | return 0; 70 | } 71 | -------------------------------------------------------------------------------- /tests/object_store/test_store_v2.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "object_store/FullBladeObjectStore.h" 6 | #include "tests/object_store/object_store_internal.h" 7 | #include "utils/CirrusTime.h" 8 | #include "utils/Stats.h" 9 | #include "client/BladeClient.h" 10 | 11 | // TODO(Tyler): Remove hardcoded IP and PORT 12 | static const uint64_t GB = (1024*1024*1024); 13 | const char PORT[] = "12345"; 14 | const char *IP; 15 | bool use_rdma_client; 16 | 17 | /** 18 | * Test simple synchronous put and get to/from the object store. 19 | * Uses simpler objects than test_fullblade_store. 20 | */ 21 | void test_store_simple() { 22 | std::unique_ptr client = 23 | cirrus::test_internal::GetClient(use_rdma_client); 24 | cirrus::serializer_simple serializer; 25 | 26 | cirrus::ostore::FullBladeObjectStoreTempl store(IP, PORT, 27 | client.get(), 28 | serializer, 29 | cirrus::deserializer_simple); 30 | for (int oid = 0; oid < 10; oid++) { 31 | store.put(oid, oid); 32 | } 33 | 34 | for (int oid = 0; oid < 10; oid++) { 35 | int retval = store.get(oid); 36 | if (retval != oid) { 37 | throw std::runtime_error("Wrong value returned."); 38 | } 39 | } 40 | } 41 | 42 | auto main(int argc, char *argv[]) -> int { 43 | use_rdma_client = cirrus::test_internal::ParseMode(argc, argv); 44 | IP = cirrus::test_internal::ParseIP(argc, argv); 45 | std::cout << "Test starting" << std::endl; 46 | test_store_simple(); 47 | std::cout << "Test successful" << std::endl; 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /tests/raw/test_atomics.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "client/BladeClient.h" 15 | #include "utils/Time.h" 16 | 17 | // TODO(Tyler): Remove hardcoded IP and PORT 18 | static const uint64_t GB = (1024*1024*1024); 19 | static const uint64_t MB = (1024*1024); 20 | static const int N_THREADS = 2; 21 | const char PORT[] = "12345"; 22 | const char IP[] = "10.10.49.83"; 23 | static const uint32_t SIZE = 1024; 24 | 25 | void test_fetchadd() { 26 | cirrus::BladeClient client; 27 | 28 | client.connect(IP, PORT); 29 | 30 | cirrus::AllocationRecord allocRec = client.allocate(SIZE); 31 | 32 | client.fetchadd_sync(allocRec, 0, 42); 33 | 34 | client.deallocate(allocRec); 35 | } 36 | 37 | auto main() -> int { 38 | test_fetchadd(); 39 | 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /tests/test_bulk_transfer_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_store_bulk" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_cache_manager_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_cache_manager" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_cache_manager_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_cache_manager" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_client_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/client/rdmaclientmain" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_client_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/client/tcpclientmain" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_iterator_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_iterator" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_iterator_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_iterator" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mem_exhaustion_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/exhaustion" 10 | # Call script to run the test 11 | test_runner.runExhaustionRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mem_exhaustion_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/exhaustion" 10 | # Call script to run the test 11 | test_runner.runExhaustionTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mt_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_mt" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mt_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_mt" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mult_clients_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_mult_clients" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_mult_clients_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_mult_clients" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_store_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_fullblade_store" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_store_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_fullblade_store" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_store_simple_RDMA.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_store_v2" 10 | # Call script to run the test 11 | test_runner.runTestRDMA(testPath) 12 | -------------------------------------------------------------------------------- /tests/test_store_simple_TCP.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import subprocess 5 | import time 6 | import test_runner 7 | 8 | # Set name of test to run 9 | testPath = "./tests/object_store/test_store_v2" 10 | # Call script to run the test 11 | test_runner.runTestTCP(testPath) 12 | -------------------------------------------------------------------------------- /third_party/get_eigen.sh: -------------------------------------------------------------------------------- 1 | wget http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2 2 | tar -xof 3.3.3.tar.bz2 3 | mv eigen-eige* eigen_source 4 | rm -f 3.3.3.tar.bz2 5 | --------------------------------------------------------------------------------