├── .clang-format ├── .gitignore ├── README.md ├── scripts ├── build_debug.sh ├── build_release.sh └── run_clang_tidy.sh └── src ├── CMakeLists.txt ├── click ├── CMakeLists.txt ├── README.md ├── call.h ├── dedup.h ├── driver.cc ├── driver.h ├── drop.h ├── dup.h ├── event_handler.h ├── example_client.cc ├── example_server.cc ├── group_by.h ├── id.h ├── map.h ├── pushable.h ├── socket_recv.h ├── socket_send.cc ├── socket_send.h ├── tee.h ├── timer.h ├── timer_file_descriptor.cc └── timer_file_descriptor.h ├── cmake ├── benchmarking.cmake └── testing.cmake ├── concurrency ├── CMakeLists.txt ├── README.md ├── barrier.cc ├── barrier.h ├── bounded_queue.h ├── bounded_queue_test.cc ├── queue.h ├── queue_test.cc ├── thread_pool.cc ├── thread_pool.h └── thread_pool_test.cc ├── flow ├── CMakeLists.txt ├── README.md ├── coroutine_operator.h ├── cross.h ├── cross_test.cc ├── empty.h ├── empty_test.cc ├── exchange.h ├── exchange_test.cc ├── filter.h ├── filter_bench.cc ├── filter_test.cc ├── iterator.h ├── iterator_test.cc ├── map.h ├── map_test.cc ├── naive_equijoin.h ├── naive_equijoin_test.cc ├── operator.h ├── template_helpers.h └── tuple_helpers.h ├── key_value_stores ├── CMakeLists.txt ├── README.md ├── client.cc ├── client.h ├── ici_client.cc ├── message.proto ├── msgqueue.cc ├── rc_client.cc ├── ruc_client.cc ├── server.cc ├── util.cc ├── util.h ├── zmq_util.cc └── zmq_util.h ├── lattices ├── CMakeLists.txt ├── README.md ├── array_lattice.h ├── array_lattice_test.cc ├── bool_and_lattice.h ├── bool_and_lattice_test.cc ├── bool_or_lattice.h ├── bool_or_lattice_test.cc ├── lattice.h ├── lattices.h ├── map_lattice.h ├── map_lattice_test.cc ├── max_lattice.h ├── max_lattice_test.cc ├── min_lattice.h ├── min_lattice_test.cc ├── pair_lattice.h ├── pair_lattice_test.cc ├── set_intersect_lattice.h ├── set_intersect_lattice_test.cc ├── set_union_lattice.h ├── set_union_lattice_test.cc ├── timestamp_lattice.h ├── timestamp_lattice_test.cc ├── tombstoned_set_union_lattice.h ├── vector_lattice.h └── vector_lattice_test.cc ├── pushpull ├── CMakeLists.txt ├── README.md ├── counter.h ├── counter_test.cc ├── dedup.h ├── dedup_test.cc ├── drop.h ├── drop_test.cc ├── dup.h ├── dup_test.cc ├── filter.h ├── filter_test.cc ├── group_by.h ├── group_by_test.cc ├── id.h ├── id_test.cc ├── if.h ├── if_test.cc ├── join.h ├── join_test.cc ├── lambda.h ├── lambda_test.cc ├── map.h ├── map_test.cc ├── phantoms.h ├── puller.h ├── pusher.h ├── queue.h ├── queue_test.cc ├── register.h ├── register_test.cc ├── round_robin.h ├── round_robin_test.cc ├── tee.h └── tee_test.cc ├── vendor ├── boost │ └── CMakeLists.txt ├── googlebenchmark │ └── CMakeLists.txt ├── googletest │ └── CMakeLists.txt ├── zeromq │ └── CMakeLists.txt └── zeromqcpp │ └── CMakeLists.txt └── zmq_util ├── CMakeLists.txt ├── README.md ├── connection_id.cc ├── connection_id.h ├── enveloped_message.cc ├── enveloped_message.h ├── hexdump.cc ├── hexdump.h ├── msg_util.cc ├── msg_util.h ├── zmq_util.cc └── zmq_util.h /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: Google 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: true 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: All 15 | AllowShortIfStatementsOnASingleLine: true 16 | AllowShortLoopsOnASingleLine: true 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: None 36 | BreakBeforeBraces: Attach 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | ColumnLimit: 80 40 | CommentPragmas: '^ IWYU pragma:' 41 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 42 | ConstructorInitializerIndentWidth: 4 43 | ContinuationIndentWidth: 4 44 | Cpp11BracedListStyle: true 45 | DerivePointerAlignment: true 46 | DisableFormat: false 47 | ExperimentalAutoDetectBinPacking: false 48 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 49 | IncludeCategories: 50 | - Regex: '^<.*\.h>' 51 | Priority: 1 52 | - Regex: '^<.*' 53 | Priority: 2 54 | - Regex: '.*' 55 | Priority: 3 56 | IndentCaseLabels: true 57 | IndentWidth: 2 58 | IndentWrappedFunctionNames: false 59 | KeepEmptyLinesAtTheStartOfBlocks: false 60 | MacroBlockBegin: '' 61 | MacroBlockEnd: '' 62 | MaxEmptyLinesToKeep: 1 63 | NamespaceIndentation: None 64 | ObjCBlockIndentWidth: 2 65 | ObjCSpaceAfterProperty: false 66 | ObjCSpaceBeforeProtocolList: false 67 | PenaltyBreakBeforeFirstCallParameter: 1 68 | PenaltyBreakComment: 300 69 | PenaltyBreakFirstLessLess: 120 70 | PenaltyBreakString: 1000 71 | PenaltyExcessCharacter: 1000000 72 | PenaltyReturnTypeOnItsOwnLine: 200 73 | PointerAlignment: Left 74 | ReflowComments: true 75 | SortIncludes: true 76 | SpaceAfterCStyleCast: false 77 | SpaceBeforeAssignmentOperators: true 78 | SpaceBeforeParens: ControlStatements 79 | SpaceInEmptyParentheses: false 80 | SpacesBeforeTrailingComments: 2 81 | SpacesInAngles: false 82 | SpacesInContainerLiterals: true 83 | SpacesInCStyleCastParentheses: false 84 | SpacesInParentheses: false 85 | SpacesInSquareBrackets: false 86 | Standard: Auto 87 | TabWidth: 8 88 | UseTab: Never 89 | ... 90 | 91 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | vendor/gtest/build 3 | vendor/gbenchmark/build 4 | 5 | # ignore compiled byte code 6 | target 7 | 8 | # ignore output files from testing 9 | output* 10 | 11 | # ignore standard Eclipse files 12 | .project 13 | .classpath 14 | .settings 15 | .checkstyle 16 | 17 | # ignore standard IntelliJ files 18 | .idea/ 19 | *.iml 20 | *.iws 21 | 22 | # ignore standard Vim and Emacs temp files 23 | *.swp 24 | *~ 25 | 26 | # ignore standard Mac OS X files/dirs 27 | .DS_Store 28 | 29 | ################################################################################ 30 | # vim 31 | ################################################################################ 32 | # swap 33 | [._]*.s[a-w][a-z] 34 | [._]s[a-w][a-z] 35 | # session 36 | Session.vim 37 | # temporary 38 | .netrwhist 39 | *~ 40 | # auto-generated tag files 41 | tags 42 | # syntastic 43 | .syntastic_clang_tidy_config 44 | .syntastic_cpp_config 45 | 46 | ################################################################################ 47 | # C++ 48 | ################################################################################ 49 | # Prerequisites 50 | *.d 51 | 52 | # Compiled Object files 53 | *.slo 54 | *.lo 55 | *.o 56 | *.obj 57 | 58 | # Precompiled Headers 59 | *.gch 60 | *.pch 61 | 62 | # Compiled Dynamic libraries 63 | *.so 64 | *.dylib 65 | *.dll 66 | 67 | # Fortran module files 68 | *.mod 69 | *.smod 70 | 71 | # Compiled Static libraries 72 | *.lai 73 | *.la 74 | *.a 75 | *.lib 76 | 77 | # Executables 78 | *.exe 79 | *.out 80 | *.app 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LatticeFlow 2 | [BOOM][boom-analytics-paper], [Bloom][bloom-paper], and [Bloom^L][blooml-paper] 3 | explored how the conjunction of data-centric declarative programming and join 4 | semilattices could make writing eventually consistent programs easier and more 5 | principled. Building upon its academic ancestors, LatticeFlow explores how to 6 | make lattice-based stream processing fast and friendly. 7 | 8 | ## Getting Started 9 | First, install `clang`, `cmake`, and `protobuf` (See 10 | [here][cpp_project_template] for details). 11 | 12 | ```bash 13 | # Building Code. 14 | ./scripts/build_debug.sh # build the code in debug mode 15 | ./scripts/build_release.sh # build the code in release mode 16 | 17 | # Running Code. 18 | ./build/lattices/pair_lattice_test # run a test 19 | (cd build && make test) # run all the tests 20 | ./build/flow/filter_bench # run a benchmark 21 | ./build/key_value_stores/msgqueue # start the message broker 22 | ./build/key_value_stores/server # start the key-value server 23 | ./build/key_value_stores/ruc_client # start a key-value client 24 | 25 | # Formatting and Linting Code. 26 | clang-format src/key_value_stores/server.cc # format a file 27 | find src -name '*.cc' -o -name '*.h' | xargs clang-format -i # format all files 28 | ./scripts/run_clang_tidy.sh src/key_value_stores/server.cc # lint a file 29 | ./scripts/run_clang_tidy.sh $(find src -name '*.cc') # lint all files 30 | ``` 31 | 32 | ## Tour 33 | | Directory | Description | 34 | | ---------------------------------------------------- | ---------------------------------------------- | 35 | | [`click`](src/click/README.md) | Click-style event driven server framework. | 36 | | [`concurrency`](src/concurrency/README.md) | Concurrency and synchronization primitives. | 37 | | [`flow`](src/flow/README.md) | Iterators and stream processors. | 38 | | [`key_value_stores`](src/key_value_stores/README.md) | Key value store with varying isolation levels. | 39 | | [`lattices`](src/lattices/README.md) | A collection of join semilattices. | 40 | | [`pushpull`](src/pushpull/README.md) | Click style push and pull operators. | 41 | | [`zmq_util`](src/zmq_util/README.md) | ZeroMQ utilities. | 42 | 43 | ## Administrivia 44 | - All code should abide by the [Google C++ Style 45 | Guide](https://google.github.io/styleguide/cppguide.html). 46 | - All code should be formatted with `clang-format` and linted using 47 | `clang-tidy` (see [Getting Started](#getting-started)). 48 | - Third party libraries should be used sparingly. 49 | - TODO: Continuous Integration. 50 | 51 | Refer also to the [RISE Lab C++ Project Template][cpp_project_template]. 52 | 53 | [boom-analytics-paper]: https://scholar.google.com/scholar?cluster=17621967364443794950&hl=en&as_sdt=0,5 54 | [bloom-paper]: https://scholar.google.com/scholar?cluster=9165311711752272482&hl=en&as_sdt=0,5 55 | [blooml-paper]: https://scholar.google.com/scholar?cluster=1332747912204910097&hl=en&as_sdt=0,5 56 | [cpp_project_template]: https://github.com/ucbrise/cpp_project_template 57 | -------------------------------------------------------------------------------- /scripts/build_debug.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -euo pipefail 4 | 5 | main() { 6 | export CC="clang" 7 | export CXX="clang++" 8 | cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -Hsrc -Bbuild 9 | cmake --build build 10 | } 11 | 12 | main 13 | -------------------------------------------------------------------------------- /scripts/build_release.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -euo pipefail 4 | 5 | main() { 6 | export CC="clang" 7 | export CXX="clang++" 8 | cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Release -Hsrc -Bbuild 9 | cmake --build build 10 | } 11 | 12 | main 13 | -------------------------------------------------------------------------------- /scripts/run_clang_tidy.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # This script runs clang-tidy on its input arguments with all of clang-tidy's 4 | # flags set up nicely. 5 | 6 | set -euo pipefail 7 | 8 | usage() { 9 | echo "usage: run_clang_tidy.sh ..." 10 | } 11 | 12 | # http://stackoverflow.com/a/17841619/3187068 13 | join() { 14 | local IFS="," 15 | echo "$*" 16 | } 17 | 18 | main() { 19 | if [[ $# -eq 0 ]]; then 20 | usage 21 | return -1 22 | fi 23 | 24 | # Arguments for the `-checks` flag. 25 | checks=() 26 | checks+=('*') 27 | checks+=('-llvm-header-guard') 28 | checks+=('-llvm-include-order') 29 | # Using googletest macros like EXPECT_EQ trigger this check. 30 | checks+=('-cppcoreguidelines-pro-type-vararg') 31 | 32 | # Argument for the header-filter command. We want to run clang-tidy on all 33 | # of our header files, but not on any of the system header files. We also 34 | # do not want to run clang-tidy on any of the generated protobuf headers. 35 | local header_filter='src/(lattices|key_value_stores)/.*[^.][^p][^b]\.h' 36 | 37 | clang-tidy \ 38 | -checks="$(join "${checks[@]}")" \ 39 | -header-filter="$header_filter" \ 40 | -p build/compile_commands.json \ 41 | "$@" 42 | } 43 | 44 | main "$@" 45 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | PROJECT(LatticeFlow) 4 | SET(LATTICEFLOW_VERSION_MAJOR 0) 5 | SET(LATTICEFLOW_VERSION_MINOR 1) 6 | SET(LATTICEFLOW_VERSION_PATCH 0) 7 | 8 | INCLUDE(cmake/benchmarking.cmake NO_POLICY_SCOPE) 9 | INCLUDE(cmake/testing.cmake NO_POLICY_SCOPE) 10 | 11 | SET(CMAKE_CXX_FLAGS_COMMON 12 | "-std=c++11 \ 13 | -stdlib=libc++ \ 14 | -Wall \ 15 | -Wextra \ 16 | -Werror \ 17 | -fsanitize=address") 18 | SET(CMAKE_CXX_FLAGS_DEBUG 19 | "${CMAKE_CXX_FLAGS_DEBUG} \ 20 | ${CMAKE_CXX_FLAGS_COMMON}") 21 | SET(CMAKE_CXX_FLAGS_RELEASE 22 | "${CMAKE_CXX_FLAGS_RELEASE} \ 23 | ${CMAKE_CXX_FLAGS_COMMON} 24 | -O3") 25 | 26 | ADD_SUBDIRECTORY(vendor/boost) 27 | ADD_SUBDIRECTORY(vendor/googlebenchmark) 28 | ADD_SUBDIRECTORY(vendor/googletest) 29 | ADD_SUBDIRECTORY(vendor/zeromq) 30 | ADD_SUBDIRECTORY(vendor/zeromqcpp) 31 | 32 | INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) 33 | INCLUDE_DIRECTORIES(.) 34 | INCLUDE_DIRECTORIES(${BOOST_INCLUDE_DIRS}) 35 | INCLUDE_DIRECTORIES(${GBENCH_INCLUDE_DIRS}) 36 | INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIRS}) 37 | INCLUDE_DIRECTORIES(${ZEROMQCPP_INCLUDE_DIRS}) 38 | INCLUDE_DIRECTORIES(${ZEROMQ_INCLUDE_DIRS}) 39 | 40 | LINK_DIRECTORIES(${BOOST_LINK_DIRS}) 41 | LINK_DIRECTORIES(${ZEROMQ_LINK_DIRS}) 42 | 43 | ENABLE_TESTING() 44 | ADD_SUBDIRECTORY(click) 45 | ADD_SUBDIRECTORY(concurrency) 46 | ADD_SUBDIRECTORY(flow) 47 | ADD_SUBDIRECTORY(key_value_stores) 48 | ADD_SUBDIRECTORY(lattices) 49 | ADD_SUBDIRECTORY(pushpull) 50 | ADD_SUBDIRECTORY(zmq_util) 51 | -------------------------------------------------------------------------------- /src/click/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | ADD_LIBRARY(click 4 | driver.cc 5 | example_server.cc 6 | socket_send.cc 7 | timer_file_descriptor.cc 8 | ) 9 | 10 | ADD_EXECUTABLE(click_example_server example_server.cc) 11 | TARGET_LINK_LIBRARIES(click_example_server click zmq_util) 12 | 13 | ADD_EXECUTABLE(click_example_client example_client.cc) 14 | TARGET_LINK_LIBRARIES(click_example_client click zmq_util) 15 | -------------------------------------------------------------------------------- /src/click/README.md: -------------------------------------------------------------------------------- 1 | # Click 2 | Kohler et al. recognized that routers were increasingly expected to do more 3 | than route. However, traditional routers were inflexible, and it was hard to 4 | modify them to meet the increasingly complex functionality demanded of them. 5 | To solve this problem, they introduced the [Click modular router][click]: a 6 | framework which enables the simple and modular construction of complex routers. 7 | 8 | Using Click, routers are a directed graph of processing units called 9 | *elements*. Click routers are 100% event driven. There is a single process; 10 | there is a single thread; there are no synchronous I/O calls. The leaves of the 11 | element graph register events and event handlers with an event loop which polls 12 | asynchronous I/O calls and dispatches to the correct event handler which pushes 13 | data through the element graph. 14 | 15 | This directory generalizes Click and implements a framework to build 16 | event-driven servers. Much like with Click, we construct directed graphs of 17 | processing elements, and the leaf elements register events and event handlers. 18 | The main difference between this framework and Click is that we include general 19 | elements (e.g. [`Map`](map.h), [`GroupBy`](group_by.h)) that are not specific 20 | to building routers. 21 | 22 | An example server written in this framework is given in 23 | [`example_server.cc`](example_server.cc). The corresponding client is 24 | [`example_client.cc`](example_client.cc). 25 | 26 | [click]: http://bit.ly/2esyPOk 27 | -------------------------------------------------------------------------------- /src/click/call.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_CALL_H_ 2 | #define CLICK_CALL_H_ 3 | 4 | #include 5 | 6 | #include "click/pushable.h" 7 | 8 | namespace latticeflow { 9 | 10 | // `Call(f).push(x)` performs `f(x)` where `x` is an lvalue or rvalue reference 11 | // and is forwarded to `f`. 12 | template 13 | class Call : public Pushable { 14 | public: 15 | explicit Call(F&& f) : f_(std::forward(f)) {} 16 | void push(T&& x) override { f_(std::forward(x)); } 17 | 18 | private: 19 | F f_; 20 | }; 21 | 22 | template 23 | Call make_call(F&& f) { 24 | return Call(std::forward(f)); 25 | } 26 | 27 | } // namespace latticeflow 28 | 29 | #endif // CLICK_CALL_H_ 30 | -------------------------------------------------------------------------------- /src/click/dedup.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_DEDUP_H_ 2 | #define CLICK_DEDUP_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "click/pushable.h" 8 | 9 | namespace latticeflow { 10 | 11 | template 12 | class Dedup : public Pushable { 13 | // TODO(mwhittaker): I don't think Dedup shouldn be owning everything passed 14 | // to it. Think about how to make this better. 15 | static_assert( 16 | !std::is_reference::value, 17 | "Dedup takes ownership of values, so they must be passed by value."); 18 | 19 | public: 20 | explicit Dedup(Pushable* downstream, const int num_dups) 21 | : downstream_(downstream), num_dups_(num_dups) { 22 | // TODO(mwhittaker): User proper assertions. 23 | assert(num_dups > 1); 24 | } 25 | 26 | void push(From x) override { 27 | auto it = counts_.find(x); 28 | if (it == std::end(counts_)) { 29 | counts_.insert(std::make_pair(std::move(x), 1)); 30 | } else { 31 | it->second++; 32 | if (it->second == num_dups_) { 33 | downstream_->push(std::move(x)); 34 | counts_.erase(it); 35 | } 36 | } 37 | } 38 | 39 | private: 40 | Pushable* downstream_; 41 | const int num_dups_; 42 | std::map counts_; 43 | }; 44 | 45 | template 46 | Dedup make_dedup(Pushable* downstream, const int num_dups) { 47 | return Dedup(downstream, num_dups); 48 | } 49 | 50 | } // namespace latticeflow 51 | 52 | #endif // CLICK_DEDUP_H_ 53 | -------------------------------------------------------------------------------- /src/click/driver.cc: -------------------------------------------------------------------------------- 1 | #include "click/driver.h" 2 | 3 | #include 4 | #include 5 | 6 | namespace latticeflow { 7 | 8 | void Driver::RegisterEventHandler(EventHandler* event_handler) { 9 | event_handlers_.push_back(event_handler); 10 | pollitems_.push_back(event_handler->GetPollitem()); 11 | } 12 | 13 | void Driver::Run() { 14 | while (true) { 15 | // `EventHandler`s may call `RegisterEventHandler` which means that if we 16 | // iterate over `event_handlers_` or `pollitems_`, the iterators may be 17 | // invalidated as we call event handlers. Thus, we move both vectors into 18 | // local variables before iterating over them. Calls to 19 | // `RegisterEventHandler` will add to the member variables, not the local 20 | // variables. At the end of each iteration, we move values back from the 21 | // local variables to the members. 22 | std::vector event_handlers = std::move(event_handlers_); 23 | std::vector pollitems = std::move(pollitems_); 24 | event_handlers_.clear(); 25 | pollitems_.clear(); 26 | 27 | zmq::poll(pollitems.data(), pollitems.size(), -1); 28 | 29 | auto pollitems_it = std::begin(pollitems); 30 | auto event_handlers_it = std::begin(event_handlers); 31 | while (pollitems_it != std::end(pollitems)) { 32 | assert(event_handlers_it != std::end(event_handlers)); 33 | 34 | if (pollitems_it->events & pollitems_it->revents) { 35 | // One-off timers might kill themselves when their event handler is 36 | // called. Thus, we need to make sure not not access the event handler 37 | // after we've run it. 38 | bool is_one_off = (*event_handlers_it)->IsOneOff(); 39 | (*event_handlers_it)->Run(*pollitems_it); 40 | if (is_one_off) { 41 | pollitems_it = pollitems.erase(pollitems_it); 42 | event_handlers_it = event_handlers.erase(event_handlers_it); 43 | continue; 44 | } 45 | } 46 | 47 | pollitems_it++; 48 | event_handlers_it++; 49 | } 50 | 51 | event_handlers_.insert(std::end(event_handlers_), 52 | std::begin(event_handlers), 53 | std::end(event_handlers)); 54 | pollitems_.insert(std::end(pollitems_), std::begin(pollitems), 55 | std::end(pollitems)); 56 | } 57 | } 58 | 59 | } // namespace latticeflow 60 | -------------------------------------------------------------------------------- /src/click/driver.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_DRIVER_H_ 2 | #define CLICK_DRIVER_H_ 3 | 4 | #include 5 | 6 | #include "click/event_handler.h" 7 | 8 | namespace latticeflow { 9 | 10 | // Kohler et al. recognized that routers were increasingly expected to do more 11 | // than route. However, traditional routers were inflexible, and it was hard to 12 | // modify them to meet the increasingly complex functionality demanded of them. 13 | // To solve this problem, they introduced the Click modular router [1]: a 14 | // framework which enables the simple and modular construction of complex 15 | // routers. 16 | // 17 | // Using Click, routers are a directed graph of processing units called 18 | // *elements*. Click routers are 100% event driven. There is a single process; 19 | // there is a single thread; there are no synchronous I/O calls. The leaves of 20 | // the element graph register events and event handlers with an event loop 21 | // which polls asynchronous I/O calls and dispatches to the correct event 22 | // handler which pushes data through the element graph. 23 | // 24 | // A `Driver` implements a main event loop in this style. Code can register 25 | // `EventHandler`s with a `Driver`. Once `Driver::Run` is called, the `Driver` 26 | // begins to poll and dispatch to the correct `EventHandler`s. Moreover, 27 | // `EventHandler`s can be registered with a `Driver` even after `Run` has been 28 | // called. 29 | // 30 | // [1]: http://bit.ly/2esyPOk 31 | class Driver { 32 | public: 33 | // Register an `EventHandler` with a `Driver`. This can be called before or 34 | // after `Run`. The `Driver` does not take ownership of the pointer. 35 | void RegisterEventHandler(EventHandler* event_handler); 36 | 37 | // Start polling and dispatching to `EventHandler`s. Calling `Run` will block 38 | // forever. 39 | void Run(); 40 | 41 | private: 42 | // The `EventHandler`s registered with `RegisterEventHandler`. The pointers 43 | // are not owned. 44 | std::vector event_handlers_; 45 | 46 | // `pollitems_[i]` is the poll item for `event_handlers_[i]` 47 | // `event_handlers_` and `pollitems_` are always the same length. 48 | std::vector pollitems_; 49 | }; 50 | 51 | } // namespace latticeflow 52 | 53 | #endif // CLICK_DRIVER_H_ 54 | -------------------------------------------------------------------------------- /src/click/drop.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_DROP_H_ 2 | #define CLICK_DROP_H_ 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "click/pushable.h" 9 | 10 | namespace latticeflow { 11 | 12 | // `Drop().push(x)` does nothing; it is a no-op. 13 | template 14 | class Drop : public Pushable { 15 | public: 16 | void push(T&&) override { std::cout << "dropped" << std::endl; } 17 | }; 18 | 19 | } // namespace latticeflow 20 | 21 | #endif // CLICK_DROP_H_ 22 | -------------------------------------------------------------------------------- /src/click/dup.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_DUP_H_ 2 | #define CLICK_DUP_H_ 3 | 4 | #include "click/pushable.h" 5 | 6 | namespace latticeflow { 7 | 8 | template 9 | class Dup : public Pushable { 10 | // TODO(mwhittaker): Figure out if we should include this. See comment in 11 | // tee.h. 12 | static_assert(!std::is_rvalue_reference::value || 13 | std::is_reference::value, 14 | "You cannot create a `Dup` which forwards rvalue references to " 15 | "downstream `Pushable`s that take arguments by value. In most " 16 | "circumstances, this is a bug. You should not be " 17 | "move-constructing multiple objects from the same " 18 | "rvalue-reference."); 19 | 20 | public: 21 | explicit Dup(Pushable* downstream, const int num_dups) 22 | : downstream_(downstream), num_dups_(num_dups) {} 23 | 24 | void push(From&& x) override { 25 | for (int i = 0; i < num_dups_; ++i) { 26 | downstream_->push(std::forward(x)); 27 | } 28 | } 29 | 30 | private: 31 | Pushable* downstream_; 32 | const int num_dups_; 33 | }; 34 | 35 | template 36 | Dup make_dup(Pushable* downstream, const int num_dups) { 37 | return Dup(downstream, num_dups); 38 | } 39 | 40 | } // namespace latticeflow 41 | 42 | #endif // CLICK_DUP_H_ 43 | -------------------------------------------------------------------------------- /src/click/event_handler.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_EVENT_HANDLER_H_ 2 | #define CLICK_EVENT_HANDLER_H_ 3 | 4 | #include "zmq.hpp" 5 | 6 | namespace latticeflow { 7 | 8 | // See `driver.h` for relevant documentation. Each `EventHandler` represents an 9 | // event to listen for and a corresponding event handler. 10 | class EventHandler { 11 | public: 12 | // If an `EventHandler` is one-off, then after its event triggers, it is not 13 | // polled for again. For example, waiting for a timer would be a one-off 14 | // event while waiting for packets from the network would not be. 15 | virtual bool IsOneOff() const = 0; 16 | 17 | // `GetPollitem()` is the poll item that the `Driver` polls on. 18 | virtual zmq::pollitem_t GetPollitem() = 0; 19 | 20 | // `Run(pollitem)` runs the event handler with the poll item returned by 21 | // `GetPollitem` modified by `zmq::poll`. `Run` will only be called if the 22 | // corresponding event happens. It won't be spuriously called. 23 | virtual void Run(zmq::pollitem_t pollitem) = 0; 24 | }; 25 | 26 | } // namespace latticeflow 27 | 28 | #endif // CLICK_EVENT_HANDLER_H_ 29 | -------------------------------------------------------------------------------- /src/click/example_client.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "zmq.hpp" 5 | 6 | #include "zmq_util/zmq_util.h" 7 | 8 | namespace lf = latticeflow; 9 | 10 | int main() { 11 | zmq::context_t context(1); 12 | zmq::socket_t socket(context, ZMQ_DEALER); 13 | const std::string address = "tcp://localhost:5555"; 14 | socket.connect(address); 15 | std::cout << "Client connected to '" << address << "'." << std::endl; 16 | 17 | std::vector pollitems = { 18 | {socket, 0, ZMQ_POLLIN, 0}, {nullptr, 0, ZMQ_POLLIN, 0}, 19 | }; 20 | 21 | std::string line; 22 | std::cout << "> " << std::flush; 23 | while (true) { 24 | lf::poll(-1, &pollitems); 25 | 26 | if (pollitems[0].revents & ZMQ_POLLIN) { 27 | std::string msg = lf::recv_string(&socket); 28 | std::cout << "Received '" << msg << "'." << std::endl; 29 | std::cout << "> " << std::flush; 30 | } 31 | 32 | if (pollitems[1].revents & ZMQ_POLLIN) { 33 | std::getline(std::cin, line); 34 | lf::send_string(line, &socket); 35 | std::cout << "Sent '" << line << "'." << std::endl; 36 | std::cout << "> " << std::flush; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/click/group_by.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_MAP_H_ 2 | #define CLICK_MAP_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "click/call.h" 10 | #include "click/pushable.h" 11 | 12 | namespace latticeflow { 13 | 14 | // A `GroupBy` implements the relational GROUP BY operator. Elements annotated 15 | // with their group can be pushed into the `GroupBy` using `GroupBy::push`. The 16 | // contents of a group can be flushed downstream by calling `GroupBy::end`. For 17 | // example, 18 | // 19 | // Group group_by(&downstream); 20 | // group_by.push().push(std::make_pair(0, 1)); 21 | // group_by.push().push(std::make_pair(1, 100)); 22 | // group_by.push().push(std::make_pair(0, 2)); 23 | // group_by.push().push(std::make_pair(1, 200)); 24 | // group_by.push().push(std::make_pair(0, 3)); 25 | // group_by.end().push(0); // pushes (0, [1, 2, 3]) to downstream 26 | // group_by.end().push(1); // pushes (1, [100, 200]) to downstream 27 | // group_by.end().push(0); // pushes (0, []) to downstream 28 | // group_by.end().push(1); // pushes (1, []) to downstream 29 | // group_by.push().push(std::make_pair(0, 42)); 30 | // group_by.push().push(std::make_pair(1, 14)); 31 | // group_by.end().push(0); // pushes (0, [42]) to downstream 32 | // group_by.end().push(1); // pushes (1, [14]) to downstream 33 | template 34 | class GroupBy { 35 | // TODO(mwhittaker): Think about whether we actually need to take ownership 36 | // of groups. I don't think we do. 37 | static_assert(!std::is_reference::value, 38 | "GroupBy takes ownership of group values, so they must be " 39 | "passed by value."); 40 | 41 | static_assert( 42 | !std::is_reference::value, 43 | "GroupBy takes ownership of values, so they must be passed by value."); 44 | 45 | static_assert( 46 | std::is_convertible>&&, To>::value, 47 | "GroupBy moves an std::pair> downstream, so it " 48 | "must be implicitly convertible to something of type To."); 49 | 50 | public: 51 | explicit GroupBy(Pushable* downstream) 52 | : downstream_(downstream), 53 | push_([this](std::pair p) { 54 | this->push_call(std::move(p)); 55 | }), 56 | end_([this](Group g) { this->end_call(std::move(g)); }) {} 57 | 58 | Pushable&&>& push() { return push_; } 59 | Pushable& end() { return end_; } 60 | 61 | private: 62 | void push_call(std::pair x) { 63 | groups_[std::move(x.first)].push_back(std::move(x.second)); 64 | } 65 | 66 | void end_call(Group group) { 67 | auto it = groups_.find(group); 68 | if (it == std::end(groups_)) { 69 | downstream_->push(std::make_pair(std::move(group), std::vector())); 70 | } else { 71 | std::vector xs = std::move(it->second); 72 | std::pair> p = 73 | std::make_pair(std::move(group), std::move(xs)); 74 | downstream_->push(std::move(p)); 75 | groups_.erase(it); 76 | } 77 | } 78 | 79 | Pushable* downstream_; 80 | using push_call_type = std::function)>; 81 | using end_call_type = std::function; 82 | Call&&, push_call_type> push_; 83 | Call end_; 84 | std::map> groups_; 85 | }; 86 | 87 | template 88 | GroupBy make_group_by(Pushable* downstream) { 89 | return GroupBy(downstream); 90 | } 91 | 92 | } // namespace latticeflow 93 | 94 | #endif // CLICK_MAP_H_ 95 | -------------------------------------------------------------------------------- /src/click/id.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_ID_H_ 2 | #define CLICK_ID_H_ 3 | 4 | #include 5 | 6 | #include "click/pushable.h" 7 | 8 | namespace latticeflow { 9 | 10 | // `Id(a).push(x)` performs `a.push(x)` where x is reference that is perfect 11 | // forwarded to `a`. 12 | template 13 | class Id : public Pushable { 14 | public: 15 | explicit Id(Pusher* downstream) : downstream_(downstream) {} 16 | 17 | void push(From&& x) override { downstream_->push(std::forward(x)); } 18 | 19 | private: 20 | Pusher* downstream_; 21 | }; 22 | 23 | } // namespace latticeflow 24 | 25 | #endif // CLICK_ID_H_ 26 | -------------------------------------------------------------------------------- /src/click/map.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_FILTER_H_ 2 | #define CLICK_FILTER_H_ 3 | 4 | #include 5 | 6 | #include "click/pushable.h" 7 | 8 | namespace latticeflow { 9 | 10 | // `Map(f, d).push(x)` performs `d->push(f(x))` where `x` is an lvalue or 11 | // rvalue reference and is forwarded to `f`. Note that the return of `f` is not 12 | // owned by the `Map` element, it is moved downstream. 13 | template 14 | class Map : public Pushable { 15 | public: 16 | Map(F&& f, Pushable* downstream) 17 | : f_(std::forward(f)), downstream_(downstream) {} 18 | 19 | void push(From&& x) override { downstream_->push(f_(std::forward(x))); } 20 | 21 | private: 22 | F f_; 23 | Pushable* downstream_; 24 | }; 25 | 26 | template 27 | Map make_map(F&& f, Pushable* downstream) { 28 | return Map(std::forward(f), downstream); 29 | } 30 | 31 | } // namespace latticeflow 32 | 33 | #endif // CLICK_FILTER_H_ 34 | -------------------------------------------------------------------------------- /src/click/pushable.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_PUSHABLE_H_ 2 | #define CLICK_PUSHABLE_H_ 3 | 4 | namespace latticeflow { 5 | 6 | // Pushables are the duals of iterators (which could be called Pullables). We 7 | // repeatedly *pull* values from iterators. We repeatedly *push* values to 8 | // Pushables. 9 | // 10 | // Note that Pushables are parametrized on a template parameter `T` which can 11 | // be instantiated with a lot of different types: 12 | // - int 13 | // - int& 14 | // - int&& 15 | // - const int 16 | // - const int& 17 | // - const int&& 18 | // - int* 19 | // - int** 20 | template 21 | class Pushable { 22 | public: 23 | virtual void push(T x) = 0; 24 | }; 25 | 26 | } // namespace latticeflow 27 | 28 | #endif // CLICK_PUSHABLE_H_ 29 | -------------------------------------------------------------------------------- /src/click/socket_recv.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_SOCKET_RECV_H_ 2 | #define CLICK_SOCKET_RECV_H_ 3 | 4 | #include "zmq.hpp" 5 | 6 | #include "click/event_handler.h" 7 | #include "click/pushable.h" 8 | #include "zmq_util/enveloped_message.h" 9 | #include "zmq_util/zmq_util.h" 10 | 11 | namespace latticeflow { 12 | 13 | // A `SocketRecv` polls on data arriving on a socket. When data arrives, it is 14 | // read into an `EnvelopedMessage` and moved to a downstream `Pushable`. 15 | template 16 | class SocketRecv : public EventHandler { 17 | public: 18 | SocketRecv(zmq::socket_t* socket, Pushable* downstream) 19 | : socket_(socket), downstream_(downstream) {} 20 | 21 | bool IsOneOff() const override { return false; } 22 | 23 | zmq::pollitem_t GetPollitem() override { 24 | return {*socket_, 0, ZMQ_POLLIN, 0}; 25 | } 26 | 27 | void Run(zmq::pollitem_t) override { 28 | downstream_->push(recv_enveloped_msg(socket_)); 29 | } 30 | 31 | private: 32 | zmq::socket_t* socket_; 33 | Pushable* downstream_; 34 | }; 35 | 36 | template 37 | SocketRecv make_socket_recv(zmq::socket_t* socket, 38 | Pushable* downstream) { 39 | return SocketRecv(socket, downstream); 40 | } 41 | 42 | } // namespace latticeflow 43 | 44 | #endif // CLICK_SOCKET_RECV_H_ 45 | -------------------------------------------------------------------------------- /src/click/socket_send.cc: -------------------------------------------------------------------------------- 1 | #include "click/socket_send.h" 2 | 3 | namespace latticeflow { 4 | 5 | SocketSend::SocketSend(zmq::socket_t* socket) : socket_(socket) {} 6 | 7 | void SocketSend::push(EnvelopedMessage e) { 8 | send_enveloped_msg(std::move(e), socket_); 9 | } 10 | 11 | } // namespace latticeflow 12 | -------------------------------------------------------------------------------- /src/click/socket_send.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_SOCKET_SEND_H_ 2 | #define CLICK_SOCKET_SEND_H_ 3 | 4 | #include 5 | 6 | #include "zmq.hpp" 7 | 8 | #include "click/pushable.h" 9 | #include "zmq_util/zmq_util.h" 10 | 11 | namespace latticeflow { 12 | 13 | // A `SocketSend` receives, and takes ownership of, packets and sends them out 14 | // on a socket. 15 | class SocketSend : public Pushable { 16 | public: 17 | explicit SocketSend(zmq::socket_t* socket); 18 | 19 | void push(EnvelopedMessage e) override; 20 | 21 | private: 22 | zmq::socket_t* socket_; 23 | }; 24 | 25 | } // namespace latticeflow 26 | #endif // CLICK_SOCKET_SEND_H_ 27 | -------------------------------------------------------------------------------- /src/click/tee.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_ROUND_TEE_H_ 2 | #define CLICK_ROUND_TEE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "click/pushable.h" 9 | 10 | namespace latticeflow { 11 | 12 | // `Tee`, inspired by the command line utility `tee`, copies its input to a 13 | // number of downstream `Pushable`s. For example, `Tee({a, b}).push(x)` 14 | // performs `a.push(x); b.push(x)`. Note that `x` is a universal reference that 15 | // is perfect forwarded to all downstream `Pushable`s. Moreover, downstream 16 | // `Pushable`s are pushed in the order they are provided to the constructor. 17 | // 18 | // Note that a Tee constructed from an empty list of downstreams acts like a 19 | // `Drop`. And a `Tee` constructed from a singleton list acts like `Id`. 20 | template 21 | class Tee : public Pushable { 22 | // TODO(mwhittaker): Figure out if we should include this. If `From` is 23 | // instantiated with something like `unique_ptr`, then this `static_assert` 24 | // is almost certainly catching a bug. However, if `From` is something like 25 | // `int`, then it can be moved from multiple times without any problem. 26 | static_assert(!std::is_rvalue_reference::value || 27 | std::is_reference::value, 28 | "You cannot create a `Tee` which forwards rvalue references to " 29 | "downstream `Pushable`s that take arguments by value. In most " 30 | "circumstances, this is a bug. You should not be " 31 | "move-constructing multiple objects from the same " 32 | "rvalue-reference."); 33 | 34 | public: 35 | explicit Tee(std::vector*> downstreams) 36 | : downstreams_(std::move(downstreams)) {} 37 | 38 | void push(From&& x) override { 39 | for (Pushable* downstream : downstreams_) { 40 | downstream->push(std::forward(x)); 41 | } 42 | } 43 | 44 | private: 45 | std::vector*> downstreams_; 46 | }; 47 | 48 | template 49 | Tee make_tee(std::vector*> downstreams) { 50 | return Tee{downstreams}; 51 | } 52 | 53 | } // namespace latticeflow 54 | 55 | #endif // CLICK_ROUND_TEE_H_ 56 | -------------------------------------------------------------------------------- /src/click/timer_file_descriptor.cc: -------------------------------------------------------------------------------- 1 | #include "click/timer_file_descriptor.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace { 8 | 9 | timespec nanoseconds_to_timespec(const std::chrono::nanoseconds d) { 10 | return {.tv_sec = d / std::chrono::seconds(1), 11 | .tv_nsec = (d % std::chrono::seconds(1)).count()}; 12 | } 13 | 14 | } // namespace 15 | 16 | TimerFileDescriptor::TimerFileDescriptor(std::chrono::nanoseconds value, 17 | std::chrono::nanoseconds interval) 18 | : value_(std::move(value)), interval_(std::move(interval)) { 19 | // TODO(mwhittaker): Handle errors better. 20 | assert(value != std::chrono::nanoseconds(0) || 21 | interval != std::chrono::nanoseconds(0)); 22 | timerfd_ = timerfd_create(CLOCK_REALTIME, 0 /* flags */); 23 | assert(timerfd_ != -1); 24 | const itimerspec new_value = { 25 | .it_interval = nanoseconds_to_timespec(interval_), 26 | .it_value = nanoseconds_to_timespec(value_), 27 | }; 28 | int err = timerfd_settime(timerfd_, 0, &new_value, nullptr); 29 | assert(err != -1); 30 | } 31 | 32 | TimerFileDescriptor::~TimerFileDescriptor() { 33 | if (timerfd_ != -1) { 34 | close(timerfd_); 35 | } 36 | } 37 | 38 | TimerFileDescriptor::TimerFileDescriptor(TimerFileDescriptor&& timer) 39 | : value_(std::move(timer.value_)), 40 | interval_(std::move(timer.interval_)), 41 | timerfd_(std::move(timer.timerfd_)) { 42 | timer.timerfd_ = -1; 43 | } 44 | 45 | void swap(TimerFileDescriptor& lhs, TimerFileDescriptor& rhs) { 46 | std::swap(lhs.value_, rhs.value_); 47 | std::swap(lhs.interval_, rhs.interval_); 48 | std::swap(lhs.timerfd_, rhs.timerfd_); 49 | } 50 | 51 | TimerFileDescriptor& TimerFileDescriptor::operator=(TimerFileDescriptor timer) { 52 | swap(*this, timer); 53 | return *this; 54 | } 55 | 56 | uint64_t TimerFileDescriptor::Read() { 57 | uint64_t num_expirations; 58 | // TODO(mwhittaker): Error checking. 59 | int err = read(timerfd_, &num_expirations, sizeof(num_expirations)); 60 | assert(err != -1); 61 | return num_expirations; 62 | } 63 | -------------------------------------------------------------------------------- /src/click/timer_file_descriptor.h: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_TIMER_FILE_DESCRIPTOR_H_ 2 | #define CLICK_TIMER_FILE_DESCRIPTOR_H_ 3 | 4 | #include 5 | #include 6 | 7 | // The `timerfd_create`, `timerfd_settime`, and `timerfd_gettime` system calls 8 | // [1] (included in "sys/timerfd.h") allow us to create one-off and periodic 9 | // timers which signal timer expiration through a file descriptor. Since the 10 | // timers send notifications through file descriptors, they can be polled. See 11 | // [2] for example usage. A `TimerFileDescriptor` constructs and uniquely owns 12 | // a timer file descriptor. 13 | // 14 | // [1]: http://bit.ly/2f4g8Nn 15 | // [2]: http://bit.ly/2fO3gwj 16 | class TimerFileDescriptor { 17 | public: 18 | // Constructs a timer which expires after `value` nanoseconds and then 19 | // expires every `interval` seconds. One of `value` or `interval` must be 20 | // non-zero. If `interval` is 0, then the timer fires once. If `value` is 0, 21 | // the timer fires for the first time after `interval` nanoseconds. 22 | TimerFileDescriptor(std::chrono::nanoseconds value, 23 | std::chrono::nanoseconds interval); 24 | 25 | // Close the file descriptor. 26 | ~TimerFileDescriptor(); 27 | 28 | // File descriptors are uniquely owned; they can be moved but not copied. 29 | TimerFileDescriptor(const TimerFileDescriptor& timer) = delete; 30 | TimerFileDescriptor(TimerFileDescriptor&& timer); 31 | 32 | // Assignment operator implemented using the copy-and-swap idiom [1]. 33 | // [1]: http://stackoverflow.com/a/3279550/3187068 34 | friend void swap(TimerFileDescriptor& lhs, TimerFileDescriptor& rhs); 35 | TimerFileDescriptor& operator=(TimerFileDescriptor timer); 36 | 37 | // Return the underlying value, interval, or timer file descriptor. 38 | const std::chrono::nanoseconds& value() const { return value_; } 39 | const std::chrono::nanoseconds& interval() const { return interval_; } 40 | int timerfd() const { return timerfd_; } 41 | 42 | // Read from the file descriptor the number of times the timer has expired. 43 | // This can only be called after the timer has expired. If you're polling the 44 | // timer, you must call this after the event is triggered. If you don't the 45 | // event will stay triggered. 46 | uint64_t Read(); 47 | 48 | private: 49 | std::chrono::nanoseconds value_; 50 | std::chrono::nanoseconds interval_; 51 | int timerfd_; 52 | }; 53 | 54 | #endif // CLICK_TIMER_FILE_DESCRIPTOR_H_ 55 | -------------------------------------------------------------------------------- /src/cmake/benchmarking.cmake: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | # CREATE_BENCHMARK(foo) creates a benchmark named `foo` from the file `foo.cc`. 4 | MACRO(CREATE_BENCHMARK NAME) 5 | ADD_EXECUTABLE(${NAME} ${NAME}.cc) 6 | ADD_DEPENDENCIES(${NAME} googlebenchmark) 7 | TARGET_LINK_LIBRARIES(${NAME} 8 | ${GBENCH_LIBS_DIR}/libbenchmark.a 9 | pthread 10 | ) 11 | ENDMACRO(CREATE_BENCHMARK) 12 | -------------------------------------------------------------------------------- /src/cmake/testing.cmake: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | # CREATE_TEST(foo) creates a test named `foo` from the file `foo.cc`. 4 | MACRO(CREATE_TEST NAME) 5 | ADD_EXECUTABLE(${NAME} ${NAME}.cc) 6 | ADD_TEST(${NAME} ${NAME}) 7 | ADD_DEPENDENCIES(${NAME} googletest) 8 | TARGET_LINK_LIBRARIES(${NAME} 9 | ${GTEST_LIBS_DIR}/libgtest.a 10 | ${GTEST_LIBS_DIR}/libgtest_main.a 11 | pthread 12 | ) 13 | ENDMACRO(CREATE_TEST) 14 | 15 | # CREATE_NAMED_TEST(foo bar.cc) creates a test named `foo` from the file 16 | # `bar.cc`. 17 | MACRO(CREATE_NAMED_TEST NAME FILENAME) 18 | ADD_EXECUTABLE(${NAME} ${FILENAME}) 19 | ADD_TEST(${NAME} ${NAME}) 20 | ADD_DEPENDENCIES(${NAME} googletest) 21 | TARGET_LINK_LIBRARIES(${NAME} 22 | ${GTEST_LIBS_DIR}/libgtest.a 23 | ${GTEST_LIBS_DIR}/libgtest_main.a 24 | pthread 25 | ) 26 | ENDMACRO(CREATE_NAMED_TEST) 27 | -------------------------------------------------------------------------------- /src/concurrency/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | ADD_LIBRARY(concurrency 4 | barrier.cc 5 | thread_pool.cc 6 | ) 7 | 8 | CREATE_TEST(bounded_queue_test) 9 | CREATE_TEST(queue_test) 10 | CREATE_TEST(thread_pool_test) 11 | 12 | TARGET_LINK_LIBRARIES(thread_pool_test concurrency) 13 | -------------------------------------------------------------------------------- /src/concurrency/README.md: -------------------------------------------------------------------------------- 1 | # Concurrency 2 | This directory contains a handy set of C++ concurrency and synchronization 3 | primitives built using standard C++11 concurrency and synchronization 4 | primitives. For example, `queue.h` implements an unbounded concurrent queue 5 | (sometimes called a channel or pipe). 6 | 7 | ## Resources 8 | - [C++ Concurrency in 9 | Action](https://www.manning.com/books/c-plus-plus-concurrency-in-action) 10 | -------------------------------------------------------------------------------- /src/concurrency/barrier.cc: -------------------------------------------------------------------------------- 1 | #include "concurrency/barrier.h" 2 | 3 | void Barrier::enter() { 4 | std::unique_lock lock(m_); 5 | count_++; 6 | 7 | // If we're the last thread to enter the barrier, wake up everyone else. 8 | if (count_ >= threshold_) { 9 | all_done_.notify_all(); 10 | return; 11 | } 12 | 13 | // Otherwise, wait for the last thread to arrive and wake us up. 14 | while (count_ < threshold_) { 15 | all_done_.wait(lock); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/concurrency/barrier.h: -------------------------------------------------------------------------------- 1 | #ifndef CONCURRENCY_BARRIER_H_ 2 | #define CONCURRENCY_BARRIER_H_ 3 | 4 | #include 5 | #include 6 | 7 | // A barrier is a synchronization primitive such that threads enter the barrier 8 | // and block until some specified threshold of threads have entered the 9 | // barrier. Pictorially, consider four threads that work for a while (denoted 10 | // by |) and then enter the barrier and wait (denoted by :). Once all threads 11 | // enter the barrier, they are woken continue working again. 12 | // 13 | // a b c d 14 | // | | | | \ working 15 | // | | | | / 16 | // | | | : \ 17 | // : | | : } 18 | // : | | : } waiting 19 | // : : | : } 20 | // : : | : / 21 | // ---------------- barrier 22 | // | | | | 23 | // | | | | 24 | // 25 | // Note that once a barrier's threshold has been met, it cannot be reused. 26 | class Barrier { 27 | public: 28 | // Construct a barrier that waits for `threshold` threads to enter. 29 | explicit Barrier(const int threshold) : count_(0), threshold_(threshold) {} 30 | Barrier(const Barrier& barrier) = delete; 31 | Barrier& operator=(const Barrier& barrier) = delete; 32 | 33 | // Enter the barrier. 34 | void enter(); 35 | 36 | private: 37 | int count_; 38 | int threshold_; 39 | std::mutex m_; 40 | std::condition_variable all_done_; 41 | }; 42 | 43 | #endif // CONCURRENCY_BARRIER_H_ 44 | -------------------------------------------------------------------------------- /src/concurrency/bounded_queue.h: -------------------------------------------------------------------------------- 1 | #ifndef CONCURRENCY_BOUNDED_QUEUE_H_ 2 | #define CONCURRENCY_BOUNDED_QUEUE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "boost/optional.hpp" 9 | 10 | template 11 | class BoundedQueue { 12 | public: 13 | BoundedQueue(std::size_t capacity) : capacity_(capacity) { 14 | // TODO(mwhittaker): Assert that capacity_ > 0; 15 | } 16 | BoundedQueue(const BoundedQueue&) = delete; 17 | BoundedQueue& operator=(const BoundedQueue&) = delete; 18 | 19 | void push(const T& x) { 20 | std::unique_lock l(m_); 21 | while (xs_.size() == capacity_) { 22 | space_available_.wait(l); 23 | } 24 | xs_.push_back(x); 25 | data_available_.notify_one(); 26 | } 27 | 28 | T pop() { 29 | std::unique_lock l(m_); 30 | while (xs_.size() == 0) { 31 | data_available_.wait(l); 32 | } 33 | const T x = xs_.front(); 34 | xs_.erase(std::begin(xs_)); 35 | space_available_.notify_one(); 36 | return x; 37 | } 38 | 39 | boost::optional try_pop() { 40 | std::unique_lock l(m_); 41 | if (xs_.size() > 0) { 42 | const T x = xs_.front(); 43 | xs_.erase(std::begin(xs_)); 44 | space_available_.notify_one(); 45 | return x; 46 | } else { 47 | return {}; 48 | } 49 | } 50 | 51 | void clear() { 52 | std::unique_lock l(m_); 53 | xs_.clear(); 54 | space_available_.notify_all(); 55 | } 56 | 57 | private: 58 | std::vector xs_; 59 | const std::size_t capacity_; 60 | std::mutex m_; 61 | std::condition_variable data_available_; 62 | std::condition_variable space_available_; 63 | }; 64 | 65 | #endif // CONCURRENCY_BOUNDED_QUEUE_H_ 66 | -------------------------------------------------------------------------------- /src/concurrency/bounded_queue_test.cc: -------------------------------------------------------------------------------- 1 | #include "concurrency/bounded_queue.h" 2 | 3 | #include "gtest/gtest.h" 4 | 5 | namespace latticeflow { 6 | 7 | TEST(BoundedQueue, BoundedSimpleQueue) { 8 | BoundedQueue q(10); 9 | q.push(1); 10 | EXPECT_EQ(1, q.pop()); 11 | } 12 | 13 | TEST(BoundedQueue, BoundedComplexQueue) { 14 | BoundedQueue q(100); 15 | 16 | for (int i = 1; i <= 100; ++i) { 17 | q.push(i); 18 | EXPECT_EQ(i, q.pop()); 19 | } 20 | 21 | for (int i = 1; i <= 100; ++i) { 22 | q.push(i); 23 | EXPECT_EQ(i, q.try_pop()); 24 | } 25 | 26 | for (int i = 1; i <= 100; ++i) { 27 | q.push(i); 28 | } 29 | q.clear(); 30 | for (int i = 1; i <= 100; ++i) { 31 | q.push(i); 32 | } 33 | for (int i = 1; i <= 100; ++i) { 34 | EXPECT_EQ(i, q.pop()); 35 | } 36 | } 37 | 38 | } // namespace latticeflow 39 | 40 | int main(int argc, char** argv) { 41 | ::testing::InitGoogleTest(&argc, argv); 42 | return RUN_ALL_TESTS(); 43 | } 44 | -------------------------------------------------------------------------------- /src/concurrency/queue.h: -------------------------------------------------------------------------------- 1 | #ifndef CONCURRENCY_QUEUE_H_ 2 | #define CONCURRENCY_QUEUE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "boost/optional.hpp" 9 | 10 | template 11 | class Queue { 12 | public: 13 | Queue() = default; 14 | explicit Queue(std::vector xs) : xs_(std::move(xs)) {} 15 | Queue(const Queue&) = delete; 16 | Queue& operator=(const Queue&) = delete; 17 | 18 | void push(const T& x) { 19 | std::unique_lock l(m_); 20 | xs_.push_back(x); 21 | data_available_.notify_one(); 22 | } 23 | 24 | T pop() { 25 | std::unique_lock l(m_); 26 | while (xs_.size() == 0) { 27 | data_available_.wait(l); 28 | } 29 | const T x = xs_.front(); 30 | xs_.erase(std::begin(xs_)); 31 | return x; 32 | } 33 | 34 | boost::optional try_pop() { 35 | std::unique_lock l(m_); 36 | if (xs_.size() > 0) { 37 | const T x = xs_.front(); 38 | xs_.erase(std::begin(xs_)); 39 | return x; 40 | } else { 41 | return {}; 42 | } 43 | } 44 | 45 | void clear() { 46 | std::unique_lock l(m_); 47 | xs_.clear(); 48 | } 49 | 50 | private: 51 | std::mutex m_; 52 | std::condition_variable data_available_; 53 | std::vector xs_; 54 | }; 55 | 56 | #endif // CONCURRENCY_QUEUE_H_ 57 | -------------------------------------------------------------------------------- /src/concurrency/queue_test.cc: -------------------------------------------------------------------------------- 1 | #include "concurrency/queue.h" 2 | 3 | #include "gtest/gtest.h" 4 | 5 | namespace latticeflow { 6 | 7 | TEST(Queue, SimpleQueue) { 8 | Queue q; 9 | q.push(1); 10 | EXPECT_EQ(1, q.pop()); 11 | q.push(2); 12 | EXPECT_EQ(2, q.try_pop()); 13 | } 14 | 15 | TEST(Queue, ComplexQueue) { 16 | Queue q({1, 2, 3, 4, 5}); 17 | 18 | for (int i = 1; i <= 5; ++i) { 19 | EXPECT_EQ(i, q.pop()); 20 | } 21 | 22 | for (int i = 1; i <= 100; ++i) { 23 | q.push(i); 24 | EXPECT_EQ(i, q.pop()); 25 | } 26 | 27 | for (int i = 1; i <= 100; ++i) { 28 | q.push(i); 29 | EXPECT_EQ(i, q.try_pop()); 30 | } 31 | 32 | for (int i = 1; i <= 100; ++i) { 33 | q.push(i); 34 | } 35 | q.clear(); 36 | for (int i = 1; i <= 100; ++i) { 37 | q.push(i); 38 | } 39 | for (int i = 1; i <= 100; ++i) { 40 | EXPECT_EQ(i, q.pop()); 41 | } 42 | } 43 | 44 | } // namespace latticeflow 45 | 46 | int main(int argc, char** argv) { 47 | ::testing::InitGoogleTest(&argc, argv); 48 | return RUN_ALL_TESTS(); 49 | } 50 | -------------------------------------------------------------------------------- /src/concurrency/thread_pool.cc: -------------------------------------------------------------------------------- 1 | #include "concurrency/thread_pool.h" 2 | 3 | namespace latticeflow { 4 | 5 | ThreadPool::ThreadPool(const int num_threads) { 6 | for (int i = 0; i < num_threads; ++i) { 7 | threads_.push_back(std::thread(&ThreadPool::DoWork, this)); 8 | } 9 | } 10 | 11 | ThreadPool::~ThreadPool() { 12 | for (std::size_t i = 0; i < threads_.size(); ++i) { 13 | work_.push({}); 14 | } 15 | for (std::thread& t : threads_) { 16 | t.join(); 17 | } 18 | } 19 | 20 | void ThreadPool::submit(const std::function& f) { work_.push(f); } 21 | 22 | void ThreadPool::DoWork() { 23 | for (boost::optional> f = work_.pop(); f; 24 | f = work_.pop()) { 25 | (*f)(); 26 | } 27 | } 28 | 29 | } // namespace latticeflow 30 | -------------------------------------------------------------------------------- /src/concurrency/thread_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CONCURRENCY_THREAD_POOL_H_ 2 | #define CONCURRENCY_THREAD_POOL_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "concurrency/queue.h" 9 | 10 | namespace latticeflow { 11 | 12 | class ThreadPool { 13 | public: 14 | explicit ThreadPool(const int num_threads); 15 | ~ThreadPool(); 16 | ThreadPool(const ThreadPool& barrier) = delete; 17 | ThreadPool& operator=(const ThreadPool& barrier) = delete; 18 | 19 | void submit(const std::function& f); 20 | 21 | private: 22 | void DoWork(); 23 | 24 | std::vector threads_; 25 | Queue>> work_; 26 | }; 27 | 28 | } // namespace latticeflow 29 | 30 | #endif // CONCURRENCY_THREAD_POOL_H_ 31 | -------------------------------------------------------------------------------- /src/concurrency/thread_pool_test.cc: -------------------------------------------------------------------------------- 1 | #include "concurrency/thread_pool.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "gtest/gtest.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(ThreadPool, SimpleThreadPool) { 11 | constexpr int num_threads = 100; 12 | std::array xs; 13 | 14 | { 15 | ThreadPool p(num_threads); 16 | for (int i = 0; i < num_threads; ++i) { 17 | int* x = &xs[i]; 18 | p.submit([i, x]() { *x = i * i; }); 19 | } 20 | } 21 | 22 | int sum = 0; 23 | for (int i = 0; i < num_threads; ++i) { 24 | sum += i * i; 25 | } 26 | EXPECT_EQ(sum, std::accumulate(std::begin(xs), std::end(xs), 0)); 27 | } 28 | 29 | } // namespace latticeflow 30 | 31 | int main(int argc, char** argv) { 32 | ::testing::InitGoogleTest(&argc, argv); 33 | return RUN_ALL_TESTS(); 34 | } 35 | -------------------------------------------------------------------------------- /src/flow/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | CREATE_TEST(cross_test) 4 | CREATE_TEST(empty_test) 5 | CREATE_TEST(exchange_test) 6 | CREATE_TEST(filter_test) 7 | CREATE_TEST(iterator_test) 8 | CREATE_TEST(map_test) 9 | CREATE_TEST(naive_equijoin_test) 10 | 11 | TARGET_LINK_LIBRARIES(cross_test boost_context) 12 | TARGET_LINK_LIBRARIES(naive_equijoin_test boost_context) 13 | 14 | CREATE_BENCHMARK(filter_bench) 15 | -------------------------------------------------------------------------------- /src/flow/README.md: -------------------------------------------------------------------------------- 1 | # Flow 2 | This directory includes a collection of relational iterators: iterators which 3 | iterate over tuples. All iterators implement the [`Operator` 4 | interface](operator.h). Some are implemented using 5 | [coroutines](coroutine_operator.h). We also include a [naive exchange 6 | operator](exchange.h) to parallelize a dataflow. The code also uses C++ 7 | variadic templates to allow for generic iteration of tuples. 8 | -------------------------------------------------------------------------------- /src/flow/coroutine_operator.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_COROUTINE_OPERATOR_H_ 2 | #define FLOW_COROUTINE_OPERATOR_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "boost/coroutine2/all.hpp" 8 | #include "boost/optional.hpp" 9 | 10 | #include "flow/operator.h" 11 | 12 | namespace latticeflow { 13 | 14 | // TODO(mwhittaker): Document. 15 | template 16 | class CoroutineOperator : public Operator { 17 | public: 18 | // TODO(mwhittaker): Document why we can't construct the coroutine_ with the 19 | // correct lambda yet. 20 | CoroutineOperator() 21 | : coroutine_([this](push_type&) {}), 22 | coroutine_iterator_(begin(coroutine_)) {} 23 | 24 | boost::optional> next() override { 25 | if (coroutine_iterator_ != end(coroutine_)) { 26 | std::tuple p = *coroutine_iterator_; 27 | ++coroutine_iterator_; 28 | return p; 29 | } else { 30 | return {}; 31 | } 32 | } 33 | 34 | virtual void reset() override { 35 | coroutine_ = pull_type([this](push_type& yield) { YieldNext(yield); }); 36 | coroutine_iterator_ = begin(coroutine_); 37 | } 38 | 39 | protected: 40 | using coroutine_type = boost::coroutines2::coroutine>; 41 | using push_type = typename coroutine_type::push_type; 42 | using pull_type = typename coroutine_type::pull_type; 43 | using iterator = typename pull_type::iterator; 44 | 45 | private: 46 | virtual void YieldNext(push_type& yield) = 0; 47 | 48 | pull_type coroutine_; 49 | iterator coroutine_iterator_; 50 | }; 51 | 52 | }; // namespace latticeflow 53 | 54 | #endif // FLOW_COROUTINE_OPERATOR_H_ 55 | -------------------------------------------------------------------------------- /src/flow/cross.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_CROSS_H_ 2 | #define FLOW_CROSS_H_ 3 | 4 | #include 5 | 6 | #include "flow/coroutine_operator.h" 7 | #include "flow/operator.h" 8 | #include "flow/template_helpers.h" 9 | 10 | namespace latticeflow { 11 | 12 | // http://stackoverflow.com/a/21023429/3187068 13 | // TODO(mwhittaker): Understand this, then document it. 14 | template 15 | class Cross; 16 | 17 | template 18 | class Cross, right> 19 | : public CoroutineOperator { 20 | public: 21 | Cross(Operator* const left, Operator* const right) 22 | : left_(left), right_(right) { 23 | reset(); 24 | } 25 | Cross(const Cross, right>&) = delete; 26 | Cross& operator=(const Cross, right>&) = delete; 27 | 28 | void reset() override { 29 | left_->reset(); 30 | right_->reset(); 31 | CoroutineOperator::reset(); 32 | } 33 | 34 | private: 35 | using push_type = typename CoroutineOperator::push_type; 36 | 37 | void YieldNext(push_type& yield) override { 38 | for (auto l = left_->next(); l; l = left_->next()) { 39 | for (auto r = right_->next(); r; r = right_->next()) { 40 | yield(std::tuple_cat(*l, *r)); 41 | } 42 | right_->reset(); 43 | } 44 | } 45 | 46 | Operator* const left_; 47 | Operator* const right_; 48 | }; 49 | 50 | } // namespace latticeflow 51 | 52 | #endif // FLOW_CROSS_H_ 53 | -------------------------------------------------------------------------------- /src/flow/cross_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/cross.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "boost/optional.hpp" 8 | #include "gtest/gtest.h" 9 | 10 | #include "flow/iterator.h" 11 | #include "flow/template_helpers.h" 12 | 13 | namespace latticeflow { 14 | 15 | TEST(Cross, SimpleCross) { 16 | using tuple = std::tuple; 17 | std::vector> ls = {1, 2, 3}; 18 | std::vector> rs = {"a", "b"}; 19 | Iterator l(std::begin(ls), std::end(ls)); 20 | Iterator r(std::begin(rs), std::end(rs)); 21 | Cross, right> crossed(&l, &r); 22 | 23 | EXPECT_EQ(boost::optional(std::make_tuple(1, "a")), crossed.next()); 24 | EXPECT_EQ(boost::optional(std::make_tuple(1, "b")), crossed.next()); 25 | EXPECT_EQ(boost::optional(std::make_tuple(2, "a")), crossed.next()); 26 | EXPECT_EQ(boost::optional(std::make_tuple(2, "b")), crossed.next()); 27 | EXPECT_EQ(boost::optional(std::make_tuple(3, "a")), crossed.next()); 28 | EXPECT_EQ(boost::optional(std::make_tuple(3, "b")), crossed.next()); 29 | EXPECT_EQ(boost::optional(), crossed.next()); 30 | 31 | crossed.reset(); 32 | EXPECT_EQ(boost::optional(std::make_tuple(1, "a")), crossed.next()); 33 | EXPECT_EQ(boost::optional(std::make_tuple(1, "b")), crossed.next()); 34 | EXPECT_EQ(boost::optional(std::make_tuple(2, "a")), crossed.next()); 35 | EXPECT_EQ(boost::optional(std::make_tuple(2, "b")), crossed.next()); 36 | EXPECT_EQ(boost::optional(std::make_tuple(3, "a")), crossed.next()); 37 | EXPECT_EQ(boost::optional(std::make_tuple(3, "b")), crossed.next()); 38 | EXPECT_EQ(boost::optional(), crossed.next()); 39 | } 40 | 41 | } // namespace latticeflow 42 | 43 | int main(int argc, char** argv) { 44 | ::testing::InitGoogleTest(&argc, argv); 45 | return RUN_ALL_TESTS(); 46 | } 47 | -------------------------------------------------------------------------------- /src/flow/empty.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_EMPTY_H_ 2 | #define FLOW_EMPTY_H_ 3 | 4 | #include "flow/operator.h" 5 | 6 | namespace latticeflow { 7 | 8 | template 9 | class Empty : public Operator { 10 | public: 11 | Empty() = default; 12 | Empty(const Empty&) = delete; 13 | Empty& operator=(const Empty&) = delete; 14 | 15 | boost::optional> next() override { return {}; } 16 | 17 | void reset() override {} 18 | }; 19 | 20 | } // namespace latticeflow 21 | 22 | #endif // FLOW_EMPTY_H_ 23 | -------------------------------------------------------------------------------- /src/flow/empty_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/empty.h" 2 | 3 | #include 4 | 5 | #include "boost/optional.hpp" 6 | #include "gtest/gtest.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(Empty, SimpleEmpty) { 11 | using tuple = std::tuple; 12 | Empty empty; 13 | EXPECT_EQ(boost::optional(), empty.next()); 14 | empty.reset(); 15 | EXPECT_EQ(boost::optional(), empty.next()); 16 | } 17 | 18 | TEST(Empty, ComplexEmpty) { 19 | using tuple = std::tuple; 20 | Empty empty; 21 | EXPECT_EQ(boost::optional(), empty.next()); 22 | empty.reset(); 23 | EXPECT_EQ(boost::optional(), empty.next()); 24 | } 25 | 26 | } // namespace latticeflow 27 | 28 | int main(int argc, char **argv) { 29 | ::testing::InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | -------------------------------------------------------------------------------- /src/flow/exchange.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_EXCHANGE_H_ 2 | #define FLOW_EXCHANGE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "concurrency/queue.h" 9 | #include "flow/operator.h" 10 | 11 | namespace { 12 | 13 | enum class Signal { 14 | RESET = 0, 15 | DIE = 1, 16 | }; 17 | 18 | } // namespace 19 | 20 | namespace latticeflow { 21 | 22 | template 23 | class Exchange : public Operator { 24 | public: 25 | Exchange(Operator* const child) 26 | : child_(child), 27 | num_resets_(0), 28 | child_thread_(&Exchange::ChildPopulateBuffer, this) {} 29 | 30 | ~Exchange() { 31 | signals_.push(Signal::DIE); 32 | child_thread_.join(); 33 | } 34 | 35 | Exchange(const Exchange&) = delete; 36 | Exchange& operator=(const Exchange&) = delete; 37 | 38 | boost::optional> next() override { 39 | auto timestamped_tuple = buffer_.pop(); 40 | while (timestamped_tuple.first != num_resets_) { 41 | timestamped_tuple = buffer_.pop(); 42 | } 43 | return timestamped_tuple.second; 44 | } 45 | 46 | void reset() override { 47 | num_resets_++; 48 | signals_.push(Signal::RESET); 49 | } 50 | 51 | private: 52 | void ChildPopulateBuffer() { 53 | int num_resets = 0; 54 | 55 | while (true) { 56 | for (auto t = child_->next(); t; t = child_->next()) { 57 | buffer_.push(std::make_pair(num_resets, t)); 58 | 59 | boost::optional signal = signals_.try_pop(); 60 | if (signal) { 61 | switch (*signal) { 62 | case Signal::RESET: { 63 | buffer_.clear(); 64 | child_->reset(); 65 | num_resets++; 66 | } 67 | case Signal::DIE: { 68 | return; 69 | } 70 | } 71 | } 72 | } 73 | buffer_.push( 74 | std::make_pair(num_resets, boost::optional>())); 75 | Signal signal = signals_.pop(); 76 | switch (signal) { 77 | case Signal::RESET: { 78 | child_->reset(); 79 | num_resets++; 80 | break; 81 | } 82 | case Signal::DIE: { 83 | return; 84 | } 85 | } 86 | } 87 | } 88 | 89 | Operator* const child_; 90 | int num_resets_; 91 | Queue>>> buffer_; 92 | Queue signals_; 93 | std::thread child_thread_; 94 | }; 95 | 96 | } // namespace latticeflow 97 | 98 | #endif // FLOW_EXCHANGE_H_ 99 | -------------------------------------------------------------------------------- /src/flow/exchange_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/exchange.h" 2 | 3 | #include 4 | 5 | #include "boost/optional.hpp" 6 | #include "gtest/gtest.h" 7 | 8 | #include "flow/iterator.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Exchange, SimplExchange) { 13 | using tuple = std::tuple; 14 | std::vector xs = {{1}, {2}, {3}}; 15 | Iterator v(std::begin(xs), std::end(xs)); 16 | Exchange exchange(&v); 17 | 18 | EXPECT_EQ(boost::optional(std::make_tuple(1)), exchange.next()); 19 | EXPECT_EQ(boost::optional(std::make_tuple(2)), exchange.next()); 20 | EXPECT_EQ(boost::optional(std::make_tuple(3)), exchange.next()); 21 | EXPECT_EQ(boost::optional(), exchange.next()); 22 | 23 | exchange.reset(); 24 | EXPECT_EQ(boost::optional(std::make_tuple(1)), exchange.next()); 25 | EXPECT_EQ(boost::optional(std::make_tuple(2)), exchange.next()); 26 | EXPECT_EQ(boost::optional(std::make_tuple(3)), exchange.next()); 27 | EXPECT_EQ(boost::optional(), exchange.next()); 28 | } 29 | 30 | } // namespace latticeflow 31 | 32 | int main(int argc, char **argv) { 33 | ::testing::InitGoogleTest(&argc, argv); 34 | return RUN_ALL_TESTS(); 35 | } 36 | -------------------------------------------------------------------------------- /src/flow/filter.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_FILTER_H_ 2 | #define FLOW_FILTER_H_ 3 | 4 | #include 5 | 6 | #include "flow/operator.h" 7 | 8 | namespace latticeflow { 9 | 10 | template 11 | class Filter : public Operator { 12 | public: 13 | Filter(Operator* const child, 14 | std::function&)> filter) 15 | : child_(child), filter_(filter) {} 16 | Filter(const Filter&) = delete; 17 | Filter& operator=(const Filter&) = delete; 18 | 19 | boost::optional> next() override { 20 | boost::optional> x = child_->next(); 21 | while (x) { 22 | if (filter_(*x)) { 23 | return x; 24 | } 25 | x = child_->next(); 26 | } 27 | return {}; 28 | } 29 | 30 | void reset() override { child_->reset(); } 31 | 32 | private: 33 | Operator* const child_; 34 | std::function&)> filter_; 35 | }; 36 | 37 | } // namespace latticeflow 38 | 39 | #endif // FLOW_FILTER_H_ 40 | -------------------------------------------------------------------------------- /src/flow/filter_bench.cc: -------------------------------------------------------------------------------- 1 | #include "flow/filter.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "benchmark/benchmark.h" 8 | 9 | #include "flow/iterator.h" 10 | 11 | namespace latticeflow { 12 | 13 | void TrueFilter(benchmark::State& state) { 14 | while (state.KeepRunning()) { 15 | state.PauseTiming(); 16 | std::vector> xs(state.range_x()); 17 | std::iota(std::begin(xs), std::end(xs), 0); 18 | Iterator v(std::begin(xs), std::end(xs)); 19 | Filter f(&v, [](const std::tuple&) { return true; }); 20 | state.ResumeTiming(); 21 | for (auto t = f.next(); t; t = f.next()) { 22 | // Do nothing. 23 | } 24 | } 25 | } 26 | BENCHMARK(TrueFilter)->Range(10, 10 << 10); 27 | 28 | } // namespace latticeflow 29 | 30 | int main(int argc, char** argv) { 31 | benchmark::Initialize(&argc, argv); 32 | benchmark::RunSpecifiedBenchmarks(); 33 | return 0; 34 | } 35 | -------------------------------------------------------------------------------- /src/flow/filter_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/filter.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | #include "gtest/gtest.h" 8 | 9 | #include "flow/iterator.h" 10 | 11 | namespace latticeflow { 12 | 13 | TEST(Filter, SimpleFilter) { 14 | using tuple = std::tuple; 15 | std::vector xs = {{1}, {2}, {3}, {4}, {5}, {6}, {7}}; 16 | Iterator::iterator, int> v(std::begin(xs), std::end(xs)); 17 | Filter evens(&v, [](const tuple& x) { return std::get<0>(x) % 2 == 0; }); 18 | 19 | EXPECT_EQ(boost::optional(std::make_tuple(2)), evens.next()); 20 | EXPECT_EQ(boost::optional(std::make_tuple(4)), evens.next()); 21 | EXPECT_EQ(boost::optional(std::make_tuple(6)), evens.next()); 22 | EXPECT_EQ(boost::optional(), evens.next()); 23 | 24 | evens.reset(); 25 | EXPECT_EQ(boost::optional(std::make_tuple(2)), evens.next()); 26 | EXPECT_EQ(boost::optional(std::make_tuple(4)), evens.next()); 27 | EXPECT_EQ(boost::optional(std::make_tuple(6)), evens.next()); 28 | EXPECT_EQ(boost::optional(), evens.next()); 29 | } 30 | 31 | TEST(Filter, ComplexFilter) { 32 | using tuple = std::tuple; 33 | std::vector xs = {{1, 'a'}, {2, 'b'}, {3, 'c'}, {4, 'd'}, 34 | {5, 'e'}, {6, 'f'}, {7, 'g'}}; 35 | Iterator::iterator, int, char> v(std::begin(xs), 36 | std::end(xs)); 37 | Filter filtered(&v, [](const tuple& x) { 38 | return std::get<0>(x) % 2 == 0 && std::get<1>(x) != 'd'; 39 | }); 40 | 41 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), filtered.next()); 42 | EXPECT_EQ(boost::optional(std::make_tuple(6, 'f')), filtered.next()); 43 | EXPECT_EQ(boost::optional(), filtered.next()); 44 | 45 | filtered.reset(); 46 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), filtered.next()); 47 | EXPECT_EQ(boost::optional(std::make_tuple(6, 'f')), filtered.next()); 48 | EXPECT_EQ(boost::optional(), filtered.next()); 49 | } 50 | 51 | } // namespace latticeflow 52 | 53 | int main(int argc, char** argv) { 54 | ::testing::InitGoogleTest(&argc, argv); 55 | return RUN_ALL_TESTS(); 56 | } 57 | -------------------------------------------------------------------------------- /src/flow/iterator.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_ITERATOR_H_ 2 | #define FLOW_ITERATOR_H_ 3 | 4 | #include "flow/operator.h" 5 | 6 | namespace latticeflow { 7 | 8 | template 9 | class Iterator : public Operator { 10 | public: 11 | explicit Iterator(const ForwardIterator& begin, const ForwardIterator& end) 12 | : begin_(begin), end_(end), it_(begin) {} 13 | Iterator(const Iterator&) = delete; 14 | Iterator& operator=(const Iterator&) = delete; 15 | 16 | boost::optional> next() override { 17 | if (it_ != end_) { 18 | boost::optional> o(*it_); 19 | it_++; 20 | return o; 21 | } else { 22 | return {}; 23 | } 24 | } 25 | 26 | void reset() override { it_ = begin_; } 27 | 28 | private: 29 | const ForwardIterator begin_; 30 | const ForwardIterator end_; 31 | ForwardIterator it_; 32 | }; 33 | 34 | } // namespace latticeflow 35 | 36 | #endif // FLOW_ITERATOR_H_ 37 | -------------------------------------------------------------------------------- /src/flow/iterator_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/iterator.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "boost/optional.hpp" 8 | #include "gtest/gtest.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Iterator, ComplexVectorIterator) { 13 | using tuple = std::tuple; 14 | std::vector v = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; 15 | Iterator::iterator, int, char> it(std::begin(v), 16 | std::end(v)); 17 | 18 | EXPECT_EQ(boost::optional(std::make_tuple(1, 'a')), it.next()); 19 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), it.next()); 20 | EXPECT_EQ(boost::optional(std::make_tuple(3, 'c')), it.next()); 21 | EXPECT_EQ(boost::optional(), it.next()); 22 | 23 | it.reset(); 24 | EXPECT_EQ(boost::optional(std::make_tuple(1, 'a')), it.next()); 25 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), it.next()); 26 | EXPECT_EQ(boost::optional(std::make_tuple(3, 'c')), it.next()); 27 | EXPECT_EQ(boost::optional(), it.next()); 28 | } 29 | 30 | TEST(Iterator, ComplexSetIterator) { 31 | using tuple = std::tuple; 32 | std::set s = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; 33 | Iterator::iterator, int, char> it(std::begin(s), std::end(s)); 34 | 35 | EXPECT_EQ(boost::optional(std::make_tuple(1, 'a')), it.next()); 36 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), it.next()); 37 | EXPECT_EQ(boost::optional(std::make_tuple(3, 'c')), it.next()); 38 | EXPECT_EQ(boost::optional(), it.next()); 39 | 40 | it.reset(); 41 | EXPECT_EQ(boost::optional(std::make_tuple(1, 'a')), it.next()); 42 | EXPECT_EQ(boost::optional(std::make_tuple(2, 'b')), it.next()); 43 | EXPECT_EQ(boost::optional(std::make_tuple(3, 'c')), it.next()); 44 | EXPECT_EQ(boost::optional(), it.next()); 45 | } 46 | 47 | } // namespace latticeflow 48 | 49 | int main(int argc, char **argv) { 50 | ::testing::InitGoogleTest(&argc, argv); 51 | return RUN_ALL_TESTS(); 52 | } 53 | -------------------------------------------------------------------------------- /src/flow/map.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_MAP_H_ 2 | #define FLOW_MAP_H_ 3 | 4 | #include 5 | 6 | #include "flow/operator.h" 7 | #include "flow/template_helpers.h" 8 | 9 | namespace latticeflow { 10 | 11 | template 12 | class Map {}; 13 | 14 | template 15 | class Map, to> : public Operator { 16 | public: 17 | Map(Operator* const child, 18 | std::function(const std::tuple&)> f) 19 | : child_(child), f_(f) {} 20 | Map(const Map, to>&) = delete; 21 | Map& operator=(const Map, to>&) = delete; 22 | 23 | boost::optional> next() override { 24 | boost::optional> x = child_->next(); 25 | if (x) { 26 | return f_(*x); 27 | } else { 28 | return {}; 29 | } 30 | } 31 | 32 | void reset() override { child_->reset(); } 33 | 34 | private: 35 | Operator* const child_; 36 | std::function(const std::tuple&)> f_; 37 | }; 38 | 39 | } // namespace latticeflow 40 | 41 | #endif // FLOW_MAP_H_ 42 | -------------------------------------------------------------------------------- /src/flow/map_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/map.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | #include "gtest/gtest.h" 8 | 9 | #include "flow/iterator.h" 10 | #include "flow/template_helpers.h" 11 | 12 | namespace latticeflow { 13 | 14 | std::tuple f(const std::tuple& t) { 15 | int x = std::get<0>(t); 16 | return std::make_tuple(x, std::to_string(x)); 17 | } 18 | 19 | TEST(Map, SimpleMap) { 20 | using tuple = std::tuple; 21 | std::vector> xs = {{1}, {2}, {3}, {4}}; 22 | Iterator>::iterator, int> v(std::begin(xs), 23 | std::end(xs)); 24 | Map, to> mapped(&v, f); 25 | 26 | EXPECT_EQ(boost::optional(std::make_tuple(1, "1")), mapped.next()); 27 | EXPECT_EQ(boost::optional(std::make_tuple(2, "2")), mapped.next()); 28 | EXPECT_EQ(boost::optional(std::make_tuple(3, "3")), mapped.next()); 29 | EXPECT_EQ(boost::optional(std::make_tuple(4, "4")), mapped.next()); 30 | EXPECT_EQ(boost::optional(), mapped.next()); 31 | 32 | mapped.reset(); 33 | EXPECT_EQ(boost::optional(std::make_tuple(1, "1")), mapped.next()); 34 | EXPECT_EQ(boost::optional(std::make_tuple(2, "2")), mapped.next()); 35 | EXPECT_EQ(boost::optional(std::make_tuple(3, "3")), mapped.next()); 36 | EXPECT_EQ(boost::optional(std::make_tuple(4, "4")), mapped.next()); 37 | EXPECT_EQ(boost::optional(), mapped.next()); 38 | } 39 | 40 | } // namespace latticeflow 41 | 42 | int main(int argc, char** argv) { 43 | ::testing::InitGoogleTest(&argc, argv); 44 | return RUN_ALL_TESTS(); 45 | } 46 | -------------------------------------------------------------------------------- /src/flow/naive_equijoin.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_NAIVE_EQUIJOIN_H_ 2 | #define FLOW_NAIVE_EQUIJOIN_H_ 3 | 4 | #include 5 | 6 | #include "flow/coroutine_operator.h" 7 | #include "flow/operator.h" 8 | #include "flow/template_helpers.h" 9 | #include "flow/tuple_helpers.h" 10 | 11 | namespace latticeflow { 12 | 13 | // http://stackoverflow.com/a/21023429/3187068 14 | // TODO(mwhittaker): Understand this, then document it. 15 | template 16 | class NaiveEquijoin; 17 | 18 | template 19 | class NaiveEquijoin, right, LeftIndex, RightIndex> 20 | : public CoroutineOperator { 21 | public: 22 | NaiveEquijoin(Operator* const left, 23 | Operator* const right) 24 | : left_(left), right_(right) { 25 | reset(); 26 | } 27 | NaiveEquijoin(const NaiveEquijoin&) = delete; 28 | NaiveEquijoin& operator=(const NaiveEquijoin&) = delete; 29 | void reset() override { 30 | left_->reset(); 31 | right_->reset(); 32 | CoroutineOperator::reset(); 33 | } 34 | 35 | private: 36 | using push_type = typename CoroutineOperator::push_type; 37 | 38 | void YieldNext(push_type& yield) override { 39 | for (auto l = left_->next(); l; l = left_->next()) { 40 | for (auto r = right_->next(); r; r = right_->next()) { 41 | if (std::get(*l) == std::get(*r)) { 42 | yield(std::tuple_cat(*l, *r)); 43 | } 44 | } 45 | right_->reset(); 46 | } 47 | } 48 | 49 | Operator* const left_; 50 | Operator* const right_; 51 | }; 52 | 53 | } // namespace latticeflow 54 | 55 | #endif // FLOW_NAIVE_EQUIJOIN_H_ 56 | -------------------------------------------------------------------------------- /src/flow/naive_equijoin_test.cc: -------------------------------------------------------------------------------- 1 | #include "flow/naive_equijoin.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "boost/optional.hpp" 8 | #include "gtest/gtest.h" 9 | 10 | #include "flow/iterator.h" 11 | #include "flow/template_helpers.h" 12 | 13 | namespace latticeflow { 14 | 15 | TEST(NaiveEquijoin, SimpleNaiveEquijoin) { 16 | using tuple = std::tuple; 17 | std::vector> ls = { 18 | {1, true}, {2, false}, {2, true}, {3, true}}; 19 | std::vector> rs = { 20 | {2, 'a'}, {2, 'b'}, {3, 'c'}, {4, 'd'}}; 21 | Iterator l(std::begin(ls), std::end(ls)); 22 | Iterator r(std::begin(rs), std::end(rs)); 23 | NaiveEquijoin, right, 0, 0> joined(&l, &r); 24 | 25 | EXPECT_EQ(boost::optional(std::make_tuple(2, false, 2, 'a')), 26 | joined.next()); 27 | EXPECT_EQ(boost::optional(std::make_tuple(2, false, 2, 'b')), 28 | joined.next()); 29 | EXPECT_EQ(boost::optional(std::make_tuple(2, true, 2, 'a')), 30 | joined.next()); 31 | EXPECT_EQ(boost::optional(std::make_tuple(2, true, 2, 'b')), 32 | joined.next()); 33 | EXPECT_EQ(boost::optional(std::make_tuple(3, true, 3, 'c')), 34 | joined.next()); 35 | EXPECT_EQ(boost::optional(), joined.next()); 36 | 37 | joined.reset(); 38 | EXPECT_EQ(boost::optional(std::make_tuple(2, false, 2, 'a')), 39 | joined.next()); 40 | EXPECT_EQ(boost::optional(std::make_tuple(2, false, 2, 'b')), 41 | joined.next()); 42 | EXPECT_EQ(boost::optional(std::make_tuple(2, true, 2, 'a')), 43 | joined.next()); 44 | EXPECT_EQ(boost::optional(std::make_tuple(2, true, 2, 'b')), 45 | joined.next()); 46 | EXPECT_EQ(boost::optional(std::make_tuple(3, true, 3, 'c')), 47 | joined.next()); 48 | EXPECT_EQ(boost::optional(), joined.next()); 49 | } 50 | 51 | } // namespace latticeflow 52 | 53 | int main(int argc, char** argv) { 54 | ::testing::InitGoogleTest(&argc, argv); 55 | return RUN_ALL_TESTS(); 56 | } 57 | -------------------------------------------------------------------------------- /src/flow/operator.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_OPERATOR_H_ 2 | #define FLOW_OPERATOR_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | namespace latticeflow { 9 | 10 | template 11 | class Operator { 12 | public: 13 | virtual boost::optional> next() = 0; 14 | virtual void reset() = 0; 15 | }; 16 | 17 | } // namespace latticeflow 18 | 19 | #endif // FLOW_OPERATOR_H_ 20 | -------------------------------------------------------------------------------- /src/flow/template_helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_TEMPLATE_HELPERS_H_ 2 | #define FLOW_TEMPLATE_HELPERS_H_ 3 | 4 | namespace latticeflow { 5 | 6 | // TODO(mwhittaker): Document. 7 | template 8 | struct left {}; 9 | 10 | // TODO(mwhittaker): Document. 11 | template 12 | struct right {}; 13 | 14 | // TODO(mwhittaker): Document. 15 | template 16 | struct from {}; 17 | 18 | // TODO(mwhittaker): Document. 19 | template 20 | struct to {}; 21 | 22 | } // namespace latticeflow 23 | 24 | #endif // FLOW_TEMPLATE_HELPERS_H_ 25 | -------------------------------------------------------------------------------- /src/flow/tuple_helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef FLOW_TUPLE_HELPERS_H_ 2 | #define FLOW_TUPLE_HELPERS_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace latticeflow { 9 | 10 | template 11 | typename std::enable_if::type print_tuple( 12 | std::ostream& stream, const std::tuple& t) { 13 | stream << std::get(t); 14 | } 15 | 16 | template 17 | typename std::enable_if < 18 | N::type print_tuple(std::ostream& stream, 19 | const std::tuple& t) { 20 | stream << std::get(t) << ", "; 21 | print_tuple(stream, t); 22 | } 23 | 24 | template 25 | std::ostream& operator<<(std::ostream& stream, const std::tuple& t) { 26 | stream << "("; 27 | print_tuple<0>(stream, t); 28 | stream << ")"; 29 | return stream; 30 | } 31 | 32 | } // namespace latticeflow 33 | 34 | #endif // FLOW_TUPLE_HELPERS_H_ 35 | -------------------------------------------------------------------------------- /src/key_value_stores/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | # Generate protocol buffers. 4 | INCLUDE(FindProtobuf) 5 | FIND_PACKAGE(Protobuf REQUIRED) 6 | INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) 7 | PROTOBUF_GENERATE_CPP(PROTO_SRC PROTO_HEADER message.proto) 8 | 9 | # The library dependencies of the key value store clients and servers. 10 | SET(KV_LIBRARY_DEPENDENCIES 11 | click 12 | concurrency 13 | protobuf 14 | pthread 15 | zmq 16 | zmq_util 17 | ) 18 | 19 | # The source dependencies of the key value store clients and servers. 20 | SET(KV_SRC_DEPENDENCIES 21 | ${PROTO_SRC} 22 | client.cc 23 | util.cc 24 | zmq_util.cc 25 | ) 26 | 27 | # The message broker that forwards traffic between clients and servers. 28 | ADD_EXECUTABLE(msgqueue msgqueue.cc) 29 | TARGET_LINK_LIBRARIES(msgqueue ${KV_LIBRARY_DEPENDENCIES}) 30 | 31 | # Read uncommitted client. 32 | ADD_EXECUTABLE(ruc_client ruc_client.cc ${KV_SRC_DEPENDENCIES}) 33 | TARGET_LINK_LIBRARIES(ruc_client ${KV_LIBRARY_DEPENDENCIES}) 34 | 35 | # Read committed client. 36 | ADD_EXECUTABLE(rc_client rc_client.cc ${KV_SRC_DEPENDENCIES}) 37 | TARGET_LINK_LIBRARIES(rc_client ${KV_LIBRARY_DEPENDENCIES}) 38 | 39 | # Isolation cut isolation client. 40 | ADD_EXECUTABLE(ici_client ici_client.cc ${KV_SRC_DEPENDENCIES}) 41 | TARGET_LINK_LIBRARIES(ici_client ${KV_LIBRARY_DEPENDENCIES}) 42 | 43 | # Key-value server. 44 | ADD_EXECUTABLE(server server.cc ${KV_SRC_DEPENDENCIES}) 45 | TARGET_LINK_LIBRARIES(server ${KV_LIBRARY_DEPENDENCIES}) 46 | -------------------------------------------------------------------------------- /src/key_value_stores/README.md: -------------------------------------------------------------------------------- 1 | # Key Value Stores 2 | This directory includes a generic key-value server and a set of key-value 3 | clients implemented using the lattices in [`lattices`](../lattices). Each 4 | key-value client implements one of the weak isolation levels described in 5 | [*Highly Available Transactions: Virtues and Limitations*][hat]. For example, 6 | [`ruc_client.cc`](ruc_client.cc) implements supports READ UNCOMMITTED 7 | transactions. 8 | 9 | ## Overview 10 | | Isolation Level | Client | 11 | | ------------------ | -------------------------------- | 12 | | READ UNCOMMITTED | [`ruc_client.cc`](ruc_client.cc) | 13 | | READ COMMITTED | [`rc_client.cc`](rc_client.cc) | 14 | | Item Cut Isolation | [`ici_client.cc`](ici_client.cc) | 15 | 16 | ## Architecture 17 | This directory contains a key-value server, a set of key-value clients, and a 18 | message broker which passes messages between the two. 19 | 20 | ### Message Broker 21 | A message broker (see [`msgqueue.cc`](msgqueue.cc)) is a simple program that 22 | can connect any number of client sockets to any number of server sockets: 23 | 24 | clients servers 25 | ======= ======= 26 | 27 | +---+ +---+ 28 | | a |\ /| 1 | 29 | +---+ \ / +---+ 30 | \ / 31 | +---+ \ +----------+ / +---+ 32 | | b |---->| msgqueue |<-----| 2 | 33 | +---+ / +----------+ \ +---+ 34 | / \ 35 | +---+ / \ +---+ 36 | | c |/ \| 3 | 37 | +---+ +---+ 38 | 39 | Messages sent by client sockets are fairly queued and sent in a round-robin 40 | fashion to the server sockets. Here, for example, messages would be sent to 1, 41 | then 2, then 3, then 1, then 2, and so on. When a server socket receives a 42 | message, it can respond with a message of its own that is routed back to the 43 | original sender. 44 | 45 | ### Client 46 | Clients repeatedly 47 | 48 | - read commands from stdin (e.g. `BEGIN TRANSACTION`, `GET x`, `PUT x foo`), 49 | - parse the commands, 50 | - serialize them into protocol buffers, 51 | - send the protos to the message broker, 52 | - await a response from a server, and 53 | - pretty print the response. 54 | 55 | Different clients perform different tricks to achieve their respective 56 | isolation level. 57 | 58 | ### Server 59 | The server runs some number of threads, and each thread maintains a copy of a 60 | key-value store represented by a map from strings to timestamped strings; this 61 | happens to be a semilattice. For example, 62 | 63 | { 64 | // key: (timestamp, value) 65 | "foo": (42, "moo"), 66 | "bar": (10, "meow"), 67 | ... 68 | } 69 | 70 | Each server thread forms a connection to the message broker so that it can 71 | receive messages from and send messages to the clients. It also forms a clique 72 | of pub-sub connections with the other threads in order to gossip updates to the 73 | key-value store. These pub-sub connections are made within the process and pass 74 | data around via pointers to data allocated on the heap. Currently, gossip is 75 | only between threads, not between servers. 76 | 77 | [hat]: https://scholar.google.com/scholar?cluster=5290590427752770563&hl=en&as_sdt=0,5 78 | -------------------------------------------------------------------------------- /src/key_value_stores/client.cc: -------------------------------------------------------------------------------- 1 | #include "key_value_stores/client.h" 2 | 3 | #include 4 | 5 | #include "key_value_stores/message.pb.h" 6 | #include "key_value_stores/util.h" 7 | 8 | namespace { 9 | 10 | std::string usage() { 11 | return "usage: \n" 12 | " BEGIN TRANSACTION\n" 13 | " GET \n" 14 | " PUT \n" 15 | " END TRANSACTION"; 16 | } 17 | 18 | } // namespace 19 | 20 | void repl(Client *const client) { 21 | while (true) { 22 | std::cout << "> " << std::flush; 23 | std::string input; 24 | getline(std::cin, input); 25 | std::vector ss; 26 | split(input, ' ', &ss); 27 | 28 | if (ss.size() == 2 && ss[0] == "BEGIN" && ss[1] == "TRANSACTION") { 29 | client->begin(); 30 | } else if (ss.size() == 2 && ss[0] == "GET") { 31 | client->get(ss[1]); 32 | } else if (ss.size() == 3 && ss[0] == "PUT") { 33 | client->put(ss[1], ss[2]); 34 | } else if (ss.size() == 2 && ss[0] == "END" && ss[1] == "TRANSACTION") { 35 | client->end(); 36 | } else { 37 | std::cout << "Invalid command: " << input << std::endl; 38 | std::cout << usage() << std::endl; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/key_value_stores/client.h: -------------------------------------------------------------------------------- 1 | #ifndef KEY_VALUE_STORES_CLIENT_H_ 2 | #define KEY_VALUE_STORES_CLIENT_H_ 3 | 4 | #include 5 | 6 | // A key-value store client. Each client processes four types of requests: 7 | // 1. BEGIN TRANSACTION 8 | // 2. GET 9 | // 3. PUT 10 | // 4. END TRANSACTION 11 | class Client { 12 | public: 13 | virtual void begin() = 0; 14 | virtual void get(const std::string& k) = 0; 15 | virtual void put(const std::string& k, const std::string& v) = 0; 16 | virtual void end() = 0; 17 | }; 18 | 19 | // Parses commands from stdin, calling the methods in `Client` when 20 | // appropriate. 21 | void repl(Client* client); 22 | 23 | #endif // KEY_VALUE_STORES_CLIENT_H_ 24 | -------------------------------------------------------------------------------- /src/key_value_stores/ici_client.cc: -------------------------------------------------------------------------------- 1 | // Item Cut Isolation key-value client. 2 | // 3 | // In Highly Available Transactions: Virtues and Limitations [1], Bailis et al. 4 | // explain that isolation cut isolation transactions can be achieved by having 5 | // clients cache the values of reads, returning the cached values on subsequent 6 | // reads. 7 | // 8 | // [1]: http://www.bailis.org/papers/hat-vldb2014.pdf 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "key_value_stores/client.h" 19 | #include "key_value_stores/message.pb.h" 20 | #include "key_value_stores/util.h" 21 | #include "key_value_stores/zmq_util.h" 22 | 23 | class IciClient : public Client { 24 | public: 25 | explicit IciClient(zmq::socket_t* socket) 26 | : socket_(socket), current_timestamp_(-1) {} 27 | 28 | void begin() { 29 | communication::Request request; 30 | request.mutable_begin_transaction(); 31 | send_proto(request, socket_); 32 | 33 | communication::Response response; 34 | recv_proto(&response, socket_); 35 | if (response.succeed()) { 36 | current_timestamp_ = response.timestamp(); 37 | std::cout << current_timestamp_ << std::endl; 38 | } else { 39 | std::cout << "ERROR" << std::endl; 40 | } 41 | } 42 | 43 | void get(const std::string& k) { 44 | if (write_buffer_.count(k) != 0) { 45 | std::cout << write_buffer_[k] << std::endl; 46 | } else if (read_buffer_.count(k) != 0) { 47 | std::cout << read_buffer_[k] << std::endl; 48 | } else { 49 | communication::Request request; 50 | request.mutable_get()->set_key(k); 51 | send_proto(request, socket_); 52 | 53 | communication::Response response; 54 | recv_proto(&response, socket_); 55 | if (response.succeed()) { 56 | read_buffer_[k] = response.value(); 57 | std::cout << response.value() << std::endl; 58 | } else { 59 | std::cout << "ERROR" << std::endl; 60 | } 61 | } 62 | } 63 | 64 | void put(const std::string& k, const std::string& v) { 65 | if (current_timestamp_ == -1) { 66 | std::cout << "First run BEGIN TRANSACTION." << std::endl; 67 | } else { 68 | write_buffer_[k] = v; 69 | } 70 | } 71 | 72 | void end() { 73 | communication::Request request; 74 | request.mutable_put()->set_timestamp(current_timestamp_); 75 | for (const std::pair& kv : write_buffer_) { 76 | communication::Request::Put::KeyValuePair* p = 77 | request.mutable_put()->add_kv_pair(); 78 | p->set_key(std::get<0>(kv)); 79 | p->set_value(std::get<1>(kv)); 80 | } 81 | send_proto(request, socket_); 82 | 83 | communication::Response response; 84 | recv_proto(&response, socket_); 85 | if (!response.succeed()) { 86 | std::cout << "ERROR" << std::endl; 87 | } 88 | write_buffer_.clear(); 89 | read_buffer_.clear(); 90 | } 91 | 92 | private: 93 | zmq::socket_t* const socket_; 94 | int current_timestamp_; 95 | std::unordered_map write_buffer_; 96 | std::unordered_map read_buffer_; 97 | }; 98 | 99 | int main() { 100 | zmq::context_t context(1); 101 | zmq::socket_t socket(context, ZMQ_REQ); 102 | socket.connect("tcp://localhost:5559"); 103 | std::cout << "client connected to " 104 | << "tcp://localhost:5559" << std::endl; 105 | 106 | IciClient client(&socket); 107 | repl(&client); 108 | } 109 | -------------------------------------------------------------------------------- /src/key_value_stores/message.proto: -------------------------------------------------------------------------------- 1 | package communication; 2 | 3 | // A request to a key-value store. 4 | message Request { 5 | // A request to begin a transaction with the key-value store. 6 | message BeginTransaction {} 7 | 8 | // A request to get a particular value from the key-value store. 9 | message Get { 10 | required string key = 1; 11 | } 12 | 13 | // A request to install a timestamped batch of key-value pairs into the 14 | // key-value sotre. 15 | message Put { 16 | message KeyValuePair { 17 | required string key = 1; 18 | required string value = 2; 19 | } 20 | required int64 timestamp = 1; 21 | repeated KeyValuePair kv_pair = 2; 22 | } 23 | 24 | oneof request { 25 | BeginTransaction begin_transaction = 1; 26 | Get get = 2; 27 | Put put = 3; 28 | } 29 | } 30 | 31 | // A response from a key-value store. 32 | message Response { 33 | // True if the RPC succeeded; 34 | optional bool succeed = 1; 35 | 36 | // The response to a PUT request; unset otherwise. 37 | optional string value = 2; 38 | 39 | // The response to a BEGIN_TRANSACTION request; unset otherwise. 40 | optional int64 timestamp = 3; 41 | } 42 | -------------------------------------------------------------------------------- /src/key_value_stores/msgqueue.cc: -------------------------------------------------------------------------------- 1 | // Copied and modified from http://zguide.zeromq.org/cpp:msgqueue. 2 | 3 | // 4 | // Simple message queuing broker in C++ 5 | // Same as request-reply broker but using QUEUE device 6 | // 7 | // Olivier Chamoux 8 | 9 | #include 10 | 11 | #include 12 | 13 | int main() { 14 | zmq::context_t context(1); 15 | 16 | // Socket facing clients 17 | zmq::socket_t frontend(context, ZMQ_ROUTER); 18 | frontend.bind("tcp://*:5559"); 19 | 20 | // Socket facing services 21 | zmq::socket_t backend(context, ZMQ_DEALER); 22 | backend.bind("tcp://*:5560"); 23 | 24 | std::cout << "message broker connecting clients on tcp://*5559 to servers on " 25 | << "tcp://*:5560" << std::endl; 26 | 27 | // Start the proxy 28 | zmq::proxy(frontend, backend, nullptr); 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /src/key_value_stores/rc_client.cc: -------------------------------------------------------------------------------- 1 | // READ COMMITTED key-value client. 2 | // 3 | // In Highly Available Transactions: Virtues and Limitations [1], Bailis et al. 4 | // explain that read committed transactions can be achieved by having servers 5 | // simply not write any uncommitted modifications to readable state. The 6 | // simplest way to achieve this is to have a client buffer its updates locally 7 | // before sending them in a single batch to the server. 8 | // 9 | // [1]: http://www.bailis.org/papers/hat-vldb2014.pdf 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | 19 | #include "key_value_stores/client.h" 20 | #include "key_value_stores/message.pb.h" 21 | #include "key_value_stores/util.h" 22 | #include "key_value_stores/zmq_util.h" 23 | 24 | class RcClient : public Client { 25 | public: 26 | explicit RcClient(zmq::socket_t *socket) 27 | : socket_(socket), current_timestamp_(-1) {} 28 | 29 | void begin() { 30 | communication::Request request; 31 | request.mutable_begin_transaction(); 32 | send_proto(request, socket_); 33 | 34 | communication::Response response; 35 | recv_proto(&response, socket_); 36 | if (response.succeed()) { 37 | current_timestamp_ = response.timestamp(); 38 | std::cout << current_timestamp_ << std::endl; 39 | } else { 40 | std::cout << "ERROR" << std::endl; 41 | } 42 | } 43 | 44 | void get(const std::string &k) { 45 | if (write_buffer_.count(k) != 0) { 46 | std::cout << write_buffer_[k] << std::endl; 47 | } else { 48 | communication::Request request; 49 | request.mutable_get()->set_key(k); 50 | send_proto(request, socket_); 51 | 52 | communication::Response response; 53 | recv_proto(&response, socket_); 54 | if (response.succeed()) { 55 | std::cout << response.value() << std::endl; 56 | } else { 57 | std::cout << "ERROR" << std::endl; 58 | } 59 | } 60 | } 61 | 62 | void put(const std::string &k, const std::string &v) { 63 | if (current_timestamp_ == -1) { 64 | std::cout << "First run BEGIN TRANSACTION." << std::endl; 65 | } else { 66 | write_buffer_[k] = v; 67 | } 68 | } 69 | 70 | void end() { 71 | communication::Request request; 72 | request.mutable_put()->set_timestamp(current_timestamp_); 73 | for (const std::pair &kv : write_buffer_) { 74 | communication::Request::Put::KeyValuePair *p = 75 | request.mutable_put()->add_kv_pair(); 76 | p->set_key(std::get<0>(kv)); 77 | p->set_value(std::get<1>(kv)); 78 | } 79 | send_proto(request, socket_); 80 | 81 | communication::Response response; 82 | recv_proto(&response, socket_); 83 | if (!response.succeed()) { 84 | std::cout << "ERROR" << std::endl; 85 | } 86 | write_buffer_.clear(); 87 | } 88 | 89 | private: 90 | zmq::socket_t *const socket_; 91 | int current_timestamp_; 92 | std::unordered_map write_buffer_; 93 | }; 94 | 95 | int main() { 96 | zmq::context_t context(1); 97 | zmq::socket_t socket(context, ZMQ_REQ); 98 | socket.connect("tcp://localhost:5559"); 99 | std::cout << "client connected to " 100 | << "tcp://localhost:5559" << std::endl; 101 | 102 | RcClient client(&socket); 103 | repl(&client); 104 | } 105 | -------------------------------------------------------------------------------- /src/key_value_stores/ruc_client.cc: -------------------------------------------------------------------------------- 1 | // READ UNCOMMITTED key-value client. 2 | // 3 | // In Highly Available Transactions: Virtues and Limitations [1], Bailis et al. 4 | // explain that read uncommitted transactions can be achieved by annotating 5 | // each transaction with a globally unique id and enforcing a last-write-wins 6 | // policy on the servers. 7 | // 8 | // [1]: http://www.bailis.org/papers/hat-vldb2014.pdf 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #include "key_value_stores/client.h" 17 | #include "key_value_stores/message.pb.h" 18 | #include "key_value_stores/util.h" 19 | #include "key_value_stores/zmq_util.h" 20 | 21 | class RucClient : public Client { 22 | public: 23 | explicit RucClient(zmq::socket_t* socket) 24 | : socket_(socket), current_timestamp_(-1) {} 25 | 26 | void begin() override { 27 | communication::Request request; 28 | request.mutable_begin_transaction(); 29 | send_proto(request, socket_); 30 | 31 | communication::Response response; 32 | recv_proto(&response, socket_); 33 | if (response.succeed()) { 34 | current_timestamp_ = response.timestamp(); 35 | std::cout << current_timestamp_ << std::endl; 36 | } else { 37 | std::cout << "ERROR" << std::endl; 38 | } 39 | } 40 | 41 | void get(const std::string& k) override { 42 | communication::Request request; 43 | request.mutable_get()->set_key(k); 44 | send_proto(request, socket_); 45 | 46 | communication::Response response; 47 | recv_proto(&response, socket_); 48 | if (response.succeed()) { 49 | std::cout << response.value() << std::endl; 50 | } else { 51 | std::cout << "ERROR" << std::endl; 52 | } 53 | } 54 | 55 | void put(const std::string& k, const std::string& v) override { 56 | if (current_timestamp_ == -1) { 57 | std::cout << "First run BEGIN TRANSACTION." << std::endl; 58 | } else { 59 | communication::Request request; 60 | request.mutable_put()->set_timestamp(current_timestamp_); 61 | communication::Request::Put::KeyValuePair* kv_pair = 62 | request.mutable_put()->add_kv_pair(); 63 | kv_pair->set_key(k); 64 | kv_pair->set_value(v); 65 | request.mutable_put()->set_timestamp(current_timestamp_); 66 | send_proto(request, socket_); 67 | 68 | communication::Response response; 69 | recv_proto(&response, socket_); 70 | if (!response.succeed()) { 71 | std::cout << "ERROR" << std::endl; 72 | } 73 | } 74 | } 75 | 76 | void end() override { 77 | std::cout << "END TRANSACTION not supported" << std::endl; 78 | } 79 | 80 | private: 81 | zmq::socket_t* const socket_; 82 | int current_timestamp_; 83 | }; 84 | 85 | int main() { 86 | zmq::context_t context(1); 87 | zmq::socket_t socket(context, ZMQ_REQ); 88 | socket.connect("tcp://localhost:5559"); 89 | std::cout << "client connected to " 90 | << "tcp://localhost:5559" << std::endl; 91 | 92 | RucClient client(&socket); 93 | repl(&client); 94 | } 95 | -------------------------------------------------------------------------------- /src/key_value_stores/util.cc: -------------------------------------------------------------------------------- 1 | #include "key_value_stores/util.h" 2 | 3 | #include 4 | #include 5 | 6 | void split(const std::string& s, char delim, std::vector* elems) { 7 | elems->clear(); 8 | std::stringstream ss(s); 9 | std::string item; 10 | while (std::getline(ss, item, delim)) { 11 | elems->push_back(item); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/key_value_stores/util.h: -------------------------------------------------------------------------------- 1 | #ifndef KEY_VALUE_STORES_UTIL_H_ 2 | #define KEY_VALUE_STORES_UTIL_H_ 3 | 4 | #include 5 | #include 6 | 7 | // `split(s, c)` splits the string `s` using the delimeter `c` and stores the 8 | // parts in `elems`. 9 | void split(const std::string& s, char delim, std::vector* elems); 10 | 11 | #endif // KEY_VALUE_STORES_UTIL_H_ 12 | -------------------------------------------------------------------------------- /src/key_value_stores/zmq_util.cc: -------------------------------------------------------------------------------- 1 | #include "key_value_stores/zmq_util.h" 2 | 3 | std::string message_to_string(const zmq::message_t& message) { 4 | return std::string(static_cast(message.data()), message.size()); 5 | } 6 | 7 | void send_string(const std::string& s, zmq::socket_t* socket) { 8 | zmq::message_t message(s.size()); 9 | memcpy(message.data(), s.c_str(), s.size()); 10 | socket->send(message); 11 | } 12 | 13 | std::string recv_string(zmq::socket_t* socket) { 14 | zmq::message_t message; 15 | socket->recv(&message); 16 | return message_to_string(message); 17 | } 18 | -------------------------------------------------------------------------------- /src/key_value_stores/zmq_util.h: -------------------------------------------------------------------------------- 1 | #ifndef KEY_VALUE_STORES_ZMQ_UTIL_H_ 2 | #define KEY_VALUE_STORES_ZMQ_UTIL_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | // Converts the data within a `zmq::message_t` into a string. 10 | std::string message_to_string(const zmq::message_t& message); 11 | 12 | // `send` a string over the socket. 13 | void send_string(const std::string& s, zmq::socket_t* socket); 14 | 15 | // `recv` a string over the socket. 16 | std::string recv_string(zmq::socket_t* socket); 17 | 18 | // Serialize a proto and `send` it over the socket. 19 | template 20 | void send_proto(const RequestProto& request, zmq::socket_t* socket) { 21 | std::string request_str; 22 | request.SerializeToString(&request_str); 23 | zmq::message_t request_msg(request_str.size()); 24 | memcpy(request_msg.data(), request_str.c_str(), request_str.size()); 25 | socket->send(request_msg); 26 | } 27 | 28 | // `recv` a message and unserialize it into a proto. 29 | template 30 | void recv_proto(ResponseProto* reply, zmq::socket_t* socket) { 31 | zmq::message_t reply_msg; 32 | socket->recv(&reply_msg); 33 | std::string reply_str = message_to_string(reply_msg); 34 | reply->ParseFromString(reply_str); 35 | } 36 | 37 | // `send` a pointer over the socket. 38 | template 39 | void send_pointer(const T* const p, zmq::socket_t* socket) { 40 | zmq::message_t message(sizeof(const T* const)); 41 | memcpy(message.data(), &p, sizeof(const T* const)); 42 | socket->send(message); 43 | } 44 | 45 | // `recv` a pointer over the socket. 46 | template 47 | T* recv_pointer(zmq::socket_t* socket) { 48 | zmq::message_t message; 49 | socket->recv(&message); 50 | // NOLINT: this code is somewhat forced to be hacky due to the low-level 51 | // nature of the zeromq API. 52 | return *reinterpret_cast(message.data()); // NOLINT 53 | } 54 | 55 | #endif // KEY_VALUE_STORES_ZMQ_UTIL_H_ 56 | -------------------------------------------------------------------------------- /src/lattices/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | CREATE_TEST(array_lattice_test) 4 | CREATE_TEST(bool_and_lattice_test) 5 | CREATE_TEST(bool_or_lattice_test) 6 | CREATE_TEST(map_lattice_test) 7 | CREATE_TEST(max_lattice_test) 8 | CREATE_TEST(min_lattice_test) 9 | CREATE_TEST(pair_lattice_test) 10 | CREATE_TEST(set_intersect_lattice_test) 11 | CREATE_TEST(set_union_lattice_test) 12 | CREATE_TEST(timestamp_lattice_test) 13 | CREATE_TEST(vector_lattice_test) 14 | -------------------------------------------------------------------------------- /src/lattices/README.md: -------------------------------------------------------------------------------- 1 | # Lattices 2 | This directory implements a library of [join semilattices][semilattice_wiki]. 3 | The properties of a semilattice make them particularly useful in eventually 4 | consistent distributed systems (e.g. [Bloom^L][blooml_paper], 5 | [CRDTs][crdt_paper]). 6 | 7 | [blooml_paper]: https://scholar.google.com/scholar?cluster=1332747912204910097&hl=en&as_sdt=0,5 8 | [crdt_paper]: https://scholar.google.com/scholar?cluster=13367952773539942258&hl=en&as_sdt=0,5 9 | [semilattice_wiki]: https://en.wikipedia.org/wiki/Semilattice 10 | -------------------------------------------------------------------------------- /src/lattices/array_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_ARRAY_LATTICE_H_ 2 | #define LATTICES_ARRAY_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "lattices/lattice.h" 10 | 11 | namespace latticeflow { 12 | 13 | // Consider an arbitrary lattice T = (S_T, join_T). We can form an array 14 | // lattice of size N A = (S_t ^ N, join_A) where 15 | // 16 | // join_A([x1, ..., xN], [y1, ..., yN]) = [x1 join_T y1, ..., xN join_T yN] 17 | // 18 | // That is, array semillatices are merged pairwise. 19 | template 20 | class ArrayLattice : public Lattice, std::array> { 21 | static_assert(std::is_base_of, T>::value, 22 | "Type of ArrayLattice does not inherit from Lattice."); 23 | 24 | public: 25 | ArrayLattice() = default; 26 | explicit ArrayLattice(std::array xs) : xs_(xs) {} 27 | ArrayLattice(const ArrayLattice& l) = default; 28 | ArrayLattice& operator=(const ArrayLattice& l) = default; 29 | 30 | const std::array& get() const override { return xs_; } 31 | 32 | void join(const ArrayLattice& l) override { 33 | for (std::size_t i = 0; i < N; ++i) { 34 | xs_[i].join(l.xs_[i]); 35 | } 36 | } 37 | 38 | private: 39 | std::array xs_; 40 | }; 41 | 42 | } // namespace latticeflow 43 | 44 | #endif // LATTICES_ARRAY_LATTICE_H_ 45 | -------------------------------------------------------------------------------- /src/lattices/array_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/array_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "lattices/bool_or_lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | using BoolArrayLattice = ArrayLattice; 12 | 13 | TEST(BoolArrayLattice, Basics) { 14 | BoolOrLattice tru(true); 15 | BoolOrLattice fls(false); 16 | BoolArrayLattice x(std::array({{tru, tru, fls}})); 17 | BoolArrayLattice y(std::array({{tru, fls, fls}})); 18 | 19 | EXPECT_EQ(tru, std::get<0>(x.get())); 20 | EXPECT_EQ(tru, std::get<1>(x.get())); 21 | EXPECT_EQ(fls, std::get<2>(x.get())); 22 | EXPECT_EQ(tru, std::get<0>(y.get())); 23 | EXPECT_EQ(fls, std::get<1>(y.get())); 24 | EXPECT_EQ(fls, std::get<2>(y.get())); 25 | 26 | y.join(x); 27 | EXPECT_EQ(tru, std::get<0>(y.get())); 28 | EXPECT_EQ(tru, std::get<1>(y.get())); 29 | EXPECT_EQ(fls, std::get<2>(y.get())); 30 | } 31 | 32 | } // namespace latticeflow 33 | 34 | int main(int argc, char** argv) { 35 | ::testing::InitGoogleTest(&argc, argv); 36 | return RUN_ALL_TESTS(); 37 | } 38 | -------------------------------------------------------------------------------- /src/lattices/bool_and_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_BOOL_AND_LATTICE_H_ 2 | #define LATTICES_BOOL_AND_LATTICE_H_ 3 | 4 | #include "lattices/lattice.h" 5 | 6 | namespace latticeflow { 7 | 8 | // The semilattice of booleans ordered by reverse implication where join is 9 | // logical and. 10 | class BoolAndLattice : public Lattice { 11 | public: 12 | BoolAndLattice() : BoolAndLattice(true) {} 13 | explicit BoolAndLattice(bool b) : b_(b) {} 14 | BoolAndLattice(const BoolAndLattice& l) = default; 15 | BoolAndLattice& operator=(const BoolAndLattice& l) = default; 16 | 17 | const bool& get() const override { return b_; } 18 | void join(const BoolAndLattice& l) override { b_ = b_ && l.b_; } 19 | 20 | friend bool operator<(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 21 | return lhs.convert() < rhs.convert(); 22 | } 23 | friend bool operator<=(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 24 | return lhs.convert() <= rhs.convert(); 25 | } 26 | friend bool operator>(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 27 | return lhs.convert() > rhs.convert(); 28 | } 29 | friend bool operator>=(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 30 | return lhs.convert() >= rhs.convert(); 31 | } 32 | friend bool operator==(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 33 | return lhs.convert() == rhs.convert(); 34 | } 35 | friend bool operator!=(const BoolAndLattice& lhs, const BoolAndLattice& rhs) { 36 | return lhs.convert() != rhs.convert(); 37 | } 38 | 39 | private: 40 | // convert(true) = 0 41 | // convert(false) = 1 42 | int convert() const { return b_ ? 0 : 1; } 43 | 44 | bool b_; 45 | }; 46 | 47 | } // namespace latticeflow 48 | 49 | #endif // LATTICES_BOOL_AND_LATTICE_H_ 50 | -------------------------------------------------------------------------------- /src/lattices/bool_and_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/bool_and_lattice.h" 2 | 3 | #include "gtest/gtest.h" 4 | 5 | namespace latticeflow { 6 | 7 | TEST(BoolAndLattice, Basics) { 8 | BoolAndLattice x(true); 9 | BoolAndLattice y(false); 10 | EXPECT_EQ(true, x.get()); 11 | EXPECT_EQ(false, y.get()); 12 | 13 | y.join(x); 14 | x.join(y); 15 | EXPECT_EQ(false, x.get()); 16 | EXPECT_EQ(false, y.get()); 17 | } 18 | 19 | TEST(BoolAndLattice, Comparison) { 20 | BoolAndLattice tru(true); 21 | BoolAndLattice fls(false); 22 | 23 | EXPECT_TRUE(tru < fls); 24 | EXPECT_TRUE(tru <= fls); 25 | EXPECT_FALSE(tru > fls); 26 | EXPECT_FALSE(tru >= fls); 27 | EXPECT_FALSE(tru == fls); 28 | EXPECT_TRUE(tru != fls); 29 | } 30 | 31 | } // namespace latticeflow 32 | 33 | int main(int argc, char** argv) { 34 | ::testing::InitGoogleTest(&argc, argv); 35 | return RUN_ALL_TESTS(); 36 | } 37 | -------------------------------------------------------------------------------- /src/lattices/bool_or_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_BOOL_OR_LATTICE_H_ 2 | #define LATTICES_BOOL_OR_LATTICE_H_ 3 | 4 | #include "lattices/lattice.h" 5 | 6 | namespace latticeflow { 7 | 8 | // The semilattice of booleans ordered by implication where join is logical or. 9 | class BoolOrLattice : public Lattice { 10 | public: 11 | BoolOrLattice() : BoolOrLattice(false) {} 12 | explicit BoolOrLattice(bool b) : b_(b) {} 13 | BoolOrLattice(const BoolOrLattice& l) = default; 14 | BoolOrLattice& operator=(const BoolOrLattice& l) = default; 15 | 16 | const bool& get() const override { return b_; } 17 | void join(const BoolOrLattice& l) override { b_ = b_ || l.b_; } 18 | 19 | friend bool operator<(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 20 | return lhs.convert() < rhs.convert(); 21 | } 22 | friend bool operator<=(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 23 | return lhs.convert() <= rhs.convert(); 24 | } 25 | friend bool operator>(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 26 | return lhs.convert() > rhs.convert(); 27 | } 28 | friend bool operator>=(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 29 | return lhs.convert() >= rhs.convert(); 30 | } 31 | friend bool operator==(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 32 | return lhs.convert() == rhs.convert(); 33 | } 34 | friend bool operator!=(const BoolOrLattice& lhs, const BoolOrLattice& rhs) { 35 | return lhs.convert() != rhs.convert(); 36 | } 37 | 38 | private: 39 | // convert(true) = 1 40 | // convert(false) = 0 41 | int convert() const { return b_ ? 1 : 0; } 42 | 43 | bool b_; 44 | }; 45 | 46 | } // namespace latticeflow 47 | 48 | #endif // LATTICES_BOOL_OR_LATTICE_H_ 49 | -------------------------------------------------------------------------------- /src/lattices/bool_or_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/bool_or_lattice.h" 2 | 3 | #include "gtest/gtest.h" 4 | 5 | namespace latticeflow { 6 | 7 | TEST(BoolOrLattice, Basics) { 8 | BoolOrLattice x(true); 9 | BoolOrLattice y(false); 10 | EXPECT_EQ(true, x.get()); 11 | EXPECT_EQ(false, y.get()); 12 | 13 | x.join(y); 14 | y.join(x); 15 | EXPECT_EQ(true, x.get()); 16 | EXPECT_EQ(true, y.get()); 17 | } 18 | 19 | TEST(BoolOrLattice, Comparison) { 20 | BoolOrLattice tru(true); 21 | BoolOrLattice fls(false); 22 | 23 | EXPECT_TRUE(fls < tru); 24 | EXPECT_TRUE(fls <= tru); 25 | EXPECT_FALSE(fls > tru); 26 | EXPECT_FALSE(fls >= tru); 27 | EXPECT_FALSE(fls == tru); 28 | EXPECT_TRUE(fls != tru); 29 | } 30 | 31 | } // namespace latticeflow 32 | 33 | int main(int argc, char** argv) { 34 | ::testing::InitGoogleTest(&argc, argv); 35 | return RUN_ALL_TESTS(); 36 | } 37 | -------------------------------------------------------------------------------- /src/lattices/lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_LATTICE_H_ 2 | #define LATTICES_LATTICE_H_ 3 | 4 | #include 5 | 6 | namespace latticeflow { 7 | 8 | // Consider a partially ordered set (S, <=). An *upper bound* z of two elements 9 | // a and b in S is an element of S such that z >= a and y >= b. An upper bound 10 | // is said to be a *least upper bound* if it is less than or equal to all other 11 | // upper bounds. That is, z is the least upper bound of a and b if 12 | // 13 | // 1. z is an upper bound of a and b, and 14 | // 2. z is less than or equal to all upper bounds of a and b. 15 | // 16 | // Note that by the antisymmetry of <=, least upper bounds are unique. 17 | // 18 | // A *join semilattice* (or upper semillatice) is a partially ordered set (S, 19 | // <=) such that every pair of elements in S has a least upper bound. The least 20 | // upper bound of two elements x and y is known as their join, which we will 21 | // denote by join(x, y). Note that join is associative, commutative, and 22 | // idempotent. 23 | // 24 | // - associative: for all x, y, z. join(x, join(y, z)) == join(join(x, y), z) 25 | // - commutative: for all x, y. join(x, y) == join(y, x) 26 | // - idempotent: for all x. join(x, x) == x 27 | // 28 | // Dually, any structure (S, join) of a set S and an associative, commutative, 29 | // idempotent operator join induces a partial order on S: x <= y if and only if 30 | // join(x, y) == y. The structure (S, <=) of the set and the induced partial 31 | // order forms a semillatice. 32 | // 33 | // We represent semilattices using CRTP where an instance of type `Lattice` represents an instance of a semillatice with type `T` implemented by a 35 | // subclass `L`. Refer to this directory for many examples. 36 | // 37 | // Note that semilattices are not required to have a bottom element, but if 38 | // they do, it is recommended that the default constructor of the implementing 39 | // class initialize to the bottom element. 40 | template 41 | class Lattice { 42 | public: 43 | // The type of the lattice. Being able to access this type comes in handy 44 | // when doing a bit of metaprogramming. See the static_asserts in 45 | // pair_lattice.h for an example. 46 | using lattice_type = T; 47 | 48 | // Returns an element of the semilattice. 49 | virtual const T& get() const = 0; 50 | 51 | // Joins another instance of the semilattice into this one. 52 | virtual void join(const L& other) = 0; 53 | }; 54 | 55 | // Returns whether `l == r` according to the partial order of the lattice. 56 | template 57 | typename std::enable_if< 58 | std::is_base_of, T>::value, bool>::type 59 | operator==(const T& l, const T& r) { 60 | return l.get() == r.get(); 61 | } 62 | 63 | // Returns whether `l != r` according to the partial order of the lattice. 64 | template 65 | typename std::enable_if< 66 | std::is_base_of, T>::value, bool>::type 67 | operator!=(const T& l, const T& r) { 68 | return l.get() != r.get(); 69 | } 70 | 71 | // Returns true if `l <= r` according to the partial order of the lattice. 72 | template 73 | typename std::enable_if< 74 | std::is_base_of, T>::value, bool>::type 75 | operator<=(T l, const T& r) { 76 | l.join(r); 77 | return l.get() == r.get(); 78 | } 79 | 80 | } // namespace latticeflow 81 | 82 | #endif // LATTICES_LATTICE_H_ 83 | -------------------------------------------------------------------------------- /src/lattices/lattices.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_LATTICES_H 2 | #define LATTICES_LATTICES_H 3 | 4 | #include "lattices/array_lattice.h" 5 | #include "lattices/bool_and_lattice.h" 6 | #include "lattices/bool_or_lattice.h" 7 | #include "lattices/lattice.h" 8 | #include "lattices/map_lattice.h" 9 | #include "lattices/max_lattice.h" 10 | #include "lattices/min_lattice.h" 11 | #include "lattices/pair_lattice.h" 12 | #include "lattices/set_intersect_lattice.h" 13 | #include "lattices/set_union_lattice.h" 14 | #include "lattices/timestamp_lattice.h" 15 | #include "lattices/vector_lattice.h" 16 | 17 | #endif // LATTICES_LATTICES_H 18 | -------------------------------------------------------------------------------- /src/lattices/map_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_MAP_LATTICE_H_ 2 | #define LATTICES_MAP_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "lattices/bool_or_lattice.h" 9 | #include "lattices/lattice.h" 10 | #include "lattices/max_lattice.h" 11 | 12 | namespace latticeflow { 13 | 14 | // A MapLattice represents a map from an arbitrary type K to a lattice V. 15 | // The value bound to a key in the join of two maps depends on the presence of 16 | // the key in the two maps. 17 | // 18 | // +----------+----------+--------------------+ 19 | // | k in m_1 | k in m_2 | (m_1 join m_2)[k] | 20 | // +----------+----------+--------------------+ 21 | // | n | n | undefined | 22 | // | n | y | m_2[k] | 23 | // | y | n | m_1[k] | 24 | // | y | y | m_1[k] join m_2[k] | 25 | // +----------+----------+--------------------+ 26 | // 27 | // For example, let m_1 = {"a": 42, "b": 19} and m_2 = {"b": 1000, "c": 99} be 28 | // two maps from string to MaxLattice. The join of m_1 and m_2 is {"a": 29 | // 42, "b": 9000, "c": 99}. 30 | template 31 | class MapLattice : public Lattice, std::unordered_map> { 32 | static_assert(std::is_base_of, V>::value, 33 | "Value type of MapLattice does not inherit from Lattice."); 34 | 35 | public: 36 | MapLattice() = default; 37 | MapLattice(const MapLattice& l) = default; 38 | MapLattice& operator=(const MapLattice& l) = default; 39 | 40 | const std::unordered_map& get() const override { return kvs_; } 41 | 42 | void join(const MapLattice& l) override { 43 | for (const auto& kv : l.kvs_) { 44 | const K& key = std::get<0>(kv); 45 | const V& value = std::get<1>(kv); 46 | if (kvs_.count(key) == 0) { 47 | kvs_.insert(kv); 48 | } else { 49 | kvs_[key].join(value); 50 | } 51 | } 52 | } 53 | 54 | // See http://en.cppreference.com/w/cpp/container/unordered_map/at. 55 | const V& get(const K& key) const { return kvs_.at(key); } 56 | 57 | // `kv.put(k, v)` binds `k` to `v` if `kv` doesn't have an existing binding 58 | // for `k`. Otherwises, it joins `v` into the existing binding. For example, 59 | // {"a": 1}.put("b", 2) produces {"a": 1, "b": 2} and {"a": 1}.put("a", 2) 60 | // produces {"a": 1 join 2}. 61 | void put(const K& key, const V& val) { 62 | if (kvs_.count(key) == 0) { 63 | kvs_.insert(std::make_pair(key, val)); 64 | } else { 65 | kvs_[key].join(val); 66 | } 67 | } 68 | 69 | // TODO(mwhittaker): Figure out very carefully which other map operations to 70 | // support. We should only support the "monotonic" ones? 71 | // http://en.cppreference.com/w/cpp/container/unordered_map 72 | 73 | private: 74 | std::unordered_map kvs_; 75 | }; 76 | 77 | } // namespace latticeflow 78 | 79 | #endif // LATTICES_MAP_LATTICE_H_ 80 | -------------------------------------------------------------------------------- /src/lattices/map_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/map_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "lattices/max_lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | using MaxIntLattice = MaxLattice; 12 | using StringIntMapLattice = MapLattice; 13 | 14 | TEST(StringIntMapLattice, Basics) { 15 | StringIntMapLattice x; 16 | x.put("a", MaxIntLattice(1)); 17 | x.put("b", MaxIntLattice(2)); 18 | x.put("c", MaxIntLattice(3)); 19 | 20 | StringIntMapLattice y; 21 | y.put("c", MaxIntLattice(30)); 22 | y.put("d", MaxIntLattice(40)); 23 | y.put("e", MaxIntLattice(50)); 24 | 25 | EXPECT_EQ(1, x.get("a").get()); 26 | EXPECT_EQ(2, x.get("b").get()); 27 | EXPECT_EQ(3, x.get("c").get()); 28 | 29 | EXPECT_EQ(30, y.get("c").get()); 30 | EXPECT_EQ(40, y.get("d").get()); 31 | EXPECT_EQ(50, y.get("e").get()); 32 | 33 | x.join(y); 34 | EXPECT_EQ(1, x.get("a").get()); 35 | EXPECT_EQ(2, x.get("b").get()); 36 | EXPECT_EQ(30, x.get("c").get()); 37 | EXPECT_EQ(40, x.get("d").get()); 38 | EXPECT_EQ(50, x.get("e").get()); 39 | } 40 | 41 | } // namespace latticeflow 42 | 43 | int main(int argc, char** argv) { 44 | ::testing::InitGoogleTest(&argc, argv); 45 | return RUN_ALL_TESTS(); 46 | } 47 | -------------------------------------------------------------------------------- /src/lattices/max_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_MAX_LATTICE_H_ 2 | #define LATTICES_MAX_LATTICE_H_ 3 | 4 | #include "lattices/lattice.h" 5 | 6 | #include 7 | 8 | namespace latticeflow { 9 | 10 | // The semilattice of an arbitrary totally ordered set where join is max. T 11 | // must support all comparison operators. 12 | template 13 | class MaxLattice : public Lattice, T> { 14 | public: 15 | MaxLattice() : x_() {} 16 | explicit MaxLattice(T x) : x_(std::move(x)) {} 17 | MaxLattice(const MaxLattice& l) = default; 18 | MaxLattice& operator=(const MaxLattice& l) = default; 19 | 20 | const T& get() const override { return x_; } 21 | void join(const MaxLattice& l) override { x_ = std::max(x_, l.x_); } 22 | 23 | friend bool operator<(const MaxLattice& lhs, const MaxLattice& rhs) { 24 | return lhs.x_ < rhs.x_; 25 | } 26 | friend bool operator<=(const MaxLattice& lhs, const MaxLattice& rhs) { 27 | return lhs.x_ <= rhs.x_; 28 | } 29 | friend bool operator>(const MaxLattice& lhs, const MaxLattice& rhs) { 30 | return lhs.x_ > rhs.x_; 31 | } 32 | friend bool operator>=(const MaxLattice& lhs, const MaxLattice& rhs) { 33 | return lhs.x_ >= rhs.x_; 34 | } 35 | friend bool operator==(const MaxLattice& lhs, const MaxLattice& rhs) { 36 | return lhs.x_ == rhs.x_; 37 | } 38 | friend bool operator!=(const MaxLattice& lhs, const MaxLattice& rhs) { 39 | return lhs.x_ != rhs.x_; 40 | } 41 | 42 | private: 43 | T x_; 44 | }; 45 | 46 | } // namespace latticeflow 47 | 48 | #endif // LATTICES_MAX_LATTICE_H_ 49 | -------------------------------------------------------------------------------- /src/lattices/max_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/max_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | namespace latticeflow { 8 | 9 | using MaxIntLattice = MaxLattice; 10 | using MaxStringLattice = MaxLattice; 11 | 12 | TEST(MaxIntLattice, Basics) { 13 | MaxIntLattice x(1); 14 | MaxIntLattice y(42); 15 | EXPECT_EQ(1, x.get()); 16 | EXPECT_EQ(42, y.get()); 17 | 18 | y.join(x); 19 | x.join(y); 20 | EXPECT_EQ(42, x.get()); 21 | EXPECT_EQ(42, y.get()); 22 | } 23 | 24 | TEST(MaxIntLattice, Comparison) { 25 | MaxIntLattice x(1); 26 | MaxIntLattice y(42); 27 | EXPECT_TRUE(x < y); 28 | EXPECT_TRUE(x <= y); 29 | EXPECT_FALSE(x > y); 30 | EXPECT_FALSE(x >= y); 31 | EXPECT_FALSE(x == y); 32 | EXPECT_TRUE(x != y); 33 | } 34 | 35 | TEST(MaxStringLattice, Basics) { 36 | MaxStringLattice x("aaa"); 37 | MaxStringLattice y("aaz"); 38 | EXPECT_EQ("aaa", x.get()); 39 | EXPECT_EQ("aaz", y.get()); 40 | 41 | y.join(x); 42 | x.join(y); 43 | EXPECT_EQ("aaz", x.get()); 44 | EXPECT_EQ("aaz", y.get()); 45 | } 46 | 47 | TEST(MaxStringLattice, Comparison) { 48 | MaxStringLattice x("aaa"); 49 | MaxStringLattice y("aaz"); 50 | EXPECT_TRUE(x < y); 51 | EXPECT_TRUE(x <= y); 52 | EXPECT_FALSE(x > y); 53 | EXPECT_FALSE(x >= y); 54 | EXPECT_FALSE(x == y); 55 | EXPECT_TRUE(x != y); 56 | } 57 | 58 | } // namespace latticeflow 59 | 60 | int main(int argc, char** argv) { 61 | ::testing::InitGoogleTest(&argc, argv); 62 | return RUN_ALL_TESTS(); 63 | } 64 | -------------------------------------------------------------------------------- /src/lattices/min_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_MIN_LATTICE_H_ 2 | #define LATTICES_MIN_LATTICE_H_ 3 | 4 | #include "lattices/lattice.h" 5 | 6 | #include 7 | 8 | namespace latticeflow { 9 | 10 | // The semilattice of an arbitrary totally ordered set where join is min. T 11 | // must support all comparison operators. 12 | template 13 | class MinLattice : public Lattice, T> { 14 | public: 15 | MinLattice() : x_() {} 16 | explicit MinLattice(const T& x) : x_(x) {} 17 | MinLattice(const MinLattice& l) = default; 18 | MinLattice& operator=(const MinLattice& l) = default; 19 | 20 | const T& get() const override { return x_; } 21 | void join(const MinLattice& l) override { x_ = std::min(x_, l.x_); } 22 | 23 | friend bool operator<(const MinLattice& lhs, const MinLattice& rhs) { 24 | return lhs.x_ > rhs.x_; 25 | } 26 | friend bool operator<=(const MinLattice& lhs, const MinLattice& rhs) { 27 | return lhs.x_ >= rhs.x_; 28 | } 29 | friend bool operator>(const MinLattice& lhs, const MinLattice& rhs) { 30 | return lhs.x_ < rhs.x_; 31 | } 32 | friend bool operator>=(const MinLattice& lhs, const MinLattice& rhs) { 33 | return lhs.x_ <= rhs.x_; 34 | } 35 | friend bool operator==(const MinLattice& lhs, const MinLattice& rhs) { 36 | return lhs.x_ == rhs.x_; 37 | } 38 | friend bool operator!=(const MinLattice& lhs, const MinLattice& rhs) { 39 | return lhs.x_ != rhs.x_; 40 | } 41 | 42 | private: 43 | T x_; 44 | }; 45 | 46 | } // namespace latticeflow 47 | 48 | #endif // LATTICES_MIN_LATTICE_H_ 49 | -------------------------------------------------------------------------------- /src/lattices/min_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/min_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | namespace latticeflow { 8 | 9 | using MinIntLattice = MinLattice; 10 | using MinStringLattice = MinLattice; 11 | 12 | TEST(MinIntLattice, Basics) { 13 | MinIntLattice x(1); 14 | MinIntLattice y(42); 15 | EXPECT_EQ(1, x.get()); 16 | EXPECT_EQ(42, y.get()); 17 | 18 | x.join(y); 19 | y.join(x); 20 | EXPECT_EQ(1, x.get()); 21 | EXPECT_EQ(1, y.get()); 22 | } 23 | 24 | TEST(MinIntLattice, Comparison) { 25 | MinIntLattice x(1); 26 | MinIntLattice y(42); 27 | EXPECT_TRUE(y < x); 28 | EXPECT_TRUE(y <= x); 29 | EXPECT_FALSE(y > x); 30 | EXPECT_FALSE(y >= x); 31 | EXPECT_FALSE(y == x); 32 | EXPECT_TRUE(y != x); 33 | } 34 | 35 | TEST(MinStringLattice, Basics) { 36 | MinStringLattice x("aaa"); 37 | MinStringLattice y("aaz"); 38 | EXPECT_EQ("aaa", x.get()); 39 | EXPECT_EQ("aaz", y.get()); 40 | 41 | x.join(y); 42 | y.join(x); 43 | EXPECT_EQ("aaa", x.get()); 44 | EXPECT_EQ("aaa", y.get()); 45 | } 46 | 47 | TEST(MinStringLattice, Comparison) { 48 | MinStringLattice x("aaa"); 49 | MinStringLattice y("aaz"); 50 | EXPECT_TRUE(y < x); 51 | EXPECT_TRUE(y <= x); 52 | EXPECT_FALSE(y > x); 53 | EXPECT_FALSE(y >= x); 54 | EXPECT_FALSE(y == x); 55 | EXPECT_TRUE(y != x); 56 | } 57 | 58 | } // namespace latticeflow 59 | 60 | int main(int argc, char** argv) { 61 | ::testing::InitGoogleTest(&argc, argv); 62 | return RUN_ALL_TESTS(); 63 | } 64 | -------------------------------------------------------------------------------- /src/lattices/pair_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_PAIR_LATTICE_H_ 2 | #define LATTICES_PAIR_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "lattices/lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | // Consider two semilattices L = (S_L, join_L) and R = (S_R, join_R). The 12 | // semillatice P = (S_L x S_R, join_P) constructed from the cross product of L 13 | // and R is itself a semillatice where 14 | // 15 | // join_P((l1, r1), (l2, r2)) = (l1 join_L l2, r1 join_R r2) 16 | // 17 | // For example, 18 | // 19 | // BoolAndLattice l1(true); 20 | // BoolAndLattice l2(false); 21 | // BoolOrLattice r1(true); 22 | // BoolOrLattice r2(false); 23 | // PairLattice p1(l1, r1); 24 | // PairLattice p2(l2, r2); 25 | // p1.join(p2); // = (true, true) join (false, false) 26 | // // = (true && false, true || false) 27 | // // = (false, true) 28 | template 29 | class PairLattice : public Lattice, std::pair> { 30 | static_assert(std::is_base_of, L>::value, 31 | "Left type of PairLattice does not inherit from Lattice."); 32 | static_assert(std::is_base_of, R>::value, 33 | "Right type of PairLattice does not inherit from Lattice."); 34 | 35 | public: 36 | PairLattice() = default; 37 | PairLattice(const L& l, const R& r) : p_(l, r) {} 38 | PairLattice(const PairLattice& l) = default; 39 | PairLattice& operator=(const PairLattice& l) = default; 40 | 41 | const std::pair& get() const override { return p_; } 42 | void join(const PairLattice& l) override { 43 | std::get<0>(p_).join(std::get<0>(l.p_)); 44 | std::get<1>(p_).join(std::get<1>(l.p_)); 45 | } 46 | 47 | private: 48 | std::pair p_; 49 | }; 50 | 51 | } // namespace latticeflow 52 | 53 | #endif // LATTICES_PAIR_LATTICE_H_ 54 | -------------------------------------------------------------------------------- /src/lattices/pair_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/pair_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "lattices/bool_and_lattice.h" 8 | #include "lattices/bool_or_lattice.h" 9 | 10 | namespace latticeflow { 11 | 12 | using BoolBoolLattice = PairLattice; 13 | 14 | TEST(BoolBoolLattice, Basics) { 15 | BoolOrLattice x1(true); 16 | BoolAndLattice y1(false); 17 | BoolOrLattice x2(false); 18 | BoolAndLattice y2(true); 19 | BoolBoolLattice p1(x1, y1); 20 | BoolBoolLattice p2(x2, y2); 21 | 22 | EXPECT_EQ(true, std::get<0>(p1.get()).get()); 23 | EXPECT_EQ(false, std::get<1>(p1.get()).get()); 24 | EXPECT_EQ(false, std::get<0>(p2.get()).get()); 25 | EXPECT_EQ(true, std::get<1>(p2.get()).get()); 26 | 27 | p2.join(p1); 28 | EXPECT_EQ(true, std::get<0>(p2.get()).get()); 29 | EXPECT_EQ(false, std::get<1>(p2.get()).get()); 30 | } 31 | 32 | TEST(BoolBoolLattice, Comparisons) { 33 | BoolOrLattice x1(true); 34 | BoolAndLattice y1(false); 35 | BoolOrLattice x2(false); 36 | BoolAndLattice y2(true); 37 | BoolBoolLattice p1(x1, y1); 38 | BoolBoolLattice p2(x2, y2); 39 | 40 | EXPECT_FALSE(p1 == p2); 41 | EXPECT_TRUE(p1 != p2); 42 | EXPECT_FALSE(p1 <= p2); 43 | } 44 | 45 | } // namespace latticeflow 46 | 47 | int main(int argc, char** argv) { 48 | ::testing::InitGoogleTest(&argc, argv); 49 | return RUN_ALL_TESTS(); 50 | } 51 | -------------------------------------------------------------------------------- /src/lattices/set_intersect_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_SET_INTERSECT_LATTICE_H_ 2 | #define LATTICES_SET_INTERSECT_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "lattices/lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | // The semilattice of sets ordered by superset where join is intersect. 12 | template 13 | class SetIntersectLattice 14 | : public Lattice, std::unordered_set> { 15 | public: 16 | SetIntersectLattice() = default; 17 | explicit SetIntersectLattice(const std::unordered_set& xs) : xs_(xs) {} 18 | SetIntersectLattice(const SetIntersectLattice& l) = default; 19 | SetIntersectLattice& operator=(const SetIntersectLattice& l) = default; 20 | 21 | const std::unordered_set& get() const override { return xs_; } 22 | 23 | void join(const SetIntersectLattice& l) override { 24 | for (auto it = std::begin(xs_); it != std::end(xs_);) { 25 | if (l.xs_.count(*it) > 0) { 26 | ++it; 27 | } else { 28 | it = xs_.erase(it); 29 | } 30 | } 31 | } 32 | 33 | private: 34 | std::unordered_set xs_; 35 | }; 36 | 37 | } // namespace latticeflow 38 | 39 | #endif // LATTICES_SET_INTERSECT_LATTICE_H_ 40 | -------------------------------------------------------------------------------- /src/lattices/set_intersect_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/set_intersect_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | namespace latticeflow { 8 | 9 | using IntSetLattice = SetIntersectLattice; 10 | 11 | TEST(IntSetLattice, Basics) { 12 | IntSetLattice x(std::unordered_set({1, 2, 3})); 13 | IntSetLattice y(std::unordered_set({3, 4, 5})); 14 | EXPECT_EQ(std::unordered_set({1, 2, 3}), x.get()); 15 | EXPECT_EQ(std::unordered_set({3, 4, 5}), y.get()); 16 | 17 | x.join(y); 18 | EXPECT_EQ(std::unordered_set({3}), x.get()); 19 | } 20 | 21 | } // namespace latticeflow 22 | 23 | int main(int argc, char** argv) { 24 | ::testing::InitGoogleTest(&argc, argv); 25 | return RUN_ALL_TESTS(); 26 | } 27 | -------------------------------------------------------------------------------- /src/lattices/set_union_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_SET_UNION_LATTICE_H_ 2 | #define LATTICES_SET_UNION_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "lattices/lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | // The semilattice of sets ordered by subset where join is union. 12 | template 13 | class SetUnionLattice 14 | : public Lattice, std::unordered_set> { 15 | public: 16 | SetUnionLattice() = default; 17 | explicit SetUnionLattice(const std::unordered_set& xs) : xs_(xs) {} 18 | SetUnionLattice(const SetUnionLattice& l) = default; 19 | SetUnionLattice& operator=(const SetUnionLattice& l) = default; 20 | 21 | const std::unordered_set& get() const override { return xs_; } 22 | 23 | void join(const SetUnionLattice& l) override { 24 | xs_.insert(std::begin(l.xs_), std::end(l.xs_)); 25 | } 26 | 27 | // See http://en.cppreference.com/w/cpp/container/set/insert. 28 | void insert(const T& x) { xs_.insert(x); } 29 | 30 | private: 31 | std::unordered_set xs_; 32 | }; 33 | 34 | } // namespace latticeflow 35 | 36 | #endif // LATTICES_SET_UNION_LATTICE_H_ 37 | -------------------------------------------------------------------------------- /src/lattices/set_union_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/set_union_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | namespace latticeflow { 8 | 9 | using IntSetLattice = SetUnionLattice; 10 | 11 | TEST(IntSetLattice, Basics) { 12 | IntSetLattice x(std::unordered_set({1, 2, 3})); 13 | 14 | IntSetLattice y; 15 | y.insert(3); 16 | y.insert(4); 17 | y.insert(5); 18 | 19 | EXPECT_EQ(std::unordered_set({1, 2, 3}), x.get()); 20 | EXPECT_EQ(std::unordered_set({3, 4, 5}), y.get()); 21 | 22 | x.join(y); 23 | EXPECT_EQ(std::unordered_set({1, 2, 3, 4, 5}), x.get()); 24 | } 25 | 26 | } // namespace latticeflow 27 | 28 | int main(int argc, char** argv) { 29 | ::testing::InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | -------------------------------------------------------------------------------- /src/lattices/timestamp_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_TIMESTAMP_LATTICE_H_ 2 | #define LATTICES_TIMESTAMP_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "lattices/lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | // Consider a *timestamp* semilattice T and an arbitrary other semilattice L. A 12 | // TimestampLattice is pair (t, l) with the following join: 13 | // 14 | // { (t1, l1), if t1 > t2 15 | // join((t1, l1), (t2, l2)) = { (t2, l2), if t1 < t2 16 | // { (t1 join t2, l1 join l2), otherwise 17 | // 18 | // In words, when merging two timestamp lattices, the one with the bigger 19 | // timestamp completely dominates. If they share the same timestamp, then the 20 | // *timestamp and value* are merged. Note that a TimestampLattice can be 21 | // instantiated with a totally ordered timestamp type, like a Lamport clock, or 22 | // a partially ordered timestamp type like a vector clock! 23 | // 24 | // Also, note that the Timestamp template must support the < operator where 25 | // - if a < b is true, then b is strictly larger than a, and 26 | // - if a < b is false, then either b is not strictly larger than a or the 27 | // two elements are not related. 28 | template 29 | class TimestampLattice 30 | : public Lattice, std::pair> { 31 | static_assert( 32 | std::is_base_of, 33 | Timestamp>::value, 34 | "Timestamp type of TimestampLattice does not inherit from Lattice."); 35 | static_assert( 36 | std::is_base_of, L>::value, 37 | "Value type of TimestampLattice does not inherit from Lattice."); 38 | 39 | public: 40 | TimestampLattice() = default; 41 | TimestampLattice(const Timestamp& t, const L& x) : p_(t, x) {} 42 | TimestampLattice(const TimestampLattice& l) = default; 43 | TimestampLattice& operator=(const TimestampLattice& l) = 44 | default; 45 | 46 | const std::pair& get() const override { return p_; } 47 | 48 | void join(const TimestampLattice& l) override { 49 | if (timestamp() < l.timestamp()) { 50 | // The other lattice's timestamp dominates ours, so we adopt its value. 51 | p_ = l.p_; 52 | } else if (l.timestamp() < timestamp()) { 53 | // Our timestamp dominates the other lattice's timestamp, so we do 54 | // nothing! 55 | } else { 56 | // If !(a < b) and !(b < a), then either a == b or the two are unrelated 57 | // with respect to the the Timestamp lattice's partial order. In either 58 | // case, we can merge both the timestamp and value. 59 | std::get<0>(p_).join(std::get<0>(l.p_)); 60 | std::get<1>(p_).join(std::get<1>(l.p_)); 61 | } 62 | } 63 | 64 | // Return the timestamp. 65 | const Timestamp& timestamp() const { return std::get<0>(p_); } 66 | 67 | // Return the value. 68 | const L& value() const { return std::get<1>(p_); } 69 | 70 | private: 71 | std::pair p_; 72 | }; 73 | 74 | } // namespace latticeflow 75 | 76 | #endif // LATTICES_TIMESTAMP_LATTICE_H_ 77 | -------------------------------------------------------------------------------- /src/lattices/timestamp_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/timestamp_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "lattices/bool_or_lattice.h" 8 | #include "lattices/max_lattice.h" 9 | 10 | namespace latticeflow { 11 | 12 | using MaxIntLattice = MaxLattice; 13 | using LamportBoolLattice = TimestampLattice; 14 | 15 | TEST(LamportBoolLattice, Basics) { 16 | BoolOrLattice tru(true); 17 | BoolOrLattice fls(false); 18 | LamportBoolLattice x(MaxIntLattice(1), tru); 19 | LamportBoolLattice y(MaxIntLattice(1), fls); 20 | LamportBoolLattice z(MaxIntLattice(42), fls); 21 | 22 | EXPECT_EQ(1, x.timestamp().get()); 23 | EXPECT_EQ(1, y.timestamp().get()); 24 | EXPECT_EQ(42, z.timestamp().get()); 25 | 26 | EXPECT_EQ(true, x.value().get()); 27 | EXPECT_EQ(false, y.value().get()); 28 | EXPECT_EQ(false, z.value().get()); 29 | 30 | y.join(x); 31 | EXPECT_EQ(1, y.timestamp().get()); 32 | EXPECT_EQ(true, y.value().get()); 33 | 34 | y.join(z); 35 | EXPECT_EQ(42, y.timestamp().get()); 36 | EXPECT_EQ(false, y.value().get()); 37 | } 38 | 39 | } // namespace latticeflow 40 | 41 | int main(int argc, char** argv) { 42 | ::testing::InitGoogleTest(&argc, argv); 43 | return RUN_ALL_TESTS(); 44 | } 45 | -------------------------------------------------------------------------------- /src/lattices/tombstoned_set_union_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_TOMBSTONED_SET_UNION_LATTICE_H_ 2 | #define LATTICES_TOMBSTONED_SET_UNION_LATTICE_H_ 3 | 4 | #include "lattices/bool_or_lattice.h" 5 | #include "lattices/map_lattice.h" 6 | 7 | namespace latticeflow { 8 | 9 | // With an increasing SetUnionLattice, elements can only be added to the set; 10 | // they can never be removed. A TombstonedSetUnionLattice allows for elements 11 | // to be deleted by associating a boolean tombstone with the item. If the 12 | // associated boolean is false, the item is thought of as in the set. If it is 13 | // false, the element is thought of as deleted. Once an element is deleted, it 14 | // can never again be inserted. 15 | template 16 | using TombstonedSetUnionLattice = MapLattice; 17 | 18 | } // namespace latticeflow 19 | 20 | #endif // LATTICES_TOMBSTONED_SET_UNION_LATTICE_H_ 21 | -------------------------------------------------------------------------------- /src/lattices/vector_lattice.h: -------------------------------------------------------------------------------- 1 | #ifndef LATTICES_VECTOR_LATTICE_H_ 2 | #define LATTICES_VECTOR_LATTICE_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "lattices/lattice.h" 9 | 10 | namespace latticeflow { 11 | 12 | // Consider a semillatice T = (S_T, join_T). We can form a vector semilattice V 13 | // = (std::vector, join_V) where join_V is defined as follows. Let a = 14 | // [a_1, ..., a_m] and b = [b_1, ..., b_n] be two vectors of S_T where m <= n. 15 | // 16 | // join_V(a, b) = [a_1 join b_1, ..., a_m join b_m, b_{m+1}, ..., b_n] 17 | // 18 | // For example, 19 | // 20 | // VectorLattice a = { 21 | // BoolAndLattice(true), 22 | // BoolAndLattice(false), 23 | // BoolAndLattice(true) 24 | // }; 25 | // VectorLattice b = { 26 | // BoolAndLattice(false), 27 | // BoolAndLattice(false), 28 | // BoolAndLattice(true), 29 | // BoolAndLattice(false), 30 | // BoolAndLattice(true) 31 | // }; 32 | // a.join(b); // = [true, false, true] join [false, false, true, false, true] 33 | // // = [true && false, false && false, true && true, false, true] 34 | // // = [false, false, true, false, true] 35 | template 36 | class VectorLattice : public Lattice, std::vector> { 37 | static_assert(std::is_base_of, T>::value, 38 | "Type of VectorLattice does not inherit from Lattice."); 39 | 40 | public: 41 | VectorLattice() = default; 42 | explicit VectorLattice(std::initializer_list xs) : xs_(xs) {} 43 | VectorLattice(const VectorLattice& l) = default; 44 | VectorLattice& operator=(const VectorLattice& l) = default; 45 | 46 | const std::vector& get() const override { return xs_; } 47 | 48 | void join(const VectorLattice& l) override { 49 | for (std::size_t i = 0; i < std::min(xs_.size(), l.xs_.size()); ++i) { 50 | xs_[i].join(l.xs_[i]); 51 | } 52 | if (xs_.size() < l.xs_.size()) { 53 | xs_.insert(xs_.end(), l.xs_.begin() + xs_.size(), l.xs_.end()); 54 | } 55 | } 56 | 57 | private: 58 | std::vector xs_; 59 | }; 60 | 61 | } // namespace latticeflow 62 | 63 | #endif // LATTICES_VECTOR_LATTICE_H_ 64 | -------------------------------------------------------------------------------- /src/lattices/vector_lattice_test.cc: -------------------------------------------------------------------------------- 1 | #include "lattices/vector_lattice.h" 2 | 3 | #include 4 | 5 | #include "gtest/gtest.h" 6 | 7 | #include "lattices/bool_or_lattice.h" 8 | 9 | namespace latticeflow { 10 | 11 | using BoolVectorLattice = VectorLattice; 12 | 13 | TEST(BoolVectorLattice, Basics) { 14 | BoolOrLattice tru(true); 15 | BoolOrLattice fls(false); 16 | BoolVectorLattice x({tru, fls, fls}); 17 | BoolVectorLattice y({tru, tru, fls, fls}); 18 | 19 | EXPECT_EQ(tru, x.get()[0]); 20 | EXPECT_EQ(fls, x.get()[1]); 21 | EXPECT_EQ(fls, x.get()[2]); 22 | 23 | EXPECT_EQ(tru, y.get()[0]); 24 | EXPECT_EQ(tru, y.get()[1]); 25 | EXPECT_EQ(fls, y.get()[2]); 26 | EXPECT_EQ(fls, y.get()[3]); 27 | 28 | x.join(y); 29 | EXPECT_EQ(tru, x.get()[0]); 30 | EXPECT_EQ(tru, x.get()[1]); 31 | EXPECT_EQ(fls, x.get()[2]); 32 | EXPECT_EQ(fls, x.get()[3]); 33 | } 34 | 35 | } // namespace latticeflow 36 | 37 | int main(int argc, char** argv) { 38 | ::testing::InitGoogleTest(&argc, argv); 39 | return RUN_ALL_TESTS(); 40 | } 41 | -------------------------------------------------------------------------------- /src/pushpull/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | CREATE_NAMED_TEST(pushpull_counter_test counter_test.cc) 4 | CREATE_NAMED_TEST(pushpull_dedup_test dedup_test.cc) 5 | CREATE_NAMED_TEST(pushpull_drop_test drop_test.cc) 6 | CREATE_NAMED_TEST(pushpull_dup_test dup_test.cc) 7 | CREATE_NAMED_TEST(pushpull_filter_test filter_test.cc) 8 | CREATE_NAMED_TEST(pushpull_group_by_test group_by_test.cc) 9 | CREATE_NAMED_TEST(pushpull_id_test id_test.cc) 10 | CREATE_NAMED_TEST(pushpull_if_test if_test.cc) 11 | CREATE_NAMED_TEST(pushpull_join_test join_test.cc) 12 | CREATE_NAMED_TEST(pushpull_lambda_test lambda_test.cc) 13 | CREATE_NAMED_TEST(pushpull_map_test map_test.cc) 14 | CREATE_NAMED_TEST(pushpull_queue_test queue_test.cc) 15 | CREATE_NAMED_TEST(pushpull_register_test register_test.cc) 16 | CREATE_NAMED_TEST(pushpull_round_robin_test round_robin_test.cc) 17 | CREATE_NAMED_TEST(pushpull_tee_test tee_test.cc) 18 | -------------------------------------------------------------------------------- /src/pushpull/README.md: -------------------------------------------------------------------------------- 1 | # Push and Pull 2 | A collection of Click-style push and pull operators. 3 | -------------------------------------------------------------------------------- /src/pushpull/counter.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_COUNTER_H_ 2 | #define PUSHPULL_COUNTER_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Counter; 14 | 15 | template 16 | class Counter : public Pusher { 17 | public: 18 | explicit Counter(Pusher* downstream) : downstream_(downstream) {} 19 | 20 | void push(T x) override { 21 | count_++; 22 | downstream_->push(x); 23 | } 24 | 25 | int get() { return count_; } 26 | 27 | private: 28 | Pusher* downstream_; 29 | int count_ = 0; 30 | }; 31 | 32 | template 33 | Counter make_counter(Pusher* downstream) { 34 | return Counter(downstream); 35 | } 36 | 37 | template 38 | class Counter : public Puller { 39 | public: 40 | explicit Counter(Puller* upstream) : upstream_(upstream) {} 41 | 42 | boost::optional pull() override { 43 | count_++; 44 | return upstream_->pull(); 45 | } 46 | 47 | int get() { return count_; } 48 | 49 | private: 50 | Puller* upstream_; 51 | int count_ = 0; 52 | }; 53 | 54 | template 55 | Counter make_counter(Puller* upstream) { 56 | return Counter(upstream); 57 | } 58 | 59 | } // namespace latticeflow 60 | 61 | #endif // PUSHPULL_COUNTER_H_ 62 | -------------------------------------------------------------------------------- /src/pushpull/counter_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/counter.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/drop.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(Counter, Push) { 11 | Drop drop; 12 | auto counter = make_counter(&drop); 13 | 14 | for (int i = 0; i < 100; ++i) { 15 | EXPECT_EQ(i, counter.get()); 16 | counter.push(i); 17 | } 18 | } 19 | 20 | TEST(Counter, Pull) { 21 | Drop drop; 22 | auto counter = make_counter(&drop); 23 | 24 | for (int i = 0; i < 100; ++i) { 25 | EXPECT_EQ(i, counter.get()); 26 | counter.pull(); 27 | } 28 | } 29 | 30 | } // namespace latticeflow 31 | 32 | int main(int argc, char **argv) { 33 | ::testing::InitGoogleTest(&argc, argv); 34 | return RUN_ALL_TESTS(); 35 | } 36 | -------------------------------------------------------------------------------- /src/pushpull/dedup.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_DEDUP_H_ 2 | #define PUSHPULL_DEDUP_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class Dedup; 16 | 17 | template 18 | class Dedup : public Pusher { 19 | public: 20 | explicit Dedup(Pusher* downstream, const int num_dups) 21 | : downstream_(downstream), num_dups_(num_dups) {} 22 | 23 | void push(T x) override { 24 | counts_[x]++; 25 | if (counts_[x] == num_dups_) { 26 | counts_[x] = 0; 27 | downstream_->push(x); 28 | } 29 | } 30 | 31 | private: 32 | Pusher* downstream_; 33 | const int num_dups_; 34 | std::map counts_; 35 | }; 36 | 37 | template 38 | Dedup make_dedup(Pusher* downstream, const int num_dups) { 39 | return Dedup(downstream, num_dups); 40 | } 41 | 42 | template 43 | class Dedup : public Puller { 44 | public: 45 | explicit Dedup(Puller* upstream, const int num_dups) 46 | : upstream_(upstream), num_dups_(num_dups) {} 47 | 48 | boost::optional pull() override { 49 | boost::optional x = upstream_->pull(); 50 | counts_[x]++; 51 | if (counts_[x] == num_dups_) { 52 | counts_[x] = 0; 53 | return x; 54 | } else { 55 | return {}; 56 | } 57 | } 58 | 59 | private: 60 | Puller* upstream_; 61 | const int num_dups_; 62 | std::map, int> counts_; 63 | }; 64 | 65 | template 66 | Dedup make_dedup(Puller* upstream, const int num_dups) { 67 | return Dedup(upstream, num_dups); 68 | } 69 | 70 | } // namespace latticeflow 71 | 72 | #endif // PUSHPULL_DEDUP_H_ 73 | -------------------------------------------------------------------------------- /src/pushpull/dedup_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/dedup.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/queue.h" 7 | #include "pushpull/register.h" 8 | #include "pushpull/round_robin.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Dup, Push) { 13 | Register x; 14 | x.push(0); 15 | auto dedup = make_dedup(&x, 3); 16 | 17 | dedup.push(42); 18 | EXPECT_EQ(0, x.get()); 19 | dedup.push(42); 20 | EXPECT_EQ(0, x.get()); 21 | dedup.push(42); 22 | EXPECT_EQ(42, x.get()); 23 | 24 | dedup.push(1); 25 | EXPECT_EQ(42, x.get()); 26 | dedup.push(2); 27 | EXPECT_EQ(42, x.get()); 28 | dedup.push(1); 29 | EXPECT_EQ(42, x.get()); 30 | dedup.push(2); 31 | EXPECT_EQ(42, x.get()); 32 | dedup.push(1); 33 | EXPECT_EQ(1, x.get()); 34 | dedup.push(2); 35 | EXPECT_EQ(2, x.get()); 36 | } 37 | 38 | TEST(Dup, Pull) { 39 | Register x(42); 40 | Register y(76); 41 | auto round_robin = make_round_robin({&x, &y}); 42 | auto dedup = make_dedup(&round_robin, 3); 43 | 44 | EXPECT_EQ(boost::optional(), dedup.pull()); 45 | EXPECT_EQ(boost::optional(), dedup.pull()); 46 | EXPECT_EQ(boost::optional(), dedup.pull()); 47 | EXPECT_EQ(boost::optional(), dedup.pull()); 48 | EXPECT_EQ(boost::optional(42), dedup.pull()); 49 | EXPECT_EQ(boost::optional(76), dedup.pull()); 50 | EXPECT_EQ(boost::optional(), dedup.pull()); 51 | EXPECT_EQ(boost::optional(), dedup.pull()); 52 | } 53 | 54 | } // namespace latticeflow 55 | 56 | int main(int argc, char **argv) { 57 | ::testing::InitGoogleTest(&argc, argv); 58 | return RUN_ALL_TESTS(); 59 | } 60 | -------------------------------------------------------------------------------- /src/pushpull/drop.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_DROP_H_ 2 | #define PUSHPULL_DROP_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Drop; 14 | 15 | template 16 | class Drop : public Pusher { 17 | public: 18 | void push(T) override {} 19 | }; 20 | 21 | template 22 | class Drop : public Puller { 23 | public: 24 | boost::optional pull() override { return {}; } 25 | }; 26 | 27 | } // namespace latticeflow 28 | 29 | #endif // PUSHPULL_DROP_H_ 30 | -------------------------------------------------------------------------------- /src/pushpull/drop_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/drop.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | namespace latticeflow { 7 | 8 | TEST(Drop, Push) { 9 | Drop drop; 10 | drop.push(1); 11 | } 12 | 13 | TEST(Drop, Pull) { 14 | Drop drop; 15 | EXPECT_EQ(boost::optional(), drop.pull()); 16 | } 17 | 18 | } // namespace latticeflow 19 | 20 | int main(int argc, char **argv) { 21 | ::testing::InitGoogleTest(&argc, argv); 22 | return RUN_ALL_TESTS(); 23 | } 24 | -------------------------------------------------------------------------------- /src/pushpull/dup.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_DUP_H_ 2 | #define PUSHPULL_DUP_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Dup; 14 | 15 | template 16 | class Dup : public Pusher { 17 | public: 18 | explicit Dup(Pusher* downstream, const int num_dups) 19 | : downstream_(downstream), num_dups_(num_dups) {} 20 | 21 | void push(T x) override { 22 | for (int i = 0; i < num_dups_; ++i) { 23 | downstream_->push(x); 24 | } 25 | } 26 | 27 | private: 28 | Pusher* downstream_; 29 | const int num_dups_; 30 | }; 31 | 32 | template 33 | Dup make_dup(Pusher* downstream, const int num_dups) { 34 | return Dup(downstream, num_dups); 35 | } 36 | 37 | template 38 | class Dup : public Puller { 39 | public: 40 | explicit Dup(Puller* upstream, const int num_dups) 41 | : upstream_(upstream), num_dups_(num_dups) {} 42 | 43 | boost::optional pull() override { 44 | if (num_dups_sent_ == num_dups_) { 45 | num_dups_sent_ = 0; 46 | } 47 | if (num_dups_sent_ == 0) { 48 | buf_ = upstream_->pull(); 49 | } 50 | num_dups_sent_++; 51 | return buf_; 52 | } 53 | 54 | private: 55 | Puller* upstream_; 56 | const int num_dups_; 57 | 58 | int num_dups_sent_ = 0; 59 | boost::optional buf_; 60 | }; 61 | 62 | template 63 | Dup make_dup(Puller* upstream, const int num_dups) { 64 | return Dup(upstream, num_dups); 65 | } 66 | 67 | } // namespace latticeflow 68 | 69 | #endif // PUSHPULL_DUP_H_ 70 | -------------------------------------------------------------------------------- /src/pushpull/dup_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/dup.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/queue.h" 7 | #include "pushpull/register.h" 8 | #include "pushpull/round_robin.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Dup, Push) { 13 | Queue q; 14 | auto dup = make_dup((Pusher *)&q, 3); 15 | 16 | dup.push(42); 17 | EXPECT_EQ(boost::optional(42), q.pull()); 18 | EXPECT_EQ(boost::optional(42), q.pull()); 19 | EXPECT_EQ(boost::optional(42), q.pull()); 20 | EXPECT_EQ(boost::optional(), q.pull()); 21 | 22 | dup.push(76); 23 | EXPECT_EQ(boost::optional(76), q.pull()); 24 | EXPECT_EQ(boost::optional(76), q.pull()); 25 | EXPECT_EQ(boost::optional(76), q.pull()); 26 | EXPECT_EQ(boost::optional(), q.pull()); 27 | } 28 | 29 | TEST(Dup, Pull) { 30 | Register x(42); 31 | Register y(76); 32 | auto round_robin = make_round_robin({&x, &y}); 33 | auto dup = make_dup(&round_robin, 3); 34 | 35 | EXPECT_EQ(boost::optional(42), dup.pull()); 36 | EXPECT_EQ(boost::optional(42), dup.pull()); 37 | EXPECT_EQ(boost::optional(42), dup.pull()); 38 | EXPECT_EQ(boost::optional(76), dup.pull()); 39 | EXPECT_EQ(boost::optional(76), dup.pull()); 40 | EXPECT_EQ(boost::optional(76), dup.pull()); 41 | } 42 | 43 | } // namespace latticeflow 44 | 45 | int main(int argc, char **argv) { 46 | ::testing::InitGoogleTest(&argc, argv); 47 | return RUN_ALL_TESTS(); 48 | } 49 | -------------------------------------------------------------------------------- /src/pushpull/filter.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_FILTER_H_ 2 | #define PUSHPULL_FILTER_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class Filter; 16 | 17 | template 18 | class Filter : public Pusher { 19 | public: 20 | explicit Filter(std::function predicate, Pusher* downstream) 21 | : predicate_(predicate), downstream_(downstream) {} 22 | 23 | void push(T x) override { 24 | if (predicate_(x)) { 25 | downstream_->push(x); 26 | } 27 | } 28 | 29 | private: 30 | std::function predicate_; 31 | Pusher* downstream_; 32 | }; 33 | 34 | template 35 | Filter make_filter(std::function predicate, 36 | Pusher* downstream) { 37 | return Filter(predicate, downstream); 38 | } 39 | 40 | template 41 | class Filter : public Puller { 42 | public: 43 | explicit Filter(std::function predicate, Puller* upstream) 44 | : predicate_(predicate), upstream_(upstream) {} 45 | 46 | boost::optional pull() override { 47 | boost::optional x = upstream_->pull(); 48 | if (x && predicate_(*x)) { 49 | return x; 50 | } else { 51 | return {}; 52 | } 53 | } 54 | 55 | private: 56 | std::function predicate_; 57 | Puller* upstream_; 58 | }; 59 | 60 | template 61 | Filter make_filter(std::function predicate, 62 | Puller* upstream) { 63 | return Filter(predicate, upstream); 64 | } 65 | 66 | } // namespace latticeflow 67 | 68 | #endif // PUSHPULL_FILTER_H_ 69 | -------------------------------------------------------------------------------- /src/pushpull/filter_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/filter.h" 2 | 3 | #include 4 | 5 | #include "boost/optional.hpp" 6 | #include "gtest/gtest.h" 7 | 8 | #include "pushpull/register.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Filter, Push) { 13 | Register x; 14 | std::function pred = [](int x) { return x % 2 == 0; }; 15 | auto evens = make_filter(pred, &x); 16 | 17 | x.push(0); 18 | evens.push(1); 19 | EXPECT_EQ(0, x.get()); 20 | 21 | evens.push(2); 22 | EXPECT_EQ(2, x.get()); 23 | 24 | evens.push(3); 25 | EXPECT_EQ(2, x.get()); 26 | } 27 | 28 | TEST(Filter, Pull) { 29 | std::function pred = [](int x) { return x % 2 == 0; }; 30 | 31 | { 32 | Register x(0); 33 | auto evens = make_filter(pred, &x); 34 | EXPECT_EQ(boost::optional(0), evens.pull()); 35 | } 36 | 37 | { 38 | Register x(1); 39 | auto evens = make_filter(pred, &x); 40 | EXPECT_EQ(boost::optional(), evens.pull()); 41 | } 42 | 43 | { 44 | Register x(2); 45 | auto evens = make_filter(pred, &x); 46 | EXPECT_EQ(boost::optional(2), evens.pull()); 47 | } 48 | 49 | { 50 | Register x(3); 51 | auto evens = make_filter(pred, &x); 52 | EXPECT_EQ(boost::optional(), evens.pull()); 53 | } 54 | } 55 | 56 | } // namespace latticeflow 57 | 58 | int main(int argc, char **argv) { 59 | ::testing::InitGoogleTest(&argc, argv); 60 | return RUN_ALL_TESTS(); 61 | } 62 | -------------------------------------------------------------------------------- /src/pushpull/group_by.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_MAP_H_ 2 | #define PUSHPULL_MAP_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "boost/optional.hpp" 10 | 11 | #include "pushpull/lambda.h" 12 | #include "pushpull/phantoms.h" 13 | #include "pushpull/puller.h" 14 | #include "pushpull/pusher.h" 15 | 16 | namespace latticeflow { 17 | 18 | template 19 | class GroupBy; 20 | 21 | template 22 | class GroupBy { 23 | public: 24 | explicit GroupBy(Pusher>>* downstream, 25 | std::function group) 26 | : downstream_(downstream), 27 | group_(group), 28 | push_group_([this](T x) { this->push_group(x); }), 29 | end_group_([this](Group group) { this->end_group(group); }) {} 30 | 31 | Pusher& push_group() { return push_group_; } 32 | Pusher& end_group() { return end_group_; } 33 | 34 | private: 35 | void push_group(T x) { 36 | Group group = group_(x); 37 | groups_[group].push_back(x); 38 | } 39 | 40 | void end_group(Group group) { 41 | std::pair> xs = 42 | std::make_pair(group, std::move(groups_[group])); 43 | groups_.erase(group); 44 | downstream_->push(xs); 45 | } 46 | 47 | Pusher>>* downstream_; 48 | std::function group_; 49 | Lambda push_group_; 50 | Lambda end_group_; 51 | std::map> groups_; 52 | }; 53 | 54 | template 55 | GroupBy make_group_by( 56 | Pusher>>* downstream, 57 | std::function group) { 58 | return GroupBy(downstream, group); 59 | } 60 | 61 | } // namespace latticeflow 62 | 63 | #endif // PUSHPULL_MAP_H_ 64 | -------------------------------------------------------------------------------- /src/pushpull/group_by_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/group_by.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "boost/optional.hpp" 8 | #include "gtest/gtest.h" 9 | 10 | #include "pushpull/id.h" 11 | #include "pushpull/queue.h" 12 | #include "pushpull/register.h" 13 | #include "pushpull/round_robin.h" 14 | 15 | namespace latticeflow { 16 | 17 | TEST(GroupBy, Push) { 18 | using T = int; 19 | using Group = int; 20 | using Out = std::pair>; 21 | 22 | Register x; 23 | std::function group = [](int x) { return x; }; 24 | auto group_by = make_group_by(&x, group); 25 | auto push_group = make_id(&group_by.push_group()); 26 | auto end_group = make_id(&group_by.end_group()); 27 | 28 | { 29 | end_group.push(42); 30 | Out out = std::make_pair(42, std::vector({})); 31 | EXPECT_EQ(out, x.get()); 32 | } 33 | 34 | { 35 | push_group.push(42); 36 | push_group.push(42); 37 | push_group.push(42); 38 | end_group.push(42); 39 | Out out = std::make_pair(42, std::vector({42, 42, 42})); 40 | EXPECT_EQ(out, x.get()); 41 | } 42 | 43 | { 44 | push_group.push(42); 45 | push_group.push(1); 46 | push_group.push(42); 47 | push_group.push(1); 48 | push_group.push(42); 49 | { 50 | end_group.push(42); 51 | Out out = std::make_pair(42, std::vector({42, 42, 42})); 52 | EXPECT_EQ(out, x.get()); 53 | } 54 | { 55 | end_group.push(1); 56 | Out out = std::make_pair(1, std::vector({1, 1})); 57 | EXPECT_EQ(out, x.get()); 58 | } 59 | } 60 | } 61 | 62 | } // namespace latticeflow 63 | 64 | int main(int argc, char **argv) { 65 | ::testing::InitGoogleTest(&argc, argv); 66 | return RUN_ALL_TESTS(); 67 | } 68 | -------------------------------------------------------------------------------- /src/pushpull/id.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_ID_H_ 2 | #define PUSHPULL_ID_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Id; 14 | 15 | template 16 | class Id : public Pusher { 17 | public: 18 | explicit Id(Pusher* downstream) : downstream_(downstream) {} 19 | 20 | void push(T x) override { downstream_->push(x); } 21 | 22 | private: 23 | Pusher* downstream_; 24 | }; 25 | 26 | template 27 | Id make_id(Pusher* downstream) { 28 | return Id(downstream); 29 | } 30 | 31 | template 32 | class Id : public Puller { 33 | public: 34 | explicit Id(Puller* upstream) : upstream_(upstream) {} 35 | 36 | boost::optional pull() override { return upstream_->pull(); } 37 | 38 | private: 39 | Puller* upstream_; 40 | }; 41 | 42 | template 43 | Id make_id(Puller* upstream) { 44 | return Id(upstream); 45 | } 46 | 47 | } // namespace latticeflow 48 | 49 | #endif // PUSHPULL_ID_H_ 50 | -------------------------------------------------------------------------------- /src/pushpull/id_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/id.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/drop.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(Id, Push) { 11 | Drop drop; 12 | auto id = make_id(&drop); 13 | id.push(42); 14 | } 15 | 16 | TEST(Id, Pull) { 17 | Drop drop; 18 | auto id = make_id(&drop); 19 | EXPECT_EQ(boost::optional(), id.pull()); 20 | } 21 | 22 | } // namespace latticeflow 23 | 24 | int main(int argc, char **argv) { 25 | ::testing::InitGoogleTest(&argc, argv); 26 | return RUN_ALL_TESTS(); 27 | } 28 | -------------------------------------------------------------------------------- /src/pushpull/if.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_IF_H_ 2 | #define PUSHPULL_IF_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class If; 16 | 17 | template 18 | class If : public Pusher { 19 | public: 20 | If(std::function predicate, Pusher* true_branch, 21 | Pusher* false_branch) 22 | : predicate_(predicate), 23 | true_branch_(true_branch), 24 | false_branch_(false_branch) {} 25 | 26 | void push(T x) override { 27 | if (predicate_(x)) { 28 | true_branch_->push(x); 29 | } else { 30 | false_branch_->push(x); 31 | } 32 | } 33 | 34 | private: 35 | std::function predicate_; 36 | Pusher* true_branch_; 37 | Pusher* false_branch_; 38 | }; 39 | 40 | template 41 | If make_if(std::function predicate, Pusher* true_branch, 42 | Pusher* false_branch) { 43 | return If(predicate, true_branch, false_branch); 44 | } 45 | 46 | } // namespace latticeflow 47 | 48 | #endif // PUSHPULL_IF_H_ 49 | -------------------------------------------------------------------------------- /src/pushpull/if_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/if.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/register.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(If, Push) { 11 | Register tru; 12 | Register fls; 13 | std::function pred = [](int x) { return x % 2 == 0; }; 14 | auto evens = make_if(pred, &tru, &fls); 15 | 16 | evens.push(42); 17 | EXPECT_EQ(42, tru.get()); 18 | 19 | evens.push(43); 20 | EXPECT_EQ(42, tru.get()); 21 | EXPECT_EQ(43, fls.get()); 22 | 23 | evens.push(0); 24 | EXPECT_EQ(0, tru.get()); 25 | EXPECT_EQ(43, fls.get()); 26 | 27 | evens.push(123); 28 | EXPECT_EQ(0, tru.get()); 29 | EXPECT_EQ(123, fls.get()); 30 | } 31 | 32 | } // namespace latticeflow 33 | 34 | int main(int argc, char **argv) { 35 | ::testing::InitGoogleTest(&argc, argv); 36 | return RUN_ALL_TESTS(); 37 | } 38 | -------------------------------------------------------------------------------- /src/pushpull/join.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_MAP_H_ 2 | #define PUSHPULL_MAP_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "boost/optional.hpp" 10 | 11 | #include "pushpull/lambda.h" 12 | #include "pushpull/phantoms.h" 13 | #include "pushpull/puller.h" 14 | #include "pushpull/pusher.h" 15 | 16 | namespace latticeflow { 17 | 18 | template 19 | class Join; 20 | 21 | template 22 | class Join { 23 | public: 24 | explicit Join(Pusher>* downstream, 25 | std::function key_left, 26 | std::function key_right) 27 | : downstream_(downstream), 28 | key_left_(key_left), 29 | key_right_(key_right), 30 | push_left_([this](Left left) { this->push_left(left); }), 31 | push_right_([this](Right right) { this->push_right(right); }) {} 32 | 33 | Pusher& left() { return push_left_; } 34 | Pusher& right() { return push_right_; } 35 | 36 | private: 37 | void push_left(Left left) { 38 | Key key = key_left_(left); 39 | left_map_[key].push_back(left); 40 | for (const Right& right : right_map_[key]) { 41 | downstream_->push(std::make_pair(left, right)); 42 | } 43 | } 44 | 45 | void push_right(Right right) { 46 | Key key = key_right_(right); 47 | right_map_[key].push_back(right); 48 | for (const Left& left : left_map_[key]) { 49 | downstream_->push(std::make_pair(left, right)); 50 | } 51 | } 52 | 53 | Pusher>* downstream_; 54 | 55 | std::function key_left_; 56 | std::function key_right_; 57 | 58 | Lambda push_left_; 59 | Lambda push_right_; 60 | 61 | std::map> left_map_; 62 | std::map> right_map_; 63 | }; 64 | 65 | template 66 | Join make_join( 67 | Pusher>* downstream, 68 | std::function key_left, std::function key_right) { 69 | return Join(downstream, key_left, key_right); 70 | } 71 | 72 | template 73 | class Join : public Puller> { 74 | public: 75 | explicit Join(Puller* left, Puller* right, 76 | std::function key_left, 77 | std::function key_right) 78 | : left_(left), 79 | right_(right), 80 | key_left_(key_left), 81 | key_right_(key_right) {} 82 | 83 | boost::optional> pull() override { 84 | if (buf_.size() > 0) { 85 | std::pair back = std::move(buf_.back()); 86 | buf_.pop_back(); 87 | return back; 88 | } 89 | 90 | if (pull_left) { 91 | boost::optional left_val = left_->pull(); 92 | if (left_val) { 93 | Key key = key_left_(*left_val); 94 | left_map_[key].push_back(*left_val); 95 | for (const Right& right_val : right_map_[key]) { 96 | buf_.push_back(std::make_pair(*left_val, right_val)); 97 | } 98 | } else { 99 | return {}; 100 | } 101 | } else { 102 | boost::optional right_val = right_->pull(); 103 | if (right_val) { 104 | Key key = key_right_(*right_val); 105 | right_map_[key].push_back(*right_val); 106 | for (const Left& left_val : left_map_[key]) { 107 | buf_.push_back(std::make_pair(left_val, *right_val)); 108 | } 109 | } else { 110 | return {}; 111 | } 112 | } 113 | pull_left = !pull_left; 114 | 115 | if (buf_.size() > 0) { 116 | std::pair back = std::move(buf_.back()); 117 | buf_.pop_back(); 118 | return back; 119 | } else { 120 | return {}; 121 | } 122 | } 123 | 124 | private: 125 | Puller* left_; 126 | Puller* right_; 127 | bool pull_left = true; 128 | 129 | std::function key_left_; 130 | std::function key_right_; 131 | 132 | std::map> left_map_; 133 | std::map> right_map_; 134 | 135 | std::vector> buf_; 136 | }; 137 | 138 | template 139 | Join make_join(Puller* left, Puller* right, 140 | std::function key_left, 141 | std::function key_right) { 142 | return Join(left, right, key_left, key_right); 143 | } 144 | 145 | } // namespace latticeflow 146 | 147 | #endif // PUSHPULL_MAP_H_ 148 | -------------------------------------------------------------------------------- /src/pushpull/join_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/join.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | #include "gtest/gtest.h" 8 | 9 | #include "pushpull/id.h" 10 | #include "pushpull/lambda.h" 11 | #include "pushpull/register.h" 12 | 13 | namespace latticeflow { 14 | namespace { 15 | 16 | class Nats { 17 | public: 18 | boost::optional operator()() { 19 | x_++; 20 | return x_; 21 | } 22 | 23 | private: 24 | int x_ = -1; 25 | }; 26 | 27 | class DoubleNats { 28 | public: 29 | boost::optional operator()() { 30 | if (i_ == 0) { 31 | i_++; 32 | } else { 33 | x_++; 34 | i_ = 0; 35 | } 36 | return x_; 37 | } 38 | 39 | private: 40 | int x_ = -1; 41 | int i_ = 1; 42 | }; 43 | 44 | } // namespace 45 | 46 | TEST(Join, Push) { 47 | using Key = int; 48 | using Left = int; 49 | using Right = int; 50 | using Out = std::pair; 51 | std::function key_left = [](int x) { return x; }; 52 | std::function key_right = [](int x) { return x; }; 53 | 54 | Register> x; 55 | auto join = make_join(&x, key_left, key_right); 56 | auto left = make_id(&join.left()); 57 | auto right = make_id(&join.right()); 58 | 59 | left.push(42); 60 | right.push(42); 61 | EXPECT_EQ(Out(42, 42), x.get()); 62 | 63 | left.push(43); 64 | right.push(43); 65 | EXPECT_EQ(Out(43, 43), x.get()); 66 | } 67 | 68 | TEST(Join, Pull) { 69 | using Key = int; 70 | using Left = int; 71 | using Right = int; 72 | using Out = std::pair; 73 | 74 | std::function key_left = [](int x) { return x; }; 75 | std::function key_right = [](int x) { return x; }; 76 | 77 | Nats left_nats; 78 | DoubleNats right_nats; 79 | Lambda left(left_nats); 80 | Lambda right(right_nats); 81 | auto join = make_join(&left, &right, key_left, key_right); 82 | 83 | EXPECT_EQ(boost::optional(), join.pull()); 84 | EXPECT_EQ(boost::optional(std::make_pair(0, 0)), join.pull()); 85 | EXPECT_EQ(boost::optional(), join.pull()); 86 | EXPECT_EQ(boost::optional(std::make_pair(0, 0)), join.pull()); 87 | EXPECT_EQ(boost::optional(), join.pull()); 88 | EXPECT_EQ(boost::optional(std::make_pair(1, 1)), join.pull()); 89 | EXPECT_EQ(boost::optional(), join.pull()); 90 | EXPECT_EQ(boost::optional(std::make_pair(1, 1)), join.pull()); 91 | } 92 | 93 | } // namespace latticeflow 94 | 95 | int main(int argc, char **argv) { 96 | ::testing::InitGoogleTest(&argc, argv); 97 | return RUN_ALL_TESTS(); 98 | } 99 | -------------------------------------------------------------------------------- /src/pushpull/lambda.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_LAMBDA_H_ 2 | #define PUSHPULL_LAMBDA_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Lambda; 14 | 15 | template 16 | class Lambda : public Pusher { 17 | public: 18 | explicit Lambda(std::function f) : f_(f) {} 19 | void push(T x) override { f_(x); } 20 | 21 | private: 22 | std::function f_; 23 | }; 24 | 25 | template 26 | class Lambda : public Puller { 27 | public: 28 | explicit Lambda(std::function()> f) : f_(f) {} 29 | 30 | boost::optional pull() override { return f_(); } 31 | 32 | private: 33 | std::function()> f_; 34 | }; 35 | 36 | } // namespace latticeflow 37 | 38 | #endif // PUSHPULL_LAMBDA_H_ 39 | -------------------------------------------------------------------------------- /src/pushpull/lambda_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/lambda.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | namespace latticeflow { 7 | namespace { 8 | 9 | class Nats { 10 | public: 11 | boost::optional operator()() { 12 | x_++; 13 | return x_; 14 | } 15 | 16 | private: 17 | int x_ = -1; 18 | }; 19 | 20 | } // namespace 21 | 22 | TEST(Lambda, Push) { 23 | int y = 0; 24 | int* yp = &y; 25 | Lambda l([yp](int x) { *yp = x; }); 26 | 27 | l.push(42); 28 | EXPECT_EQ(42, y); 29 | l.push(100); 30 | EXPECT_EQ(100, y); 31 | } 32 | 33 | TEST(Lambda, Pull) { 34 | Nats n; 35 | Lambda l(n); 36 | EXPECT_EQ(boost::optional(0), l.pull()); 37 | EXPECT_EQ(boost::optional(1), l.pull()); 38 | EXPECT_EQ(boost::optional(2), l.pull()); 39 | } 40 | 41 | } // namespace latticeflow 42 | 43 | int main(int argc, char** argv) { 44 | ::testing::InitGoogleTest(&argc, argv); 45 | return RUN_ALL_TESTS(); 46 | } 47 | -------------------------------------------------------------------------------- /src/pushpull/map.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_FILTER_H_ 2 | #define PUSHPULL_FILTER_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class Map; 16 | 17 | template 18 | class Map : public Pusher { 19 | public: 20 | explicit Map(std::function f, Pusher* downstream) 21 | : f_(f), downstream_(downstream) {} 22 | 23 | void push(From x) override { downstream_->push(f_(x)); } 24 | 25 | private: 26 | std::function f_; 27 | Pusher* downstream_; 28 | }; 29 | 30 | template 31 | Map make_map(std::function f, 32 | Pusher* downstream) { 33 | return Map(f, downstream); 34 | } 35 | 36 | template 37 | class Map : public Puller { 38 | public: 39 | explicit Map(std::function f, Puller* upstream) 40 | : f_(f), upstream_(upstream) {} 41 | 42 | boost::optional pull() override { 43 | boost::optional x = upstream_->pull(); 44 | if (x) { 45 | return f_(*x); 46 | } else { 47 | return {}; 48 | } 49 | } 50 | 51 | private: 52 | std::function f_; 53 | Puller* upstream_; 54 | }; 55 | 56 | template 57 | Map make_map(std::function f, 58 | Puller* upstream) { 59 | return Map(f, upstream); 60 | } 61 | 62 | } // namespace latticeflow 63 | 64 | #endif // PUSHPULL_FILTER_H_ 65 | -------------------------------------------------------------------------------- /src/pushpull/map_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/map.h" 2 | 3 | #include 4 | 5 | #include "boost/optional.hpp" 6 | #include "gtest/gtest.h" 7 | 8 | #include "pushpull/register.h" 9 | 10 | namespace latticeflow { 11 | 12 | TEST(Map, Push) { 13 | Register x; 14 | std::function f = [](int x) { return x % 2 == 0; }; 15 | auto is_even = make_map(f, &x); 16 | 17 | is_even.push(0); 18 | EXPECT_EQ(true, x.get()); 19 | is_even.push(1); 20 | EXPECT_EQ(false, x.get()); 21 | is_even.push(2); 22 | EXPECT_EQ(true, x.get()); 23 | } 24 | 25 | TEST(Map, Pull) { 26 | std::function f = [](int x) { return x % 2 == 0; }; 27 | 28 | { 29 | Register x(0); 30 | auto is_even = make_map(f, &x); 31 | EXPECT_EQ(boost::optional(true), is_even.pull()); 32 | } 33 | 34 | { 35 | Register x(1); 36 | auto is_even = make_map(f, &x); 37 | EXPECT_EQ(boost::optional(false), is_even.pull()); 38 | } 39 | 40 | { 41 | Register x(2); 42 | auto is_even = make_map(f, &x); 43 | EXPECT_EQ(boost::optional(true), is_even.pull()); 44 | } 45 | 46 | { 47 | Register x(3); 48 | auto is_even = make_map(f, &x); 49 | EXPECT_EQ(boost::optional(false), is_even.pull()); 50 | } 51 | } 52 | 53 | } // namespace latticeflow 54 | 55 | int main(int argc, char **argv) { 56 | ::testing::InitGoogleTest(&argc, argv); 57 | return RUN_ALL_TESTS(); 58 | } 59 | -------------------------------------------------------------------------------- /src/pushpull/phantoms.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_PHANTOMS_H_ 2 | #define PUSHPULL_PHANTOMS_H_ 3 | 4 | namespace latticeflow { 5 | 6 | struct Push; 7 | struct Pull; 8 | struct PushPull; 9 | 10 | } // namespace latticeflow 11 | 12 | #endif // PUSHPULL_PHANTOMS_H_ 13 | -------------------------------------------------------------------------------- /src/pushpull/puller.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_PULLER_H_ 2 | #define PUSHPULL_PULLER_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | namespace latticeflow { 7 | 8 | template 9 | class Puller { 10 | public: 11 | virtual boost::optional pull() = 0; 12 | }; 13 | 14 | } // namespace latticeflow 15 | 16 | #endif // PUSHPULL_PULLER_H_ 17 | -------------------------------------------------------------------------------- /src/pushpull/pusher.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_PUSHER_H_ 2 | #define PUSHPULL_PUSHER_H_ 3 | 4 | namespace latticeflow { 5 | 6 | template 7 | class Pusher { 8 | public: 9 | virtual void push(T x) = 0; 10 | }; 11 | 12 | } // namespace latticeflow 13 | 14 | #endif // PUSHPULL_PUSHER_H_ 15 | -------------------------------------------------------------------------------- /src/pushpull/queue.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_QUEUE_H_ 2 | #define PUSHPULL_QUEUE_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class Queue : public Pusher, Puller { 16 | public: 17 | void push(T x) override { xs_.push(std::move(x)); } 18 | 19 | boost::optional pull() override { 20 | if (xs_.size() == 0) { 21 | return {}; 22 | } else { 23 | T x = std::move(xs_.front()); 24 | xs_.pop(); 25 | return x; 26 | } 27 | } 28 | 29 | private: 30 | std::queue xs_; 31 | }; 32 | 33 | } // namespace latticeflow 34 | 35 | #endif // PUSHPULL_QUEUE_H_ 36 | -------------------------------------------------------------------------------- /src/pushpull/queue_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/queue.h" 2 | 3 | #include 4 | 5 | #include "boost/optional.hpp" 6 | #include "gtest/gtest.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(Queue, PushPull) { 11 | Queue q; 12 | 13 | q.push(0); 14 | q.push(1); 15 | q.push(2); 16 | EXPECT_EQ(boost::optional(0), q.pull()); 17 | EXPECT_EQ(boost::optional(1), q.pull()); 18 | q.push(3); 19 | q.push(4); 20 | EXPECT_EQ(boost::optional(2), q.pull()); 21 | EXPECT_EQ(boost::optional(3), q.pull()); 22 | EXPECT_EQ(boost::optional(4), q.pull()); 23 | EXPECT_EQ(boost::optional(), q.pull()); 24 | } 25 | 26 | } // namespace latticeflow 27 | 28 | int main(int argc, char **argv) { 29 | ::testing::InitGoogleTest(&argc, argv); 30 | return RUN_ALL_TESTS(); 31 | } 32 | -------------------------------------------------------------------------------- /src/pushpull/register.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_REGISTER_H_ 2 | #define PUSHPULL_REGISTER_H_ 3 | 4 | #include "boost/optional.hpp" 5 | 6 | #include "pushpull/phantoms.h" 7 | #include "pushpull/puller.h" 8 | #include "pushpull/pusher.h" 9 | 10 | namespace latticeflow { 11 | 12 | template 13 | class Register; 14 | 15 | template 16 | class Register : public Pusher { 17 | public: 18 | void push(T x) override { x_ = x; } 19 | const T& get() { return x_; } 20 | 21 | private: 22 | T x_; 23 | }; 24 | 25 | template 26 | class Register : public Puller { 27 | public: 28 | explicit Register(T x) : x_(x) {} 29 | 30 | boost::optional pull() override { return x_; } 31 | 32 | private: 33 | T x_; 34 | }; 35 | 36 | } // namespace latticeflow 37 | 38 | #endif // PUSHPULL_REGISTER_H_ 39 | -------------------------------------------------------------------------------- /src/pushpull/register_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/register.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | namespace latticeflow { 7 | 8 | TEST(Register, Push) { 9 | Register x; 10 | x.push(42); 11 | EXPECT_EQ(42, x.get()); 12 | } 13 | 14 | TEST(Register, Pull) { 15 | Register x(42); 16 | EXPECT_EQ(boost::optional(42), x.pull()); 17 | } 18 | 19 | } // namespace latticeflow 20 | 21 | int main(int argc, char **argv) { 22 | ::testing::InitGoogleTest(&argc, argv); 23 | return RUN_ALL_TESTS(); 24 | } 25 | -------------------------------------------------------------------------------- /src/pushpull/round_robin.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_ROUND_ROBIN_H_ 2 | #define PUSHPULL_ROUND_ROBIN_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class RoundRobin; 16 | 17 | template 18 | class RoundRobin : public Pusher { 19 | public: 20 | explicit RoundRobin(std::vector*> downstreams) 21 | : downstreams_(std::move(downstreams)) {} 22 | 23 | void push(T x) override { 24 | if (downstreams_.size() > 0) { 25 | downstreams_[index_]->push(x); 26 | index_ = (index_ + 1) % downstreams_.size(); 27 | } 28 | } 29 | 30 | private: 31 | std::vector*> downstreams_; 32 | int index_ = 0; 33 | }; 34 | 35 | template 36 | RoundRobin make_round_robin(std::vector*> downstreams) { 37 | return RoundRobin(std::move(downstreams)); 38 | } 39 | 40 | template 41 | class RoundRobin : public Puller { 42 | public: 43 | explicit RoundRobin(std::vector*> upstreams) 44 | : upstreams_(std::move(upstreams)) {} 45 | 46 | boost::optional pull() override { 47 | if (upstreams_.size() > 0) { 48 | boost::optional x = upstreams_[index_]->pull(); 49 | index_ = (index_ + 1) % upstreams_.size(); 50 | return x; 51 | } else { 52 | return {}; 53 | } 54 | } 55 | 56 | private: 57 | std::vector*> upstreams_; 58 | int index_ = 0; 59 | }; 60 | 61 | template 62 | RoundRobin make_round_robin(std::vector*> upstreams) { 63 | return RoundRobin(std::move(upstreams)); 64 | } 65 | 66 | } // namespace latticeflow 67 | 68 | #endif // PUSHPULL_ROUND_ROBIN_H_ 69 | -------------------------------------------------------------------------------- /src/pushpull/round_robin_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/round_robin.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/register.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(RoundRobin, PushEmpty) { 11 | RoundRobin round_robin({}); 12 | round_robin.push(1); 13 | } 14 | 15 | TEST(RoundRobin, Push) { 16 | Register a; 17 | Register b; 18 | Register c; 19 | a.push(0); 20 | b.push(0); 21 | c.push(0); 22 | auto round_robin = make_round_robin({&a, &b, &c}); 23 | 24 | round_robin.push(1); 25 | EXPECT_EQ(1, a.get()); 26 | EXPECT_EQ(0, b.get()); 27 | EXPECT_EQ(0, c.get()); 28 | 29 | round_robin.push(2); 30 | EXPECT_EQ(1, a.get()); 31 | EXPECT_EQ(2, b.get()); 32 | EXPECT_EQ(0, c.get()); 33 | 34 | round_robin.push(3); 35 | EXPECT_EQ(1, a.get()); 36 | EXPECT_EQ(2, b.get()); 37 | EXPECT_EQ(3, c.get()); 38 | 39 | round_robin.push(4); 40 | EXPECT_EQ(4, a.get()); 41 | EXPECT_EQ(2, b.get()); 42 | EXPECT_EQ(3, c.get()); 43 | 44 | round_robin.push(5); 45 | EXPECT_EQ(4, a.get()); 46 | EXPECT_EQ(5, b.get()); 47 | EXPECT_EQ(3, c.get()); 48 | 49 | round_robin.push(6); 50 | EXPECT_EQ(4, a.get()); 51 | EXPECT_EQ(5, b.get()); 52 | EXPECT_EQ(6, c.get()); 53 | } 54 | 55 | TEST(RoundRobin, Pull) { 56 | Register a(0); 57 | Register b(1); 58 | Register c(2); 59 | auto round_robin = make_round_robin({&a, &b, &c}); 60 | 61 | EXPECT_EQ(boost::optional(0), round_robin.pull()); 62 | EXPECT_EQ(boost::optional(1), round_robin.pull()); 63 | EXPECT_EQ(boost::optional(2), round_robin.pull()); 64 | EXPECT_EQ(boost::optional(0), round_robin.pull()); 65 | EXPECT_EQ(boost::optional(1), round_robin.pull()); 66 | EXPECT_EQ(boost::optional(2), round_robin.pull()); 67 | } 68 | 69 | } // namespace latticeflow 70 | 71 | int main(int argc, char **argv) { 72 | ::testing::InitGoogleTest(&argc, argv); 73 | return RUN_ALL_TESTS(); 74 | } 75 | -------------------------------------------------------------------------------- /src/pushpull/tee.h: -------------------------------------------------------------------------------- 1 | #ifndef PUSHPULL_ROUND_TEE_H_ 2 | #define PUSHPULL_ROUND_TEE_H_ 3 | 4 | #include 5 | 6 | #include "boost/optional.hpp" 7 | 8 | #include "pushpull/phantoms.h" 9 | #include "pushpull/puller.h" 10 | #include "pushpull/pusher.h" 11 | 12 | namespace latticeflow { 13 | 14 | template 15 | class Tee; 16 | 17 | template 18 | class Tee : public Pusher { 19 | public: 20 | explicit Tee(std::vector*> downstreams) 21 | : downstreams_(std::move(downstreams)) {} 22 | 23 | void push(T x) override { 24 | for (Pusher* downstream : downstreams_) { 25 | downstream->push(x); 26 | } 27 | } 28 | 29 | private: 30 | std::vector*> downstreams_; 31 | }; 32 | 33 | template 34 | Tee make_tee(std::vector*> downstreams) { 35 | return Tee(std::move(downstreams)); 36 | } 37 | 38 | } // namespace latticeflow 39 | 40 | #endif // PUSHPULL_ROUND_TEE_H_ 41 | -------------------------------------------------------------------------------- /src/pushpull/tee_test.cc: -------------------------------------------------------------------------------- 1 | #include "pushpull/tee.h" 2 | 3 | #include "boost/optional.hpp" 4 | #include "gtest/gtest.h" 5 | 6 | #include "pushpull/register.h" 7 | 8 | namespace latticeflow { 9 | 10 | TEST(Tee, PushEmpty) { 11 | Tee tee({}); 12 | tee.push(1); 13 | } 14 | 15 | TEST(Tee, Push) { 16 | Register a; 17 | Register b; 18 | Register c; 19 | a.push(0); 20 | b.push(0); 21 | c.push(0); 22 | auto tee = make_tee({&a, &b, &c}); 23 | 24 | tee.push(1); 25 | EXPECT_EQ(1, a.get()); 26 | EXPECT_EQ(1, b.get()); 27 | EXPECT_EQ(1, c.get()); 28 | 29 | tee.push(2); 30 | EXPECT_EQ(2, a.get()); 31 | EXPECT_EQ(2, b.get()); 32 | EXPECT_EQ(2, c.get()); 33 | 34 | tee.push(3); 35 | EXPECT_EQ(3, a.get()); 36 | EXPECT_EQ(3, b.get()); 37 | EXPECT_EQ(3, c.get()); 38 | 39 | tee.push(4); 40 | EXPECT_EQ(4, a.get()); 41 | EXPECT_EQ(4, b.get()); 42 | EXPECT_EQ(4, c.get()); 43 | 44 | tee.push(5); 45 | EXPECT_EQ(5, a.get()); 46 | EXPECT_EQ(5, b.get()); 47 | EXPECT_EQ(5, c.get()); 48 | 49 | tee.push(6); 50 | EXPECT_EQ(6, a.get()); 51 | EXPECT_EQ(6, b.get()); 52 | EXPECT_EQ(6, c.get()); 53 | } 54 | 55 | } // namespace latticeflow 56 | 57 | int main(int argc, char **argv) { 58 | ::testing::InitGoogleTest(&argc, argv); 59 | return RUN_ALL_TESTS(); 60 | } 61 | -------------------------------------------------------------------------------- /src/vendor/boost/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://bit.ly/2dfK5L1 3 | 4 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 5 | 6 | include(ExternalProject) 7 | 8 | ExternalProject_Add(boost 9 | URL "http://kent.dl.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.gz" 10 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 11 | BUILD_IN_SOURCE 1 12 | UPDATE_COMMAND "" 13 | CONFIGURE_COMMAND ./bootstrap.sh --with-toolset=clang --with-libraries=context,coroutine2 14 | BUILD_COMMAND ./b2 toolset=clang cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" 15 | INSTALL_COMMAND "" 16 | ) 17 | 18 | set(BOOST_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/boost PARENT_SCOPE) 19 | set(BOOST_LINK_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/boost/stage/lib PARENT_SCOPE) 20 | -------------------------------------------------------------------------------- /src/vendor/googlebenchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://www.kaizou.org/2014/11/gtest-cmake/ 3 | # - http://bit.ly/2cx70Pk 4 | # - https://github.com/snikulov/google-test-examples 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 7 | 8 | include(ExternalProject) 9 | 10 | ExternalProject_Add(googlebenchmark 11 | GIT_REPOSITORY "https://github.com/google/benchmark" 12 | GIT_TAG "v1.0.0" 13 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 14 | CMAKE_ARGS "-DCMAKE_CXX_FLAGS=-stdlib=libc++" 15 | INSTALL_COMMAND "" 16 | ) 17 | 18 | ExternalProject_Get_Property(googlebenchmark SOURCE_DIR) 19 | SET(GBENCH_INCLUDE_DIRS ${SOURCE_DIR}/include PARENT_SCOPE) 20 | 21 | ExternalProject_Get_Property(googlebenchmark BINARY_DIR) 22 | SET(GBENCH_LIBS_DIR ${BINARY_DIR}/src PARENT_SCOPE) 23 | -------------------------------------------------------------------------------- /src/vendor/googletest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://www.kaizou.org/2014/11/gtest-cmake/ 3 | # - http://bit.ly/2cx70Pk 4 | # - https://github.com/snikulov/google-test-examples 5 | 6 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 7 | 8 | include(ExternalProject) 9 | 10 | ExternalProject_Add(googletest 11 | GIT_REPOSITORY "https://github.com/google/googletest" 12 | GIT_TAG "release-1.8.0" 13 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 14 | CMAKE_ARGS "-DCMAKE_CXX_FLAGS=-stdlib=libc++" 15 | INSTALL_COMMAND "" 16 | ) 17 | 18 | ExternalProject_Get_Property(googletest SOURCE_DIR) 19 | set(GTEST_INCLUDE_DIRS ${SOURCE_DIR}/googletest/include PARENT_SCOPE) 20 | 21 | ExternalProject_Get_Property(googletest BINARY_DIR) 22 | set(GTEST_LIBS_DIR ${BINARY_DIR}/googlemock/gtest PARENT_SCOPE) 23 | -------------------------------------------------------------------------------- /src/vendor/zeromq/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://zeromq.org/intro:get-the-software 3 | # - http://bit.ly/2dK0UBT 4 | # 5 | # Remember to have libtool, pkg-config, build-essential, autoconf, and automake 6 | # installed. 7 | 8 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 9 | 10 | include(ExternalProject) 11 | 12 | SET(CXX "clang++ -stdlib=c++") 13 | 14 | ExternalProject_Add(zeromq 15 | URL "https://github.com/zeromq/zeromq4-1/releases/download/v4.1.5/zeromq-4.1.5.tar.gz" 16 | PREFIX ${CMAKE_CURRENT_BINARY_DIR} 17 | BUILD_IN_SOURCE 1 18 | UPDATE_COMMAND "" 19 | CONFIGURE_COMMAND ./configure 20 | BUILD_COMMAND make 21 | INSTALL_COMMAND "" 22 | ) 23 | 24 | set(ZEROMQ_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/zeromq/include PARENT_SCOPE) 25 | set(ZEROMQ_LINK_DIRS ${CMAKE_CURRENT_BINARY_DIR}/src/zeromq/.libs PARENT_SCOPE) 26 | -------------------------------------------------------------------------------- /src/vendor/zeromqcpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Resources: 2 | # - http://zeromq.org/intro:get-the-software 3 | # - http://bit.ly/2dK0UBT 4 | # 5 | # Remember to have libtool, pkg-config, build-essential, autoconf, and automake 6 | # installed. 7 | 8 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 9 | 10 | include(ExternalProject) 11 | 12 | ExternalProject_Add(zeromqcpp 13 | GIT_REPOSITORY "https://github.com/zeromq/cppzmq.git" 14 | GIT_TAG "master" 15 | BUILD_IN_SOURCE 1 16 | UPDATE_COMMAND "" 17 | CONFIGURE_COMMAND "" 18 | BUILD_COMMAND "" 19 | INSTALL_COMMAND "" 20 | ) 21 | 22 | set(ZEROMQCPP_INCLUDE_DIRS 23 | ${CMAKE_CURRENT_BINARY_DIR}/zeromqcpp-prefix/src/zeromqcpp PARENT_SCOPE) 24 | -------------------------------------------------------------------------------- /src/zmq_util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | CMAKE_MINIMUM_REQUIRED(VERSION 2.6) 2 | 3 | ADD_LIBRARY(zmq_util 4 | connection_id.cc 5 | enveloped_message.cc 6 | hexdump.cc 7 | msg_util.cc 8 | zmq_util.cc 9 | ) 10 | TARGET_LINK_LIBRARIES(zmq_util zmq) 11 | -------------------------------------------------------------------------------- /src/zmq_util/README.md: -------------------------------------------------------------------------------- 1 | # ZeroMQ Utilities 2 | This directory contains miscellaneous helper functions for working with the 3 | ZeroMQ library. 4 | -------------------------------------------------------------------------------- /src/zmq_util/connection_id.cc: -------------------------------------------------------------------------------- 1 | #include "zmq_util/connection_id.h" 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "zmq_util/msg_util.h" 8 | 9 | namespace latticeflow { 10 | 11 | ConnectionId ConnectionId::Empty() { return ConnectionId({}); } 12 | 13 | ConnectionId ConnectionId::Clone() const { 14 | std::vector clone(connection_ids_.size()); 15 | for (std::size_t i = 0; i < clone.size(); ++i) { 16 | clone[i].copy(&connection_ids_[i]); 17 | } 18 | return ConnectionId(std::move(clone)); 19 | } 20 | 21 | std::ostream& operator<<(std::ostream& out, const ConnectionId& cid) { 22 | bool first = true; 23 | for (const zmq::message_t& connection_id : cid.connection_ids_) { 24 | if (!first) { 25 | out << std::endl; 26 | } 27 | out << connection_id; 28 | first = false; 29 | } 30 | return out; 31 | } 32 | 33 | bool operator<(const ConnectionId& lhs, const ConnectionId& rhs) { 34 | auto comp = [](const zmq::message_t& lhs, const zmq::message_t& rhs) { 35 | return lhs < rhs; 36 | }; 37 | return std::lexicographical_compare( 38 | lhs.connection_ids_.cbegin(), lhs.connection_ids_.cbegin(), 39 | rhs.connection_ids_.cend(), rhs.connection_ids_.cend(), comp); 40 | } 41 | 42 | } // namespace latticeflow 43 | -------------------------------------------------------------------------------- /src/zmq_util/connection_id.h: -------------------------------------------------------------------------------- 1 | #ifndef ZMQ_UTIL_CONNECTION_ID_H_ 2 | #define ZMQ_UTIL_CONNECTION_ID_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "zmq.hpp" 8 | 9 | namespace latticeflow { 10 | 11 | struct EnvelopedMessage; 12 | 13 | // Multipart messages in ZeroMQ are often prefixed with a number of connection 14 | // identities. For example, ROUTER sockets internally maintain a map from 15 | // connection id to connection. When it receives a message over a connection, 16 | // it prefixes the received multipart message with the connection id of the 17 | // connection. A ConnectionId represents the connection id prefix. 18 | // 19 | // For example, a connection id prefixed multipart message may look something 20 | // like the following. A `ConnectionId` stores the connection identities and 21 | // empty delimiter in some abstracted form. 22 | // 23 | // +---+----------+ \ 24 | // frame 1 | 6 | 0x310831 | | 25 | // +---+----------+ | 26 | // +---+----------+ | 27 | // frame 2 | 6 | 0x913759 | | connection identity 28 | // +---+----------+ | 29 | // +---+----------+ | 30 | // frame 3 | 6 | 0x138481 | | 31 | // +---+----------+ / 32 | // +---+ \ 33 | // frame 4 | 0 | | empty delimiter 34 | // +---+ / 35 | // +---+------------+ \ 36 | // frame 5 | 8 | "hi world" | | data frame 37 | // +---+------------+ / 38 | // 39 | // Note that `ConnectionId`s have no public constructor. They must be 40 | // constructed by calling `recv_enveloped_msg`. 41 | // 42 | // See [1] for more information. 43 | // 44 | // [1]: http://zguide.zeromq.org/page:all#The-Extended-Reply-Envelope 45 | class ConnectionId { 46 | public: 47 | // Returns an empty ConnectionId. 48 | static ConnectionId Empty(); 49 | 50 | // Copy a `ConnectionId`. `ConnectionId` doesn't have a copy constructor to 51 | // avoid accidental copies. 52 | ConnectionId Clone() const; 53 | 54 | // Pretty print a `ConnectionId`. 55 | friend std::ostream& operator<<(std::ostream& out, const ConnectionId& cid); 56 | 57 | // Compare a `ConnectionId`. The details are unspecified. 58 | friend bool operator<(const ConnectionId& lhs, const ConnectionId& rhs); 59 | 60 | // `recv_enveloped_msg` is the only way to construct a `ConnectionId`. 61 | // `ConnectionId`s can't be used for much besides sending using 62 | // `send_enveloped_msg`. 63 | friend EnvelopedMessage recv_enveloped_msg(zmq::socket_t* socket); 64 | friend void send_enveloped_msg(EnvelopedMessage&& s, zmq::socket_t* socket); 65 | 66 | private: 67 | // See connection_ids_ documentation below. 68 | explicit ConnectionId(std::vector&& connection_ids) 69 | : connection_ids_(std::move(connection_ids)) {} 70 | 71 | // Internally, we store all connection identities and the empty delimiter in 72 | // a single vector of zeromq messages. 73 | std::vector connection_ids_; 74 | }; 75 | 76 | } // namespace latticeflow 77 | 78 | #endif // ZMQ_UTIL_CONNECTION_ID_H_ 79 | -------------------------------------------------------------------------------- /src/zmq_util/enveloped_message.cc: -------------------------------------------------------------------------------- 1 | #include "zmq_util/enveloped_message.h" 2 | 3 | #include "zmq_util/msg_util.h" 4 | 5 | namespace latticeflow { 6 | 7 | EnvelopedMessage Clone(const EnvelopedMessage& env) { 8 | zmq::message_t clone; 9 | clone.copy(&env.msg); 10 | return {env.cid.Clone(), std::move(clone)}; 11 | } 12 | 13 | std::ostream& operator<<(std::ostream& out, const EnvelopedMessage& emsg) { 14 | out << "Envelope" << std::endl 15 | << "========" << std::endl 16 | << emsg.cid << std::endl 17 | << "Message" << std::endl 18 | << "=======" << std::endl 19 | << emsg.msg; 20 | return out; 21 | } 22 | 23 | bool operator<(const EnvelopedMessage& lhs, const EnvelopedMessage& rhs) { 24 | return lhs.cid < rhs.cid && lhs.msg < rhs.msg; 25 | } 26 | 27 | } // namespace latticeflow 28 | -------------------------------------------------------------------------------- /src/zmq_util/enveloped_message.h: -------------------------------------------------------------------------------- 1 | #ifndef ZMQ_UTIL_ENVELOPED_MESSAGE_H_ 2 | #define ZMQ_UTIL_ENVELOPED_MESSAGE_H_ 3 | 4 | #include 5 | 6 | #include "zmq.hpp" 7 | 8 | #include "zmq_util/connection_id.h" 9 | 10 | namespace latticeflow { 11 | 12 | // Multipart messages in ZeroMQ are often prefixed with a number of connection 13 | // identities. For example, ROUTER sockets internally maintain a map from 14 | // connection id to connection. When it receives a message over a connection, 15 | // it prefixes the received multipart message with the connection id of the 16 | // connection. An EnvelopedMessage represents an connection id prefixed 17 | // multipart message. 18 | // 19 | // For example, an EnvelopedMessage message may looks something like the 20 | // following where `cid` stores some representation of the connection identity 21 | // and empty delimiter, and `msg` stores the data frame. 22 | // 23 | // +---+----------+ \ 24 | // frame 1 | 6 | 0x310831 | | 25 | // +---+----------+ | 26 | // +---+----------+ | 27 | // frame 2 | 6 | 0x913759 | | connection identity 28 | // +---+----------+ | 29 | // +---+----------+ | 30 | // frame 3 | 6 | 0x138481 | | 31 | // +---+----------+ / 32 | // +---+ \ 33 | // frame 4 | 0 | | empty delimiter 34 | // +---+ / 35 | // +---+------------+ \ 36 | // frame 5 | 8 | "hi world" | | data frame 37 | // +---+------------+ / 38 | // 39 | // See [1] for more information. 40 | // 41 | // [1]: http://zguide.zeromq.org/page:all#The-Extended-Reply-Envelope 42 | struct EnvelopedMessage { 43 | ConnectionId cid; 44 | zmq::message_t msg; 45 | }; 46 | 47 | // Copy `env`. `EnvelopedMessage` doesn't have a copy constructor to avoid 48 | // accidental copying of data. 49 | EnvelopedMessage Clone(const EnvelopedMessage& env); 50 | 51 | // Pretty print an `EnvelopedMessage`. 52 | std::ostream& operator<<(std::ostream& out, const EnvelopedMessage& emsg); 53 | 54 | // Compare two `EnvelopedMessage`s. The comparison is a pairwise comparison of 55 | // `cid` and `msg`. 56 | bool operator<(const EnvelopedMessage& lhs, const EnvelopedMessage& rhs); 57 | 58 | } // namespace latticeflow 59 | 60 | #endif // ZMQ_UTIL_ENVELOPED_MESSAGE_H_ 61 | -------------------------------------------------------------------------------- /src/zmq_util/hexdump.cc: -------------------------------------------------------------------------------- 1 | #include "zmq_util/hexdump.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace latticeflow { 10 | namespace { 11 | 12 | // `std::cout << Hex(data, size)` prints each byte of `data` in hex separated by 13 | // a space. For example, `Hex("the sun did not ", 16)` looks like this: 14 | // 15 | // 74 68 65 20 73 75 6e 20 64 69 64 20 6e 6f 74 20 16 | class Hex { 17 | public: 18 | Hex(const unsigned char* const data, const std::size_t size) 19 | : data_(data), size_(size) {} 20 | 21 | friend std::ostream& operator<<(std::ostream& out, const Hex& hex) { 22 | for (std::size_t i = 0; i < hex.size_; ++i) { 23 | if (i != 0) { 24 | out << ' '; 25 | } 26 | out << std::setw(2) << std::setfill('0') << std::hex << +hex.data_[i]; 27 | } 28 | return out; 29 | } 30 | 31 | private: 32 | const unsigned char* const data_; 33 | const std::size_t size_; 34 | }; 35 | 36 | // `std::cout << Ascii(data, size)` prints each printable byte in `data` or `.` 37 | // if the byte is not printable. For example `Hex("the sun did not ", 16)` 38 | // looks like this: 39 | // 40 | // the sun did not 41 | class Ascii { 42 | public: 43 | Ascii(const unsigned char* const data, const std::size_t size) 44 | : data_(data), size_(size) {} 45 | 46 | friend std::ostream& operator<<(std::ostream& out, const Ascii& ascii) { 47 | for (std::size_t i = 0; i < ascii.size_; ++i) { 48 | if (std::isprint(ascii.data_[i])) { 49 | out << ascii.data_[i]; 50 | } else { 51 | out << "."; 52 | } 53 | } 54 | return out; 55 | } 56 | 57 | private: 58 | const unsigned char* const data_; 59 | const std::size_t size_; 60 | }; 61 | 62 | } // namespace 63 | 64 | // index 65 | // width half row width half row width row width 66 | // _____ _____________________ _____________________ ________________ 67 | // / \ / \ / \ / \ 68 | // 0000000 74 68 65 20 73 75 6e 20 64 69 64 20 6e 6f 74 20 |the sun did not | 69 | // 0000010 73 68 69 6e 65 2e 20 69 74 20 77 61 73 20 74 6f |shine. it was to| 70 | // 0000020 6f 20 77 65 74 20 74 6f 20 70 6c 61 79 2e 0a 73 |o wet to play..s| 71 | // 0000030 6f 20 77 65 20 73 61 74 20 69 6e 20 74 68 65 20 |o we sat in the | 72 | // 0000040 68 6f 75 73 65 0a 61 6c 6c 20 74 68 61 74 20 63 |house.all that c| 73 | // 0000050 6f 6c 64 2c 20 63 6f 6c 64 2c 20 77 65 74 20 64 |old, cold, wet d| 74 | std::ostream& operator<<(std::ostream& out, const HexDump& hexdump) { 75 | const std::size_t index_width = 8; 76 | const std::size_t row_width = 16; 77 | const std::size_t half_row_width = row_width / 2; 78 | 79 | if (hexdump.size_ == 0) { 80 | out << std::setw(index_width) << std::setfill('0') << 0 << ' ' 81 | << std::string(3 * half_row_width, ' ') << ' ' 82 | << std::string(3 * half_row_width, ' ') << " ||"; 83 | return out; 84 | } 85 | 86 | // Output full rows. 87 | std::size_t i = 0; 88 | for (i = 0; i + row_width < hexdump.size_; i += row_width) { 89 | if (i != 0) { 90 | out << std::endl; 91 | } 92 | out << std::setw(index_width) << std::setfill('0') << i << " " 93 | << Hex(&hexdump.data_[i], half_row_width) << " " 94 | << Hex(&hexdump.data_[i + half_row_width], half_row_width) << " |" 95 | << Ascii(&hexdump.data_[i], row_width) << "|"; 96 | } 97 | 98 | const std::size_t num_chars_left = hexdump.size_ - i; 99 | 100 | if (num_chars_left > 0 && i != 0) { 101 | out << std::endl; 102 | } 103 | 104 | if (num_chars_left <= 0) { 105 | return out; 106 | } else if (num_chars_left <= half_row_width) { 107 | // Partial first half row. 108 | out << std::setw(index_width) << std::setfill('0') << i << " " 109 | << Hex(&hexdump.data_[i], num_chars_left) 110 | << std::string((half_row_width - num_chars_left) * 3, ' ') << ' ' 111 | << std::string(half_row_width * 3, ' ') << " |" 112 | << Ascii(&hexdump.data_[i], num_chars_left) << "|"; 113 | } else if (num_chars_left <= row_width) { 114 | // Partial second half row. 115 | out << std::setw(index_width) << std::setfill('0') << i << " " 116 | << Hex(&hexdump.data_[i], half_row_width) << " " 117 | << Hex(&hexdump.data_[i + half_row_width], 118 | num_chars_left - half_row_width) 119 | << std::string((row_width - num_chars_left) * 3, ' ') << " |" 120 | << Ascii(&hexdump.data_[i], num_chars_left) << "|"; 121 | } else { 122 | assert(false); 123 | } 124 | 125 | return out; 126 | } 127 | 128 | } // namespace latticeflow 129 | -------------------------------------------------------------------------------- /src/zmq_util/hexdump.h: -------------------------------------------------------------------------------- 1 | #ifndef ZMQ_UTIL_HEXDUMP_H_ 2 | #define ZMQ_UTIL_HEXDUMP_H_ 3 | 4 | #include 5 | #include 6 | 7 | namespace latticeflow { 8 | 9 | // Consider a file `cat.txt` with the following contents: 10 | // 11 | // the sun did not shine. it was too wet to play. 12 | // so we sat in the house 13 | // all that cold, cold, wet day. 14 | // 15 | // If we run `hexdump -C cat.txt`, we get the following output: 16 | // 17 | // 0000000 74 68 65 20 73 75 6e 20 64 69 64 20 6e 6f 74 20 |the sun did not | 18 | // 0000010 73 68 69 6e 65 2e 20 69 74 20 77 61 73 20 74 6f |shine. it was to| 19 | // 0000020 6f 20 77 65 74 20 74 6f 20 70 6c 61 79 2e 0a 73 |o wet to play..s| 20 | // 0000030 6f 20 77 65 20 73 61 74 20 69 6e 20 74 68 65 20 |o we sat in the | 21 | // 0000040 68 6f 75 73 65 0a 61 6c 6c 20 74 68 61 74 20 63 |house.all that c| 22 | // 0000050 6f 6c 64 2c 20 63 6f 6c 64 2c 20 77 65 74 20 64 |old, cold, wet d| 23 | // 0000060 61 79 2e 0a |ay..| 24 | // 0000064 25 | // 26 | // `hexdump` provides us with a nice way to visualize an array of bytes. This 27 | // class implements the `hexdump` pretty pretting. That is, `std::cout << 28 | // HexDump(data, size)` pretty prints `data`, an byte array of length `size`, 29 | // in a `hexdump -C` style. 30 | class HexDump { 31 | public: 32 | HexDump(const unsigned char* const data, const std::size_t size) 33 | : data_(data), size_(size) {} 34 | 35 | friend std::ostream& operator<<(std::ostream& out, const HexDump& hexdump); 36 | 37 | private: 38 | const unsigned char* const data_; 39 | const std::size_t size_; 40 | }; 41 | 42 | } // namespace latticeflow 43 | 44 | #endif // ZMQ_UTIL_HEXDUMP_H_ 45 | -------------------------------------------------------------------------------- /src/zmq_util/msg_util.cc: -------------------------------------------------------------------------------- 1 | #include "zmq_util/msg_util.h" 2 | 3 | #include 4 | 5 | #include "zmq_util/hexdump.h" 6 | 7 | namespace latticeflow { 8 | 9 | std::ostream& operator<<(std::ostream& out, const zmq::message_t& msg) { 10 | out << HexDump(reinterpret_cast(msg.data()), 11 | msg.size()); 12 | return out; 13 | } 14 | 15 | bool operator<(const zmq::message_t& lhs, const zmq::message_t& rhs) { 16 | const unsigned char* const lhs_begin = 17 | reinterpret_cast(lhs.data()); 18 | const unsigned char* const lhs_end = lhs_begin + lhs.size(); 19 | 20 | const unsigned char* const rhs_begin = 21 | reinterpret_cast(rhs.data()); 22 | const unsigned char* const rhs_end = rhs_begin + rhs.size(); 23 | 24 | return std::lexicographical_compare(lhs_begin, lhs_end, rhs_begin, rhs_end); 25 | } 26 | 27 | } // namespace latticeflow 28 | -------------------------------------------------------------------------------- /src/zmq_util/msg_util.h: -------------------------------------------------------------------------------- 1 | #ifndef ZMQ_UTIL_MSG_UTIL_H 2 | #define ZMQ_UTIL_MSG_UTIL_H 3 | 4 | #include 5 | 6 | #include "zmq.hpp" 7 | 8 | namespace latticeflow { 9 | 10 | // Pretty prints a message. 11 | std::ostream& operator<<(std::ostream& out, const zmq::message_t& msg); 12 | 13 | // Compares two messages. 14 | bool operator<(const zmq::message_t& lhs, const zmq::message_t& rhs); 15 | 16 | } // namespace latticeflow 17 | 18 | #endif // ZMQ_UTIL_MSG_UTIL_H 19 | -------------------------------------------------------------------------------- /src/zmq_util/zmq_util.cc: -------------------------------------------------------------------------------- 1 | #include "zmq_util/zmq_util.h" 2 | 3 | #include 4 | #include 5 | 6 | #include "zmq_util/hexdump.h" 7 | #include "zmq_util/msg_util.h" 8 | 9 | namespace latticeflow { 10 | 11 | std::string message_to_string(const zmq::message_t& message) { 12 | return std::string(static_cast(message.data()), message.size()); 13 | } 14 | 15 | zmq::message_t string_to_message(const std::string& s) { 16 | zmq::message_t msg(s.size()); 17 | memcpy(msg.data(), s.c_str(), s.size()); 18 | return msg; 19 | } 20 | 21 | void send_string(const std::string& s, zmq::socket_t* socket) { 22 | socket->send(string_to_message(s)); 23 | } 24 | 25 | std::string recv_string(zmq::socket_t* socket) { 26 | zmq::message_t message; 27 | socket->recv(&message); 28 | return message_to_string(message); 29 | } 30 | 31 | void send_msgs(std::vector&& msgs, zmq::socket_t* socket) { 32 | for (std::size_t i = 0; i < msgs.size(); ++i) { 33 | socket->send(msgs[i], i == msgs.size() - 1 ? 0 : ZMQ_SNDMORE); 34 | } 35 | } 36 | 37 | std::vector recv_msgs(zmq::socket_t* socket) { 38 | std::vector msgs; 39 | int more = true; 40 | std::size_t more_size = sizeof(more); 41 | while (more) { 42 | msgs.emplace_back(); 43 | socket->recv(&msgs.back()); 44 | socket->getsockopt(ZMQ_RCVMORE, static_cast(&more), &more_size); 45 | } 46 | return msgs; 47 | } 48 | 49 | EnvelopedMessage recv_enveloped_msg(zmq::socket_t* socket) { 50 | std::vector msgs = recv_msgs(socket); 51 | assert(msgs.size() > 0); 52 | zmq::message_t msg = std::move(msgs.back()); 53 | msgs.pop_back(); 54 | return {ConnectionId(std::move(msgs)), std::move(msg)}; 55 | } 56 | 57 | void send_enveloped_msg(EnvelopedMessage&& env, zmq::socket_t* socket) { 58 | for (zmq::message_t& msg : env.cid.connection_ids_) { 59 | socket->send(msg, ZMQ_SNDMORE); 60 | } 61 | socket->send(env.msg); 62 | } 63 | 64 | int poll(long timeout, std::vector* items) { 65 | return zmq::poll(items->data(), items->size(), timeout); 66 | } 67 | 68 | } // namespace latticeflow 69 | -------------------------------------------------------------------------------- /src/zmq_util/zmq_util.h: -------------------------------------------------------------------------------- 1 | #ifndef ZMQ_UTIL_ZMQ_UTIL_H_ 2 | #define ZMQ_UTIL_ZMQ_UTIL_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "zmq_util/enveloped_message.h" 13 | 14 | namespace latticeflow { 15 | 16 | // Converts the data within a `zmq::message_t` into a string. 17 | std::string message_to_string(const zmq::message_t& message); 18 | 19 | // Converts a string into a `zmq::message_t`. 20 | zmq::message_t string_to_message(const std::string& s); 21 | 22 | // `send` a string over the socket. 23 | void send_string(const std::string& s, zmq::socket_t* socket); 24 | 25 | // `recv` a string over the socket. 26 | std::string recv_string(zmq::socket_t* socket); 27 | 28 | // `send` a multipart message. 29 | void send_msgs(std::vector&& msgs, zmq::socket_t* socket); 30 | 31 | // `recv` an enveloped message over the socket. 32 | EnvelopedMessage recv_enveloped_msg(zmq::socket_t* socket); 33 | 34 | // `send` an enveloped message over the socket. 35 | void send_enveloped_msg(EnvelopedMessage&& emsg, zmq::socket_t* socket); 36 | 37 | // `recv` a multipart message. 38 | std::vector recv_msgs(zmq::socket_t* socket); 39 | 40 | // Wraps a proto into a message. 41 | template 42 | zmq::message_t proto_to_message(const Proto& proto) { 43 | std::string s; 44 | proto.SerializeToString(&s); 45 | zmq::message_t msg(s.size()); 46 | memcpy(msg.data(), s.c_str(), s.size()); 47 | return msg; 48 | } 49 | 50 | // Unwraps a message into a proto. 51 | template 52 | void message_to_proto(const zmq::message_t& msg, Proto* p) { 53 | std::string s = message_to_string(msg); 54 | p->ParseFromString(s); 55 | } 56 | 57 | // Serialize a proto and `send` it over the socket. 58 | template 59 | void send_proto(const RequestProto& request, zmq::socket_t* socket) { 60 | socket->send(proto_to_message(request)); 61 | } 62 | 63 | // `recv` a message and unserialize it into a proto. 64 | template 65 | void recv_proto(ResponseProto* reply, zmq::socket_t* socket) { 66 | zmq::message_t reply_msg; 67 | socket->recv(&reply_msg); 68 | message_to_proto(reply_msg, reply); 69 | } 70 | 71 | // Wraps a pointer into a message. 72 | template 73 | zmq::message_t pointer_to_message(const T* const p) { 74 | zmq::message_t msg(sizeof(const T* const)); 75 | memcpy(msg.data(), &p, sizeof(const T* const)); 76 | } 77 | 78 | // Unwraps a message into a pointer. 79 | template 80 | T* message_to_pointer(zmq::message_t* msg) { 81 | // NOLINT: this code is somewhat forced to be hacky due to the low-level 82 | // nature of the zeromq API. 83 | return *reinterpret_cast(msg->data()); // NOLINT 84 | } 85 | 86 | // `send` a pointer over the socket. 87 | template 88 | void send_pointer(const T* const p, zmq::socket_t* socket) { 89 | socket->send(pointer_to_message(p)); 90 | } 91 | 92 | // `recv` a pointer over the socket. 93 | template 94 | T* recv_pointer(zmq::socket_t* socket) { 95 | zmq::message_t msg; 96 | socket->recv(&msg); 97 | return message_to_pointer(msg); 98 | } 99 | 100 | // `poll` is a wrapper around `zmq::poll` that takes a vector instead of a 101 | // pointer and a size. 102 | int poll(long timeout, std::vector* items); 103 | 104 | } // namespace latticeflow 105 | 106 | #endif // ZMQ_UTIL_ZMQ_UTIL_H_ 107 | --------------------------------------------------------------------------------