├── scripts ├── .gitignore └── license.sh ├── examples ├── page-rank │ ├── testdata │ │ ├── CMakeLists.txt │ │ └── prg-convert.cpp │ ├── CMakeLists.txt │ └── README.md ├── CMakeLists.txt ├── word-count │ ├── testdata │ │ ├── CMakeLists.txt │ │ └── generate_text.cpp │ ├── CMakeLists.txt │ ├── run_wordcount.sh │ ├── README.md │ ├── seq_wc.cpp │ └── pico_wc.cpp ├── stock-market │ ├── testdata │ │ ├── CMakeLists.txt │ │ ├── tag_options.cpp │ │ └── nasdaq_europe.txt │ ├── CMakeLists.txt │ ├── common.hpp │ ├── README.md │ ├── defs.h │ ├── stock_pricing.cpp │ ├── seq_stock_pricing.cpp │ ├── binomial_tree.hpp │ ├── stock_pricing_stream.cpp │ ├── explicit_finite_difference.hpp │ └── stock_tweets.cpp └── common │ └── utils.hpp ├── tests ├── testdata │ ├── CMakeLists.txt │ ├── generate_pairs.cpp │ └── generate_lines.cpp ├── test_driver.cpp ├── run_stream_tests.sh ├── create_inputs_tests.sh ├── CMakeLists.txt ├── common │ ├── io.hpp │ ├── common_functions.hpp │ ├── basic_pipes.hpp │ └── utils.hpp ├── input_output_file.cpp ├── reduce_by_key.cpp ├── flatmap.cpp ├── read_from_stdin.cpp ├── wordcount.cpp └── streaming_reduce_by_key.cpp ├── cmake ├── GenerateVersionFromGit.cmake ├── CheckClangTools.cmake └── config.cmake ├── include └── pico │ ├── defines │ └── Global.hpp │ ├── Internals │ ├── PEGOptimization │ │ └── defs.hpp │ ├── Token.hpp │ ├── utils.hpp │ └── TimedToken.hpp │ ├── ff_implementation │ ├── SupportFFNodes │ │ ├── ForwardingNode.hpp │ │ ├── farms.hpp │ │ ├── emitters.hpp │ │ ├── collectors.hpp │ │ ├── PReduceCollector.hpp │ │ └── ByKeyEmitter.hpp │ ├── defs.hpp │ ├── iteration │ │ └── fixed_length.hpp │ ├── ff_config.hpp │ └── OperatorsFFNodes │ │ ├── InOut │ │ ├── WriteToStdOutFFNode.hpp │ │ ├── ReadFromStdInFFNode.hpp │ │ └── WriteToDiskFFNode.hpp │ │ ├── MapBatch.hpp │ │ ├── FMapBatch.hpp │ │ ├── FoldReduceBatch.hpp │ │ └── PReduceWin.hpp │ ├── TerminationCondition.hpp │ ├── pico.hpp │ ├── Operators │ ├── UnaryOperator.hpp │ ├── BinaryOperator.hpp │ ├── Reduce.hpp │ ├── InOut │ │ ├── OutputOperator.hpp │ │ ├── InputOperator.hpp │ │ ├── ReadFromHDFS.hpp │ │ ├── ReadFromFile.hpp │ │ ├── ReadFromStdIn.hpp │ │ ├── ReadFromSocket.hpp │ │ ├── WriteToStdOut.hpp │ │ └── WriteToDisk.hpp │ ├── FoldReduce.hpp │ ├── Operator.hpp │ ├── JoinFlatMapByKey.hpp │ └── ReduceByKey.hpp │ ├── WindowPolicy.hpp │ ├── KeyValue.hpp │ ├── FlatMapCollector.hpp │ └── PEGOptimizations.hpp ├── .travis.yml ├── .clang-format ├── CMakeLists.txt ├── README.md └── .gitignore /scripts/.gitignore: -------------------------------------------------------------------------------- 1 | licenseheaders/ 2 | -------------------------------------------------------------------------------- /examples/page-rank/testdata/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(prg-convert prg-convert.cpp) 2 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(page-rank) 2 | add_subdirectory(stock-market) 3 | add_subdirectory(word-count) -------------------------------------------------------------------------------- /examples/word-count/testdata/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(generate_text generate_text.cpp) 2 | 3 | FILE (COPY dictionary.txt 4 | DESTINATION .) -------------------------------------------------------------------------------- /examples/page-rank/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(pagerank pagerank.cpp) 2 | target_link_libraries(pagerank ${PICO_RUNTIME_LIB}) 3 | 4 | add_subdirectory(testdata) -------------------------------------------------------------------------------- /tests/testdata/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(generate_lines generate_lines.cpp) 2 | add_executable(generate_pairs generate_pairs.cpp) 3 | 4 | FILE (COPY dictionary.txt 5 | DESTINATION .) -------------------------------------------------------------------------------- /examples/word-count/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(pico_wc pico_wc.cpp) 2 | target_link_libraries(pico_wc ${PICO_RUNTIME_LIB}) 3 | 4 | add_executable(seq_wc seq_wc.cpp) 5 | 6 | add_subdirectory(testdata) -------------------------------------------------------------------------------- /examples/stock-market/testdata/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(generate_tweets generate_tweets.cpp) 2 | add_executable(tag_options tag_options.cpp) 3 | 4 | FILE (COPY nasdaq_europe.txt stock_options_64K.txt 5 | DESTINATION .) -------------------------------------------------------------------------------- /examples/stock-market/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | foreach(e stock_pricing stock_pricing_stream stock_tweets) 2 | add_executable(${e} ${e}.cpp) 3 | target_link_libraries(${e} ${PICO_RUNTIME_LIB}) 4 | endforeach(e) 5 | 6 | add_subdirectory(testdata) -------------------------------------------------------------------------------- /scripts/license.sh: -------------------------------------------------------------------------------- 1 | # must be executed from the script directory 2 | root=`pwd`/.. 3 | 4 | if [ ! -d licenseheaders ]; then 5 | git clone git@github.com:johann-petrak/licenseheaders.git 6 | fi 7 | cd licenseheaders 8 | 9 | for d in include tests examples; do 10 | python licenseheaders.py -v -o "alpha group, CS department, University of Torino" -n pico -u "https://github.com/alpha-unito/pico" -y `date +"%Y"` -t lgpl-v3 -d $root/$d 11 | done 12 | -------------------------------------------------------------------------------- /cmake/GenerateVersionFromGit.cmake: -------------------------------------------------------------------------------- 1 | # Add Git revision information to the variable VERS. 2 | # The revinios information is determined using git describe. 3 | function(add_version_from_git VERS) 4 | set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) 5 | 6 | if (EXISTS "${SOURCE_DIR}/.git") 7 | find_program(git_executable NAMES git git.exe git.cmd) 8 | if (git_executable) 9 | execute_process(COMMAND 10 | ${git_executable} describe --all --dirty --long 11 | WORKING_DIRECTORY ${SOURCE_DIR} 12 | TIMEOUT 5 13 | RESULT_VARIABLE git_result 14 | OUTPUT_VARIABLE git_output) 15 | 16 | if (git_result EQUAL 0) 17 | string(STRIP "${git_output}" git_ref_id) 18 | set(${VERS} ${git_ref_id} PARENT_SCOPE) 19 | endif() 20 | endif() 21 | endif() 22 | endfunction() 23 | -------------------------------------------------------------------------------- /cmake/CheckClangTools.cmake: -------------------------------------------------------------------------------- 1 | find_program( 2 | CLANG_TIDY_EXE 3 | NAMES "clang-tidy" 4 | DOC "Path to clang-tidy executable") 5 | 6 | if(CLANG_TIDY_EXE) 7 | set( 8 | DO_CLANG_TIDY "${CLANG_TIDY_EXE}" "-p=${SHAD_BINARY_DIR} -format-style=file") 9 | message(STATUS "ClangTidy found: ${CLANG_TIDY_EXE}") 10 | if (NOT CMAKE_EXPORT_COMPILE_COMMANDS) 11 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 12 | endif() 13 | else() 14 | message(STATUS "ClangTidy not found.") 15 | endif() 16 | 17 | find_program( 18 | CLANG_FORMAT_EXE 19 | NAMES "clang-format" 20 | DOC "Path to clang-format executable") 21 | 22 | if (CLANG_FORMAT_EXE) 23 | message(STATUS "ClangFormat found: ${CLANG_FORMAT_EXE}") 24 | if (NOT CMAKE_EXPORT_COMPILE_COMMANDS) 25 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 26 | endif() 27 | else() 28 | message(STATUS "ClangFormat not found.") 29 | endif() 30 | -------------------------------------------------------------------------------- /tests/test_driver.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #define CATCH_CONFIG_MAIN 22 | 23 | #include "catch.hpp" 24 | -------------------------------------------------------------------------------- /tests/run_stream_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## Copyright (c) 2019 alpha group, CS department, University of Torino. 4 | ## 5 | ## This file is part of pico 6 | ## (see https://github.com/alpha-unito/pico). 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify 9 | ## it under the terms of the GNU Lesser General Public License as published by 10 | ## the Free Software Foundation, either version 3 of the License, or 11 | ## (at your option) any later version. 12 | ## 13 | ## This program is distributed in the hope that it will be useful, 14 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | ## GNU Lesser General Public License for more details. 17 | ## 18 | ## You should have received a copy of the GNU Lesser General Public License 19 | ## along with this program. If not, see . 20 | ## 21 | 22 | echo "streaming tests" 23 | cat testdata/pairs.txt | nc -l -p 4000 -q 0 & 24 | ./stream_tests 25 | -------------------------------------------------------------------------------- /examples/page-rank/README.md: -------------------------------------------------------------------------------- 1 | A PiCo implementation of the well-known [PageRank](https://en.wikipedia.org/wiki/PageRank) algorithm. 2 | 3 | The code in `pagerank.cpp` shows the usage of **iterative pipelines**. 4 | 5 | ## Run the application 6 | See the home [README](../../README.md) for build instructions. 7 | 8 | This simplified implementation takes in input two text files: 9 | 10 | - a list of pages, one per line 11 | - a list of edges, one per line, where each edge is a pair of space-separated pages 12 | 13 | and produces a list of page-rank pairs. 14 | 15 | The `prg-convert` utility converts edge-only graphs to their pages+edges representation. 16 | Grab a graph from the [SNAP repository](https://snap.stanford.edu/data/index.html) and convert it: 17 | 18 | ```bash 19 | cd /path/to/build/examples/page-rank/testdata 20 | wget https://snap.stanford.edu/data/cit-HepPh.txt.gz 21 | gunzip cit-HepPh.txt.gz 22 | ./prg-convert cit-HepPh.txt foo 23 | ``` 24 | 25 | Compute the ranks: 26 | ```bash 27 | cd .. 28 | ./pagerank testdata/foo-nodes testdata/foo-edges bar 29 | ``` 30 | 31 | ## See the application graph 32 | To visualize the graph for the `pageRank` pipeline: 33 | 34 | ```bash 35 | dot -Tpng page-rank.dot -o page-rank.png 36 | ``` 37 | -------------------------------------------------------------------------------- /tests/create_inputs_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## Copyright (c) 2019 alpha group, CS department, University of Torino. 4 | ## 5 | ## This file is part of pico 6 | ## (see https://github.com/alpha-unito/pico). 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify 9 | ## it under the terms of the GNU Lesser General Public License as published by 10 | ## the Free Software Foundation, either version 3 of the License, or 11 | ## (at your option) any later version. 12 | ## 13 | ## This program is distributed in the hope that it will be useful, 14 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | ## GNU Lesser General Public License for more details. 17 | ## 18 | ## You should have received a copy of the GNU Lesser General Public License 19 | ## along with this program. If not, see . 20 | ## 21 | 22 | 23 | # generate input files 24 | echo "> generating input data" 25 | cd testdata 26 | make 27 | if [ ! -f lines.txt ]; then 28 | ./generate_lines dictionary.txt 1k > lines.txt 29 | fi 30 | if [ ! -f pairs.txt ]; then 31 | ./generate_pairs 1k > pairs.txt 32 | fi 33 | if [ ! -f pairs_64.txt ]; then 34 | ./generate_pairs 64 > pairs_64.txt 35 | fi 36 | cd .. 37 | -------------------------------------------------------------------------------- /include/pico/defines/Global.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef DEFINES_GLOBAL_HPP_ 22 | #define DEFINES_GLOBAL_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | 31 | namespace pico { 32 | 33 | struct { 34 | int MICROBATCH_SIZE = 8; 35 | } global_params; 36 | 37 | } /* namespace pico */ 38 | 39 | #endif /* DEFINES_GLOBAL_HPP_ */ 40 | -------------------------------------------------------------------------------- /examples/word-count/run_wordcount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## 3 | ## Copyright (c) 2019 alpha group, CS department, University of Torino. 4 | ## 5 | ## This file is part of pico 6 | ## (see https://github.com/alpha-unito/pico). 7 | ## 8 | ## This program is free software: you can redistribute it and/or modify 9 | ## it under the terms of the GNU Lesser General Public License as published by 10 | ## the Free Software Foundation, either version 3 of the License, or 11 | ## (at your option) any later version. 12 | ## 13 | ## This program is distributed in the hope that it will be useful, 14 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | ## GNU Lesser General Public License for more details. 17 | ## 18 | ## You should have received a copy of the GNU Lesser General Public License 19 | ## along with this program. If not, see . 20 | ## 21 | B=512 22 | N=20 23 | 24 | for j in 42 #1 2 4 8 16 32 48 25 | do 26 | fname=times_wordcount_w$j"_"b$B 27 | echo "[-w $j -b $B]" 28 | rm -rf $fname 29 | for i in `seq 1 $N` 30 | do 31 | echo $i / $N 32 | ./pico_wc -w $j -b $B testdata/words1m foo.txt 2> /dev/null | awk '{for (i=0; i<=NF; i++){if ($i == "in"){print $(i+1);exit}}}' >> $fname 33 | done 34 | done 35 | -------------------------------------------------------------------------------- /cmake/config.cmake: -------------------------------------------------------------------------------- 1 | # Configure checks 2 | 3 | set(CMAKE_CXX_STANDARD 17) 4 | 5 | ## Workaround for the intel compiler 6 | if (Intel) 7 | if (${CMAKE_VERSION} VERSION_LESS "3.6") 8 | if (CMAKE_CXX_STANDARD STREQUAL 17) 9 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") 10 | else() 11 | message(FATAL_ERROR "Unsupported C++ Standard requested") 12 | endif() 13 | endif() 14 | endif() 15 | 16 | # include files checks 17 | 18 | # library checks: 19 | if (PICO_RUNTIME_SYSTEM STREQUAL "FF") 20 | message(STATUS "Using FastFlow runtime system.") 21 | 22 | find_package(Threads REQUIRED) 23 | include_directories(${THREADS_PTHREADS_INCLUDE_DIR}) 24 | 25 | #todo FindFF 26 | include_directories(${PROJECT_SOURCE_DIR}) 27 | 28 | set(HAVE_FF 1) 29 | set(PICO_RUNTIME_LIB ${CMAKE_THREAD_LIBS_INIT}) 30 | else() 31 | message(FATAL_ERROR "${PICO_RUNTIME_SYSTEM} is not a supported runtime system.") 32 | endif() 33 | 34 | if (PICO_ENABLE_UNIT_TEST) 35 | include_directories(tests/include) 36 | endif() 37 | 38 | # tools: 39 | include(CheckClangTools) 40 | 41 | # Documentation: 42 | if (PICO_ENABLE_DOXYGEN) 43 | message(STATUS "Doxygen enabled.") 44 | find_package(Doxygen REQUIRED) 45 | add_custom_target(doxygen ALL) 46 | else() 47 | message(STATUS "Doxygen disabled.") 48 | endif() 49 | 50 | -------------------------------------------------------------------------------- /examples/stock-market/common.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_STOCK_MARKET_COMMON_HPP_ 22 | #define EXAMPLES_STOCK_MARKET_COMMON_HPP_ 23 | 24 | #include "pico/pico.hpp" 25 | 26 | #include "defs.h" 27 | 28 | static pico::ReduceByKey SPReducer() { 29 | return pico::ReduceByKey( 30 | [](StockPrice p1, StockPrice p2) { return std::max(p1, p2); }); 31 | } 32 | 33 | #endif /* EXAMPLES_STOCK_MARKET_COMMON_HPP_ */ 34 | -------------------------------------------------------------------------------- /include/pico/Internals/PEGOptimization/defs.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_PEGOPTIMIZATION_DEFS_HPP_ 22 | #define INTERNALS_PEGOPTIMIZATION_DEFS_HPP_ 23 | 24 | namespace pico { 25 | 26 | class Operator; // forward 27 | 28 | enum PEGOptimization_t { MAP_PREDUCE, FMAP_PREDUCE, PJFMAP_PREDUCE }; 29 | 30 | union opt_args_t { 31 | Operator *op; 32 | }; 33 | 34 | } /* namespace pico */ 35 | 36 | #endif /* INTERNALS_PEGOPTIMIZATION_DEFS_HPP_ */ 37 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(BATCH_TESTS_SRCS flatmap.cpp input_output_file.cpp reduce_by_key.cpp 2 | wordcount.cpp flatmap_join_by_key.cpp iteration.cpp 3 | read_from_stdin.cpp ) 4 | set(TESTS_INPUTS_FILES testdata/lines.txt testdata/pairs.txt testdata/pairs_64.txt ) 5 | add_subdirectory(testdata) 6 | 7 | #batch tests 8 | add_executable(batch_tests ${BATCH_TESTS_SRCS} test_driver.cpp) 9 | 10 | target_link_libraries(batch_tests ${PICO_RUNTIME_LIB}) 11 | target_link_libraries(batch_tests Catch) 12 | add_test(NAME batch_unit_tests COMMAND batch_tests) 13 | 14 | 15 | #streaming tests 16 | set(STREAMING_TESTS_SRCS streaming_reduce_by_key.cpp ) 17 | add_executable(stream_tests ${STREAMING_TESTS_SRCS} test_driver.cpp) 18 | 19 | target_link_libraries(stream_tests ${PICO_RUNTIME_LIB}) 20 | target_link_libraries(stream_tests Catch) 21 | add_test(NAME stream_unit_tests COMMAND /bin/sh ${CMAKE_CURRENT_SOURCE_DIR}/run_stream_tests.sh) 22 | 23 | #generate tests inputs 24 | add_custom_target(tests_inputs DEPENDS ${TESTS_INPUTS_FILES} generate_lines generate_pairs) 25 | add_dependencies(batch_tests tests_inputs) 26 | add_dependencies(stream_tests tests_inputs) 27 | 28 | 29 | add_custom_command(OUTPUT ${TESTS_INPUTS_FILES} 30 | COMMAND /bin/sh ${CMAKE_CURRENT_SOURCE_DIR}/create_inputs_tests.sh 31 | ) 32 | -------------------------------------------------------------------------------- /include/pico/Internals/Token.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_TYPES_TOKEN_HPP_ 22 | #define INTERNALS_TYPES_TOKEN_HPP_ 23 | 24 | namespace pico { 25 | 26 | /** 27 | * Token descriptor for decorating collection data items with meta-data. 28 | */ 29 | template 30 | class Token { 31 | public: 32 | typedef T datatype; 33 | 34 | /* 35 | * create Token as decoration of a T value 36 | */ 37 | Token(const T&) {} 38 | }; 39 | 40 | } /* namespace pico */ 41 | 42 | #endif /* INTERNALS_TYPES_TOKENMD_HPP_ */ 43 | -------------------------------------------------------------------------------- /tests/common/io.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef TESTS_COMMON_IO_HPP_ 22 | #define TESTS_COMMON_IO_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | /* 30 | * read a file line by line into a vector of lines 31 | */ 32 | static std::vector read_lines(std::string fname) { 33 | std::vector res; 34 | std::ifstream in(fname); 35 | assert(in.is_open()); 36 | 37 | for (std::string line; std::getline(in, line);) 38 | if (!line.empty()) res.push_back(line); 39 | 40 | return res; 41 | } 42 | 43 | #endif /* TESTS_COMMON_IO_HPP_ */ 44 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | 3 | matrix: 4 | include: 5 | - os: osx 6 | osx_image: xcode10.1 7 | env: 8 | - MATRIX_EVAL="CC=clang && CXX=clang++" 9 | 10 | - os: linux 11 | addons: 12 | apt: 13 | sources: 14 | - ubuntu-toolchain-r-test 15 | packages: 16 | - g++-5 17 | 18 | env: 19 | - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" 20 | - os: linux 21 | addons: 22 | apt: 23 | sources: 24 | - ubuntu-toolchain-r-test 25 | packages: 26 | - g++-6 27 | 28 | env: 29 | - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6" 30 | - os: linux 31 | addons: 32 | apt: 33 | sources: 34 | - ubuntu-toolchain-r-test 35 | packages: 36 | - g++-7 37 | 38 | env: 39 | - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" 40 | - os: linux 41 | addons: 42 | apt: 43 | sources: 44 | - ubuntu-toolchain-r-test 45 | - llvm-toolchain-trusty-5.0 46 | packages: 47 | - g++-7 48 | - clang-5.0 49 | env: 50 | - MATRIX_EVAL="CC=clang-5.0 && CXX=clang++-5.0" 51 | 52 | before_install: 53 | - eval "${MATRIX_EVAL}" 54 | install: 55 | - git clone --single-branch https://github.com/fastflow/fastflow.git 56 | - ln -s fastflow/ff ff 57 | script: 58 | - mkdir build && cd build 59 | - cmake .. 60 | - make 61 | - ctest 62 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/ForwardingNode.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef FF_IMPLEMENTATION_SUPPORTFFNODES_FORWARDINGNODE_HPP_ 22 | #define FF_IMPLEMENTATION_SUPPORTFFNODES_FORWARDINGNODE_HPP_ 23 | 24 | #include 25 | 26 | #include "base_nodes.hpp" 27 | 28 | /* 29 | * TODO only works with non-decorating token 30 | */ 31 | 32 | /* implements the empty pipeline as forwarding node (i.e., identity map) */ 33 | class ForwardingNode : public base_filter { 34 | public: 35 | void kernel(pico::base_microbatch* task) { ff_send_out(task); } 36 | }; 37 | 38 | #endif /* FF_IMPLEMENTATION_SUPPORTFFNODES_FORWARDINGNODE_HPP_ */ 39 | -------------------------------------------------------------------------------- /examples/common/utils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_COMMON_UTILS_HPP_ 22 | #define EXAMPLES_COMMON_UTILS_HPP_ 23 | 24 | #include 25 | 26 | /* 27 | * progress bar 28 | */ 29 | void print_progress(float progress) { 30 | int barWidth = 70; 31 | int pos = barWidth * progress; 32 | 33 | std::cerr << "["; 34 | for (int i = 0; i < barWidth; ++i) { 35 | if (i < pos) 36 | std::cerr << "="; 37 | else if (i == pos) 38 | std::cerr << ">"; 39 | else 40 | std::cerr << " "; 41 | } 42 | std::cerr << "] " << int(progress * 100.0) << " %\r"; 43 | std::cerr.flush(); 44 | } 45 | 46 | #endif /* EXAMPLES_COMMON_UTILS_HPP_ */ 47 | -------------------------------------------------------------------------------- /tests/common/common_functions.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef TESTS__COMMON_FUNCTIONS_HPP_ 22 | #define TESTS__COMMON_FUNCTIONS_HPP_ 23 | 24 | #include "pico/pico.hpp" 25 | 26 | #include "./io.hpp" 27 | 28 | template 29 | /* parse test output into char-int pairs */ 30 | static std::unordered_map> result_fltmapjoin( 31 | std::string output_file) { 32 | std::unordered_map> observed; 33 | auto output_pairs_str = read_lines(output_file); 34 | for (auto pair : output_pairs_str) { 35 | auto kv = KV::from_string(pair); 36 | observed[kv.Key()].insert(kv.Value()); 37 | } 38 | return observed; 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/pico/Internals/utils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_UTILS_HPP_ 22 | #define INTERNALS_UTILS_HPP_ 23 | 24 | /* 25 | * todo - move 26 | */ 27 | #include 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | #include "pico/defines/Global.hpp" 34 | 35 | namespace pico { 36 | 37 | using TypeInfoRef = std::reference_wrapper; 38 | 39 | enum StructureType { 40 | BAG, 41 | STREAM // LIST, UBAG 42 | }; 43 | 44 | enum OpClass { 45 | MAP, 46 | FMAP, 47 | BMAP, 48 | BFMAP, 49 | REDUCE, 50 | FOLDREDUCE, 51 | INPUT, 52 | OUTPUT, 53 | MERGE, 54 | none 55 | }; 56 | 57 | typedef std::map st_map_t; 58 | 59 | } /* namespace pico */ 60 | 61 | #endif /* INTERNALS_UTILS_HPP_ */ 62 | -------------------------------------------------------------------------------- /tests/common/basic_pipes.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef TESTS_COMMON_BASIC_PIPES_HPP_ 22 | #define TESTS_COMMON_BASIC_PIPES_HPP_ 23 | 24 | #include "pico/pico.hpp" 25 | 26 | template 27 | static pico::Pipe pipe_pairs_creator(std::string input_file) { 28 | /* define input operator from file */ 29 | pico::ReadFromFile reader(input_file); 30 | 31 | /* define map operator */ 32 | pico::Map pairs_creator( 33 | [](std::string line) { // creates the pairs 34 | auto res = KV::from_string(line); 35 | return res; 36 | }); 37 | 38 | /* compose the pipeline */ 39 | 40 | auto p = pico::Pipe() // the empty pipeline 41 | .add(reader) // 42 | .add(pairs_creator); 43 | 44 | return p; 45 | } 46 | 47 | #endif /* TESTS_COMMON_BASIC_PIPES_HPP_ */ 48 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/defs.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_FF_IMPLEMENTATION_DEFS_HPP_ 22 | #define PICO_FF_IMPLEMENTATION_DEFS_HPP_ 23 | 24 | #include 25 | 26 | enum run_mode { DEFAULT, FORCE_NONBLOCKING }; 27 | 28 | /* FF tokens for pico protocol */ 29 | static size_t PICO_EOS = (size_t)ff::FF_EOS; 30 | static char *PICO_BEGIN = (char *)(PICO_EOS - 0xb); 31 | static char *PICO_END = (char *)(PICO_EOS - 0xc); 32 | 33 | static char *PICO_CSTREAM_BEGIN = (char *)(PICO_EOS - 0xd); 34 | static char *PICO_CSTREAM_END = (char *)(PICO_EOS - 0xe); 35 | static char *PICO_CSTREAM_FROM_LEFT = (char *)(PICO_EOS - 0xf); 36 | static char *PICO_CSTREAM_FROM_RIGHT = (char *)(PICO_EOS - 0x10); 37 | 38 | static inline bool is_sync(char *token) { 39 | return token <= PICO_BEGIN && token >= PICO_CSTREAM_END; 40 | } 41 | 42 | #endif /* PICO_FF_IMPLEMENTATION_DEFS_HPP_ */ 43 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/farms.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_SUPPORTFFNODES_FARMWRAPPER_HPP_ 22 | #define INTERNALS_FFOPERATORS_SUPPORTFFNODES_FARMWRAPPER_HPP_ 23 | 24 | #include 25 | 26 | /* 27 | * A non-ordering farm. 28 | */ 29 | class NonOrderingFarm : public ff::ff_farm { 30 | public: 31 | using lb_t = ff::ff_farm::lb_t; 32 | void setEmitterF(ff::ff_node* f) { this->add_emitter(f); } 33 | void setCollectorF(ff::ff_node* f) { this->add_collector(f); } 34 | }; 35 | 36 | /* 37 | * An ordering farm. 38 | */ 39 | 40 | class OrderingFarm : public ff::ff_farm { 41 | public: 42 | OrderingFarm() { set_ordered(); } 43 | void setEmitterF(ff::ff_node* f) { this->add_emitter(f); } 44 | 45 | void setCollectorF(ff::ff_node* f) { this->add_collector(f); } 46 | }; 47 | 48 | #endif /* INTERNALS_FFOPERATORS_SUPPORTFFNODES_FARMWRAPPER_HPP_ */ 49 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/emitters.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_EMITTER_HPP_ 22 | #define INTERNALS_FFOPERATORS_EMITTER_HPP_ 23 | 24 | #include 25 | 26 | #include "base_nodes.hpp" 27 | #include "farms.hpp" 28 | 29 | /* 30 | * Forwards non-sync tokens and broadcasts sync tokens. 31 | */ 32 | 33 | class ForwardingEmitter : public base_emitter { 34 | public: 35 | ForwardingEmitter(unsigned nw) : base_emitter(nw) {} 36 | 37 | void kernel(pico::base_microbatch *mb) { this->ff_send_out(mb); } 38 | }; 39 | 40 | /* 41 | * Forwards non-sync tokens and broadcasts sync tokens. 42 | * (for ordered farm) 43 | */ 44 | 45 | class OrdForwardingEmitter : public base_ord_emitter { 46 | public: 47 | OrdForwardingEmitter(unsigned nw) : base_ord_emitter(nw) {} 48 | 49 | void kernel(pico::base_microbatch *mb) { this->ff_send_out(mb); } 50 | }; 51 | 52 | #endif /* INTERNALS_FFOPERATORS_EMITTER_HPP_ */ 53 | -------------------------------------------------------------------------------- /include/pico/TerminationCondition.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_TERMINATIONCONDITION_HPP_ 22 | #define PICO_TERMINATIONCONDITION_HPP_ 23 | 24 | #include "pico/ff_implementation/iteration/fixed_length.hpp" 25 | 26 | namespace pico { 27 | 28 | class TerminationCondition { 29 | public: 30 | virtual ~TerminationCondition() {} 31 | 32 | virtual base_switch *iteration_switch() = 0; 33 | virtual TerminationCondition *clone() = 0; 34 | }; 35 | 36 | class FixedIterations : public TerminationCondition { 37 | public: 38 | FixedIterations(unsigned iters_) : iters(iters_) {} 39 | 40 | FixedIterations(const FixedIterations ©) : iters(copy.iters) {} 41 | 42 | base_switch *iteration_switch() { 43 | return new fixed_length_iteration_dispatcher(iters); 44 | } 45 | 46 | FixedIterations *clone() { return new FixedIterations(*this); } 47 | 48 | private: 49 | unsigned iters; 50 | }; 51 | 52 | } /* namespace pico */ 53 | 54 | #endif /* PICO_TERMINATIONCONDITION_HPP_ */ 55 | -------------------------------------------------------------------------------- /examples/stock-market/README.md: -------------------------------------------------------------------------------- 1 | Some PiCo pipelines for analyzing stock-market data. 2 | 3 | ## Run the applications 4 | See the home [README](../../README.md) for build instructions. 5 | 6 | ### Batch Processing 7 | The `stock_pricing.cpp` code is an example of batch pipeline (like [word-count](../word-count)), meaning file-based I/O. 8 | It takes in input a series of stock records and computes the maximum price for each stock name. 9 | Pricing is computed by the Black-Scholes function, from the [PARSEC](http://parsec.cs.princeton.edu) benchmark suite. 10 | 11 | ```bash 12 | cd /path/to/build/examples/stock-market 13 | ./stock_pricing testdata/stock_options_64K.txt max_prices.txt 14 | ``` 15 | 16 | ### Stream Processing 17 | The `stock_pricing_stream.cpp` is similar but works in a *streaming* fashion. 18 | A stream of records is read from a network socket and the output stream is written to the standard output. 19 | This example also shows window-based stream processing. 20 | 21 | The following code uses netcat for streaming the records to a socket on port `4000` of `localhost`: 22 | ```bash 23 | cat testdata/stock_options_64K.txt | nc -l 4000 & ./stock_pricing_stream localhost 4000 24 | ``` 25 | :warning: The `nc` synopsis may be different on your system. 26 | 27 | ### Analyzing Tweet Streams 28 | The `stock_tweets.cpp` processes a stream of tweets. 29 | It extracts tweets containing stock names and performs some window-based reduction on the resulting sub-stream. 30 | 31 | Generate 1000 synthetic tweets, some of them containing a stock name from the NADSAQ index: 32 | ```bash 33 | cd testdata 34 | ./generate_tweets nasdaq_europe.txt 1000 >> tweets.txt 35 | ``` 36 | 37 | Analyze them: 38 | ```bash 39 | cd .. 40 | cat testdata/tweets.txt | nc -l 4000 & ./stock_tweets testdata/nasdaq_europe.txt localhost 4000 41 | ``` 42 | :warning: The `nc` synopsis may be different on your system. 43 | -------------------------------------------------------------------------------- /include/pico/pico.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_PICO_HPP_ 22 | #define PICO_PICO_HPP_ 23 | 24 | 25 | /* implementation */ 26 | #include "pico/ff_implementation/FastFlowExecutor.hpp" 27 | 28 | /* basic */ 29 | #include "pico/FlatMapCollector.hpp" 30 | #include "pico/KeyValue.hpp" 31 | #include "pico/Pipe.hpp" 32 | #include "pico/SemanticGraph.hpp" 33 | #include "pico/WindowPolicy.hpp" 34 | 35 | /* operators */ 36 | #include "pico/Operators/FlatMap.hpp" 37 | #include "pico/Operators/FoldReduce.hpp" 38 | #include "pico/Operators/InOut/ReadFromFile.hpp" 39 | #include "pico/Operators/InOut/ReadFromSocket.hpp" 40 | #include "pico/Operators/InOut/ReadFromStdIn.hpp" 41 | #include "pico/Operators/InOut/WriteToDisk.hpp" 42 | #include "pico/Operators/InOut/WriteToStdOut.hpp" 43 | #include "pico/Operators/JoinFlatMapByKey.hpp" 44 | #include "pico/Operators/Map.hpp" 45 | #include "pico/Operators/Reduce.hpp" 46 | #include "pico/Operators/ReduceByKey.hpp" 47 | 48 | 49 | 50 | #endif /* PICO_PICO_HPP_ */ 51 | -------------------------------------------------------------------------------- /tests/input_output_file.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "pico/pico.hpp" 27 | 28 | #include "common/io.hpp" 29 | 30 | TEST_CASE("read and write", "read and write tag") { 31 | std::string input_file = "./testdata/lines.txt"; 32 | std::string output_file = "output.txt"; 33 | 34 | /* define i/o operators from/to file */ 35 | pico::ReadFromFile reader(input_file); 36 | pico::WriteToDisk writer(output_file); 37 | 38 | /* compose the pipeline */ 39 | auto io_file_pipe = pico::Pipe() // the empty pipeline 40 | .add(reader) 41 | .add(writer); 42 | 43 | io_file_pipe.run(); 44 | 45 | /* forget the order and compare */ 46 | auto input_lines = read_lines(input_file); 47 | auto output_lines = read_lines(output_file); 48 | std::sort(input_lines.begin(), input_lines.end()); 49 | std::sort(output_lines.begin(), output_lines.end()); 50 | 51 | REQUIRE(input_lines == output_lines); 52 | } 53 | -------------------------------------------------------------------------------- /tests/testdata/generate_pairs.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "pico/KeyValue.hpp" 28 | 29 | #include "../common/utils.hpp" 30 | 31 | typedef pico::KeyValue KV; 32 | 33 | int main(int argc, char** argv) { 34 | /* parse command line */ 35 | if (argc < 2) { 36 | std::cerr << "Usage: " << argv[0]; 37 | std::cerr << " \n"; 38 | return -1; 39 | } 40 | unsigned long long n_lines = get_size(argv[1]); 41 | 42 | /* generate and emit random lines */ 43 | std::default_random_engine rng; 44 | std::uniform_int_distribution sym_dist('a', 'a' + 25); 45 | std::uniform_int_distribution num_dist(-10, 10); 46 | for (unsigned line_i = 0; line_i < n_lines; ++line_i) { 47 | KV kv(sym_dist(rng), num_dist(rng)); 48 | std::cout << kv.to_string() << std::endl; 49 | print_progress((float)(line_i + 1) / n_lines); 50 | } 51 | 52 | std::cerr << "\ndone\n"; 53 | 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /include/pico/Operators/UnaryOperator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef OPERATORS_UNARYOPERATOR_HPP_ 22 | #define OPERATORS_UNARYOPERATOR_HPP_ 23 | 24 | #include "pico/Internals/PEGOptimization/defs.hpp" 25 | 26 | #include "Operator.hpp" 27 | 28 | namespace pico { 29 | 30 | /** 31 | * Base class for actor nodes with *one* input stream and *one* output stream, 32 | * either bound or unbound and grouped or plain. It is provided with methods for 33 | * input/output type checking. 34 | */ 35 | class base_UnaryOperator : public Operator { 36 | public: 37 | virtual ff::ff_node* node_operator(int, StructureType) = 0; 38 | 39 | virtual ff::ff_node* opt_node(int, PEGOptimization_t, StructureType, // 40 | opt_args_t) { 41 | assert(false); 42 | return nullptr; 43 | } 44 | }; 45 | 46 | template 47 | class UnaryOperator : public base_UnaryOperator { 48 | public: 49 | typedef In inT; 50 | typedef Out outT; 51 | }; 52 | 53 | } /* namespace pico */ 54 | 55 | #endif /* OPERATORS_UNARYOPERATOR_HPP_ */ 56 | -------------------------------------------------------------------------------- /examples/word-count/README.md: -------------------------------------------------------------------------------- 1 | Word-Count (a.k.a. the "Hello, World!" for data analytics) counts the occurrences of each distinct word from a text file. 2 | 3 | This examples shows the *killer feature* in the PiCo API: **there is no data**! 4 | A PiCo application is described only in terms of *processing* (i.e., pipeline stages) - 5 | rather than processing *and* data. 6 | 7 | A simple PiCo pipeline for word-count: 8 | 9 | 1. reads an input file line by line, by a `ReadFromFile` stage 10 | 2. tokenizes each line into words, by a `FlatMap` stage - a `FlatMap` maps an item (line) to multiple items (words) 11 | 3. maps each word `w` to a key-value pair <`w`,`1`>, by a `Map` stage 12 | 4. groups the pairs by word and sums up them, by a `ReduceByKey` stage 13 | 5. finally, the word-occurrences pairs <`w`,`n`> are written to an output file, by a `WriteToFile` stage 14 | 15 | In `pico_wc.cpp`, we show a common optimization known as stage fusion. The `wc` pipeline fuses step 3 into step 2, letting the `FlatMap` stage produce the <`w`,`1`> pairs from each word in the processed line. 16 | 17 | ## Run the application 18 | See the home [README](../../README.md) for build instructions. 19 | 20 | Generate 1k random lines: 21 | 22 | ```bash 23 | cd /path/to/build/examples/word-count 24 | cd testdata 25 | ./generate_text dictionary.txt 1000 >> lines.txt 26 | ``` 27 | 28 | Count those words: 29 | ```bash 30 | cd .. 31 | ./pico_wc testdata/lines.txt count.txt 32 | ``` 33 | 34 | :bulb: Parallelism degree can be set: 35 | - externally, by the application-wise `PARDEG` environment variable 36 | - within the code, for each operator, by passing an (optional) argument to operators' constructors; 37 | per-operator parallelism overrides `PARDEG` 38 | 39 | ## See the application graph 40 | Call the `to_dotfile()` function on a PiCo pipeline to produce a `dot` representation of its semantics. 41 | 42 | To visualize the graph for the `wc` pipeline: 43 | 44 | ```bash 45 | dot -Tpng word-count.dot -o word-count.png 46 | ``` 47 | -------------------------------------------------------------------------------- /examples/stock-market/defs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_STOCK_MARKET_DEFS_H_ 22 | #define EXAMPLES_STOCK_MARKET_DEFS_H_ 23 | 24 | #include "pico/KeyValue.hpp" 25 | 26 | #include "black_scholes.hpp" 27 | 28 | /* define some types */ 29 | typedef std::string StockName; 30 | typedef double StockPrice; 31 | 32 | typedef pico::KeyValue StockAndOption; 33 | typedef pico::KeyValue StockAndTweet; 34 | typedef pico::KeyValue StockAndCount; 35 | typedef pico::KeyValue StockAndPrice; 36 | 37 | // return the payoff of the function you want to evaluate 38 | // payoff from the European call option 39 | double payoff(double S, double strikePrice) { 40 | return std::max(S - strikePrice, 0.); 41 | } 42 | 43 | void parse_opt(OptionData &opt, char &ot, char *name, const std::string &in) { 44 | sscanf(in.c_str(), "%s %lf %lf %lf %lf %lf %lf %c %lf %lf", name, // 45 | &opt.s, &opt.strike, &opt.r, &opt.divq, &opt.v, &opt.t, // 46 | &ot, &opt.divs, &opt.DGrefval); 47 | } 48 | 49 | #endif /* EXAMPLES_STOCK_MARKET_DEFS_H_ */ 50 | -------------------------------------------------------------------------------- /include/pico/Operators/BinaryOperator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef OPERATORS_BINARYOPERATOR_HPP_ 22 | #define OPERATORS_BINARYOPERATOR_HPP_ 23 | 24 | #include "pico/Internals/PEGOptimization/defs.hpp" 25 | 26 | #include "Operator.hpp" 27 | 28 | namespace pico { 29 | 30 | /** 31 | * Base class for actor nodes with *two* input streams and *one* output stream, 32 | * either bound or unbound and grouped or plain. It is provided with methods for 33 | * input/output type checking. 34 | */ 35 | class base_BinaryOperator : public Operator { 36 | public: 37 | virtual ff::ff_node* node_operator(int, bool, StructureType) = 0; 38 | 39 | virtual ff::ff_node* opt_node(int, bool, PEGOptimization_t, StructureType, // 40 | opt_args_t) { 41 | assert(false); 42 | return nullptr; 43 | } 44 | }; 45 | 46 | template 47 | class BinaryOperator : public base_BinaryOperator { 48 | public: 49 | typedef In1 inFirstT; 50 | typedef In2 inSecondT; 51 | typedef Out outT; 52 | }; 53 | 54 | } /* namespace pico */ 55 | 56 | #endif /* OPERATORS_BINARYOPERATOR_HPP_ */ 57 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/iteration/fixed_length.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_FF_IMPLEMENTATION_ITERATION_FIXED_LENGTH_HPP_ 22 | #define PICO_FF_IMPLEMENTATION_ITERATION_FIXED_LENGTH_HPP_ 23 | 24 | #include "pico/Internals/Microbatch.hpp" 25 | 26 | #include "base_iteration.hpp" 27 | 28 | class fixed_length_iteration_dispatcher : public base_iteration_dispatcher { 29 | typedef pico::base_microbatch::tag_t tag_t; 30 | 31 | public: 32 | fixed_length_iteration_dispatcher(unsigned niters_) : niters(niters_) {} 33 | 34 | private: 35 | void go_ahead() { 36 | if (!closed()) { 37 | bool accepting = true; 38 | unsigned iters_cnt = n_iterations(); 39 | while (iters_cnt < niters && accepting) { 40 | new_iteration(); 41 | accepting = n_iterations() - iters_cnt; 42 | iters_cnt = n_iterations(); 43 | }; 44 | if (iters_cnt == niters) close(); 45 | } 46 | } 47 | 48 | void cstream_iteration_heartbeat_callback(tag_t) { go_ahead(); } 49 | 50 | void cstream_iteration_end_callback(tag_t) { go_ahead(); } 51 | 52 | const unsigned niters; 53 | }; 54 | 55 | #endif /* PICO_FF_IMPLEMENTATION_ITERATION_FIXED_LENGTH_HPP_ */ 56 | -------------------------------------------------------------------------------- /examples/stock-market/testdata/tag_options.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include /* srand, rand */ 22 | #include /* time */ 23 | #include 24 | #include 25 | #include 26 | 27 | int main(int argc, char** argv) { 28 | /* parse command line */ 29 | if (argc < 4) { 30 | std::cerr << "Usage: " << argv[0]; 31 | std::cerr << " \n"; 32 | return -1; 33 | } 34 | std::string IN_OPTIONS = argv[1]; 35 | std::string IN_TAGS = argv[2]; 36 | std::string OUT_TAGGED = argv[3]; 37 | 38 | /* open file streams */ 39 | std::ifstream tags_(IN_TAGS); 40 | std::ifstream options(IN_OPTIONS); 41 | std::ofstream tagged(OUT_TAGGED); 42 | 43 | std::vector tags; 44 | 45 | /* bring tags to memory */ 46 | std::string tag; 47 | while (tags_.good()) { 48 | tags_ >> tag; 49 | tags.push_back(tag); 50 | } 51 | 52 | /* tag each option with random tag */ 53 | char buf[1024]; 54 | while (options.good()) { 55 | options.getline(buf, 1024); 56 | tagged << tags[rand() % tags.size()]; 57 | tagged << "\t"; 58 | tagged << buf; 59 | tagged << std::endl; 60 | } 61 | 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/collectors.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_FARMCOLLECTOR_HPP_ 22 | #define INTERNALS_FFOPERATORS_FARMCOLLECTOR_HPP_ 23 | 24 | #include "base_nodes.hpp" 25 | 26 | class ForwardingCollector : public base_sync_duplicate { 27 | public: 28 | using base_sync_duplicate::base_sync_duplicate; 29 | 30 | void kernel(pico::base_microbatch *mb) { ff_send_out(mb); } 31 | }; 32 | 33 | /* unpacks and streams token-collectors */ 34 | template 35 | class UnpackingCollector : public base_sync_duplicate { 36 | typedef typename coll_t::cnode cnode_t; 37 | 38 | public: 39 | using base_sync_duplicate::base_sync_duplicate; 40 | 41 | void kernel(pico::base_microbatch *mb) { 42 | auto wmb = reinterpret_cast *>(mb); 43 | cnode_t *it_, *it = wmb->get(); 44 | /* send out all the micro-batches in the list */ 45 | while (it) { 46 | ff_send_out(reinterpret_cast(it->mb)); 47 | 48 | /* clean up and skip to the next micro-batch */ 49 | it_ = it; 50 | it = it->next; 51 | FREE(it_); 52 | }; 53 | DELETE(wmb); 54 | } 55 | }; 56 | 57 | #endif /* INTERNALS_FFOPERATORS_FARMCOLLECTOR_HPP_ */ 58 | -------------------------------------------------------------------------------- /tests/common/utils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_COMMON_UTILS_HPP_ 22 | #define EXAMPLES_COMMON_UTILS_HPP_ 23 | 24 | #include 25 | 26 | /* 27 | * progress bar 28 | */ 29 | static void print_progress(float progress) { 30 | int barWidth = 70; 31 | int pos = barWidth * progress; 32 | 33 | std::cerr << "["; 34 | for (int i = 0; i < barWidth; ++i) { 35 | if (i < pos) 36 | std::cerr << "="; 37 | else if (i == pos) 38 | std::cerr << ">"; 39 | else 40 | std::cerr << " "; 41 | } 42 | std::cerr << "] " << int(progress * 100.0) << " %\r"; 43 | std::cerr.flush(); 44 | } 45 | 46 | /* parse a size string */ 47 | static long long unsigned get_size(char *str) { 48 | long long unsigned size; 49 | char mod[32]; 50 | 51 | switch (sscanf(str, "%llu%1[mMkK]", &size, mod)) { 52 | case 1: 53 | return (size); 54 | case 2: 55 | switch (*mod) { 56 | case 'm': 57 | case 'M': 58 | return (size << 20); 59 | case 'k': 60 | case 'K': 61 | return (size << 10); 62 | default: 63 | return (size); 64 | } 65 | break; // suppress warning 66 | default: 67 | return (-1); 68 | } 69 | } 70 | 71 | #endif /* EXAMPLES_COMMON_UTILS_HPP_ */ 72 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/ff_config.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_FF_CONFIG_HPP_ 22 | #define INTERNALS_FFOPERATORS_FF_CONFIG_HPP_ 23 | 24 | #ifdef FF_ALLOC 25 | #include 26 | 27 | static inline void *MALLOC(size_t size) { return ff::ff_malloc(size); } 28 | static inline void FREE(void *ptr) { ff::ff_free(ptr); } 29 | static inline int POSIX_MEMALIGN(void **dst, size_t align, size_t size) { 30 | return ff::ff_posix_memalign(dst, align, size); 31 | } 32 | template 33 | static inline _Tp *NEW(_Args &&... __args) { 34 | auto ptr = (_Tp *)MALLOC(sizeof(_Tp)); 35 | return new (ptr) _Tp(std::forward<_Args>(__args)...); 36 | } 37 | template 38 | static inline void DELETE(_Tp *p) { 39 | (p)->~_Tp(); 40 | FREE(p); 41 | } 42 | #else 43 | #include 44 | static inline void *MALLOC(size_t size) { return ::malloc(size); } 45 | static inline void FREE(void *ptr) { ::free(ptr); } 46 | static inline int POSIX_MEMALIGN(void **dst, size_t align, size_t size) { 47 | return ::posix_memalign(dst, align, size); 48 | } 49 | template 50 | static inline _Tp *NEW(_Args &&... __args) { 51 | return new _Tp(std::forward<_Args>(__args)...); 52 | } 53 | template 54 | static inline void DELETE(_Tp *p) { 55 | delete p; 56 | } 57 | #endif 58 | 59 | #endif /* INTERNALS_FFOPERATORS_FF_CONFIG_HPP_ */ 60 | -------------------------------------------------------------------------------- /include/pico/Internals/TimedToken.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_TYPES_TIMEDTOKEN_HPP_ 22 | #define INTERNALS_TYPES_TIMEDTOKEN_HPP_ 23 | 24 | namespace pico { 25 | 26 | template 27 | class TimedToken { 28 | public: 29 | typedef T datatype; 30 | 31 | TimedToken() : data(nullptr), timestamp(0) {} 32 | 33 | TimedToken(const T& item_) : data(item_), timestamp(0) {} 34 | 35 | TimedToken(T&& item_, size_t timestamp_) 36 | : data(std::move(item_)), timestamp(timestamp_) {} 37 | 38 | TimedToken(T&& item) : data(std::move(item)), timestamp(0) {} 39 | 40 | TimedToken(T&& item_, const TimedToken& tt) 41 | : data(std::move(item_)), timestamp(tt.get_timestamp()) {} 42 | 43 | TimedToken(TimedToken&& tt) 44 | : data(std::move(tt.data)), timestamp(std::move(tt.timestamp)) {} 45 | 46 | TimedToken& operator=(TimedToken&& tt) { 47 | data = std::move(tt.data); 48 | timestamp = std::move(tt.timestamp); 49 | return *this; 50 | } 51 | 52 | friend std::ostream& operator<<(std::ostream& os, const TimedToken& tt) { 53 | os << "<" << tt.data << ", " << tt.timestamp << ">"; 54 | return os; 55 | } 56 | 57 | T& get_data() { return data; } 58 | 59 | inline size_t get_timestamp() const { return timestamp; } 60 | 61 | void set_timestamp(size_t t) { timestamp = t; } 62 | 63 | private: 64 | T data; 65 | size_t timestamp; 66 | }; 67 | 68 | } /* namespace pico */ 69 | 70 | #endif /* INTERNALS_TYPES_TIMEDTOKEN_HPP_ */ 71 | -------------------------------------------------------------------------------- /include/pico/Operators/Reduce.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef OPERATORS_REDUCE_HPP_ 22 | #define OPERATORS_REDUCE_HPP_ 23 | 24 | #include "UnaryOperator.hpp" 25 | 26 | namespace pico { 27 | 28 | /** 29 | * Defines a Reduce operator performing a tree reduce function. 30 | * The reduce kernel is defined by the user and can be a lambda function, a 31 | * functor or a function. 32 | * 33 | * The reduce kernel function operates on windows and/or groups if defined on 34 | * the Pipe. 35 | * 36 | * It implements a tree reduce operator where input and output value are the 37 | * same. 38 | */ 39 | #if 0 40 | template 41 | class Reduce: public UnaryOperator { 42 | public: 43 | /** 44 | * \ingroup op-api 45 | * Reduce Constructor 46 | * 47 | * Creates a new Reduce operator by defining its kernel function. 48 | */ 49 | Reduce(std::function reducef_) : 50 | reducef(reducef_) { 51 | this->set_input_degree(1); 52 | this->set_output_degree(1); 53 | this->stype(StructureType::BAG, true); 54 | this->stype(StructureType::STREAM, false); 55 | } 56 | 57 | /** 58 | * Returns the name of the operator, consisting in the name of the class. 59 | */ 60 | std::string name_short() { 61 | return "Reduce"; 62 | } 63 | 64 | protected: 65 | Reduce *clone() { 66 | return new Reduce(reducef); 67 | } 68 | 69 | const OpClass operator_class() { 70 | return OpClass::REDUCE; 71 | } 72 | 73 | private: 74 | std::function reducef; 75 | }; 76 | #endif 77 | 78 | } /* namespace pico */ 79 | 80 | #endif /* OPERATORS_REDUCE_HPP_ */ 81 | -------------------------------------------------------------------------------- /tests/reduce_by_key.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "pico/pico.hpp" 26 | 27 | #include "common/io.hpp" 28 | 29 | typedef pico::KeyValue KV; 30 | 31 | TEST_CASE("reduce by key", "reduce by key tag") { 32 | std::string input_file = "./testdata/pairs.txt"; 33 | std::string output_file = "output.txt"; 34 | 35 | /* define i/o operators from/to file */ 36 | pico::ReadFromFile reader(input_file); 37 | 38 | pico::WriteToDisk writer(output_file, 39 | [&](KV in) { return in.to_string(); }); 40 | 41 | /* compose the pipeline */ 42 | auto test_pipe = 43 | pico::Pipe() 44 | .add(reader) 45 | .add(pico::Map( 46 | [](std::string line) { return KV::from_string(line); })) 47 | .add(pico::ReduceByKey([](int v1, int v2) { return v1 + v2; })) 48 | .add(writer); 49 | 50 | test_pipe.run(); 51 | 52 | /* parse output into char-int pairs */ 53 | std::unordered_map observed; 54 | auto output_pairs_str = read_lines(output_file); 55 | for (auto pair : output_pairs_str) { 56 | auto kv = KV::from_string(pair); 57 | observed[kv.Key()] = kv.Value(); 58 | } 59 | 60 | /* compute expected output */ 61 | std::unordered_map expected; 62 | auto input_pairs_str = read_lines(input_file); 63 | for (auto pair : input_pairs_str) { 64 | auto kv = KV::from_string(pair); 65 | expected[kv.Key()] += kv.Value(); 66 | } 67 | 68 | REQUIRE(expected == observed); 69 | } 70 | -------------------------------------------------------------------------------- /tests/flatmap.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "pico/pico.hpp" 27 | 28 | #include "common/io.hpp" 29 | 30 | static auto duplicate = [](std::string& in, 31 | pico::FlatMapCollector& collector) { 32 | collector.add(in); 33 | collector.add(in); 34 | }; 35 | 36 | std::vector seq_duplicate(std::vector& vec) { 37 | std::vector duplicated; 38 | for (auto el : vec) { 39 | duplicated.push_back(el); 40 | duplicated.push_back(el); 41 | } 42 | return duplicated; 43 | } 44 | 45 | TEST_CASE("flatmap", "flatmap tag") { 46 | std::string input_file = "./testdata/lines.txt"; 47 | std::string output_file = "output.txt"; 48 | 49 | /* define i/o operators from/to file */ 50 | pico::ReadFromFile reader(input_file); 51 | pico::WriteToDisk writer(output_file); 52 | 53 | /* compose the pipeline */ 54 | auto io_file_pipe = 55 | pico::Pipe() // the empty pipeline 56 | .add(reader) 57 | .add(pico::FlatMap(duplicate)) 58 | .add(writer); 59 | 60 | io_file_pipe.run(); 61 | 62 | auto input_lines = read_lines(input_file); 63 | auto duplicated_lines = seq_duplicate(input_lines); 64 | auto output_lines = read_lines(output_file); 65 | 66 | /* forget the order and compare */ 67 | std::sort(duplicated_lines.begin(), duplicated_lines.end()); 68 | std::sort(output_lines.begin(), output_lines.end()); 69 | 70 | REQUIRE(duplicated_lines == output_lines); 71 | } 72 | -------------------------------------------------------------------------------- /include/pico/WindowPolicy.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_WINDOWPOLICY_HPP_ 22 | #define INTERNALS_WINDOWPOLICY_HPP_ 23 | 24 | namespace pico { 25 | 26 | class WindowPolicy { 27 | public: 28 | WindowPolicy() : w_size(1), w_slide(1) {} 29 | 30 | WindowPolicy(size_t w_size_, size_t w_slide_) 31 | : w_size(w_size_), w_slide(w_slide_) {} 32 | 33 | WindowPolicy(const WindowPolicy &w) : w_size(w.w_size), w_slide(w.w_slide) {} 34 | 35 | WindowPolicy &operator=(const WindowPolicy &w) { 36 | w_slide = w.w_slide; 37 | w_size = w.w_size; 38 | return *this; 39 | } 40 | 41 | size_t slide_factor() { return w_slide; } 42 | 43 | size_t win_size() { return w_size; } 44 | 45 | virtual WindowPolicy *clone() = 0; 46 | 47 | virtual ~WindowPolicy() {} 48 | 49 | protected: 50 | size_t w_size; 51 | size_t w_slide; 52 | }; 53 | 54 | template 55 | class BatchWindow : public WindowPolicy { 56 | public: 57 | BatchWindow() : WindowPolicy() {} 58 | 59 | BatchWindow(const BatchWindow ©) : WindowPolicy(copy) {} 60 | 61 | BatchWindow(size_t w_size_) : WindowPolicy(w_size_, w_size_) {} 62 | 63 | BatchWindow *clone() { return new BatchWindow(*this); } 64 | }; 65 | 66 | template 67 | class ByKeyWindow : public WindowPolicy { 68 | public: 69 | ByKeyWindow() : WindowPolicy() {} 70 | 71 | ByKeyWindow(const ByKeyWindow ©) : WindowPolicy(copy) {} 72 | 73 | ByKeyWindow(size_t w_size_) : WindowPolicy(w_size_, w_size_) {} 74 | 75 | ByKeyWindow *clone() { return new ByKeyWindow(*this); } 76 | }; 77 | } /* namespace pico */ 78 | 79 | #endif /* INTERNALS_WINDOWPOLICY_HPP_ */ 80 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/InOut/WriteToStdOutFFNode.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_INOUT_WRITETOSTDOUTFFNODE_HPP_ 22 | #define INTERNALS_FFOPERATORS_INOUT_WRITETOSTDOUTFFNODE_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/Microbatch.hpp" 27 | #include "pico/Internals/TimedToken.hpp" 28 | #include "pico/Internals/Token.hpp" 29 | #include "pico/Internals/utils.hpp" 30 | 31 | /* 32 | * TODO only works with non-decorating token 33 | */ 34 | 35 | template 36 | class WriteToStdOutFFNode : public base_filter { 37 | public: 38 | WriteToStdOutFFNode(std::function kernel_) 39 | : wkernel(kernel_) {} 40 | 41 | /* sink node */ 42 | bool propagate_cstream_sync() { return false; } 43 | 44 | void kernel(pico::base_microbatch* in_mb) { 45 | auto in_microbatch = reinterpret_cast*>(in_mb); 46 | for (In& tt : *in_microbatch) std::cout << wkernel(tt) << std::endl; 47 | DELETE(in_microbatch); 48 | } 49 | 50 | private: 51 | std::function wkernel; 52 | }; 53 | 54 | template 55 | class WriteToStdOutFFNode_ostream : public base_filter { 56 | public: 57 | /* sink node */ 58 | bool propagate_cstream_sync() { return false; } 59 | 60 | void kernel(pico::base_microbatch* in_mb) { 61 | auto in_microbatch = reinterpret_cast*>(in_mb); 62 | for (In& tt : *in_microbatch) std::cout << tt << std::endl; 63 | DELETE(in_microbatch); 64 | } 65 | }; 66 | 67 | #endif /* INTERNALS_FFOPERATORS_INOUT_WRITETOSTDOUTFFNODE_HPP_ */ 68 | -------------------------------------------------------------------------------- /tests/read_from_stdin.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "pico/pico.hpp" 27 | 28 | #include "common/io.hpp" 29 | 30 | TEST_CASE("read from stdin and write", "read from stdin and write tag") { 31 | std::string input_file = "./testdata/lines.txt"; 32 | std::string output_file = "output.txt"; 33 | 34 | /* redirect input file to stdin */ 35 | auto cinbuf = std::cin.rdbuf(); // save old buf 36 | std::ifstream in(input_file); 37 | std::cin.rdbuf(in.rdbuf()); // redirect 38 | 39 | /* redirect stdout to output file */ 40 | auto coutbuf = std::cout.rdbuf(); // save old buf 41 | std::ofstream out(output_file); 42 | std::cout.rdbuf(out.rdbuf()); // redirect 43 | 44 | std::cin.tie(0); // untie cin and cout 45 | 46 | /* define i/o operators from/to file */ 47 | pico::ReadFromStdIn reader('\n'); 48 | pico::WriteToStdOut writer; 49 | 50 | /* compose the pipeline */ 51 | auto io_file_pipe = pico::Pipe() // the empty pipeline 52 | .add(reader) 53 | .add(writer); 54 | 55 | io_file_pipe.run(); 56 | 57 | /* undo redirection */ 58 | std::cout.rdbuf(coutbuf); 59 | std::cin.rdbuf(cinbuf); 60 | out.close(); 61 | in.close(); 62 | std::cin.tie(&std::cout); // tie again cin and cout 63 | 64 | /* forget the order and compare */ 65 | auto input_lines = read_lines(input_file); 66 | auto output_lines = read_lines(output_file); 67 | std::sort(input_lines.begin(), input_lines.end()); 68 | std::sort(output_lines.begin(), output_lines.end()); 69 | 70 | REQUIRE(input_lines == output_lines); 71 | } 72 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/OutputOperator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * OutputOperator.hpp 22 | * 23 | * Created on: Aug 9, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef ACTORS_OUTPUTACTORNODE_HPP_ 28 | #define ACTORS_OUTPUTACTORNODE_HPP_ 29 | 30 | #include "pico/Operators/UnaryOperator.hpp" 31 | 32 | namespace pico { 33 | 34 | /** 35 | * Defines an operator performing the policy for managing output (i.e. write on 36 | * file or standard output). It automatically performs a type sanity check on 37 | * input type. 38 | * 39 | * The output kernel is defined by the user and can be a lambda function, a 40 | * functor or a function. 41 | * 42 | * 43 | * Its behaviour w.r.t. composed Pipes (with both append and pair) has to be 44 | * defined. 45 | * 46 | */ 47 | template 48 | class OutputOperator : public UnaryOperator { 49 | public: 50 | /** 51 | * Constructor. 52 | * Creates a new Collector by defining its kernel function 53 | * outf: In -> void 54 | */ 55 | OutputOperator(StructureType st_) { 56 | this->set_input_degree(1); 57 | this->set_output_degree(0); 58 | this->stype(st_, true); 59 | } 60 | 61 | /** 62 | * Copy constructor 63 | */ 64 | OutputOperator(const OutputOperator ©) : UnaryOperator(copy) {} 65 | 66 | /** 67 | * Returns the name of the operator, consisting in the name of the class. 68 | */ 69 | std::string name_short() { return "Collector"; } 70 | 71 | virtual ~OutputOperator() {} 72 | 73 | const OpClass operator_class() { return OpClass::OUTPUT; } 74 | 75 | protected: 76 | }; 77 | 78 | } /* namespace pico */ 79 | 80 | #endif /* ACTORS_OUTPUTACTORNODE_HPP_ */ 81 | -------------------------------------------------------------------------------- /tests/testdata/generate_lines.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "../common/utils.hpp" 28 | 29 | #define MIN_WPL 1 // min words per line 30 | #define MAX_WPL 128 // max words per line 31 | 32 | static std::vector dictionary; 33 | static std::default_random_engine rng; 34 | 35 | /* 36 | * return true if the emitted tweet does mention a unique stock name 37 | */ 38 | static inline void generate_line( // 39 | std::uniform_int_distribution& ds) { 40 | /* pick a random line length */ 41 | static std::uniform_int_distribution ll(MIN_WPL, MAX_WPL); 42 | unsigned line_len = ll(rng); 43 | 44 | for (; line_len > 1; --line_len) { 45 | std::cout << dictionary[ds(rng)] << " "; 46 | } 47 | std::cout << dictionary[ds(rng)] << std::endl; 48 | } 49 | 50 | int main(int argc, char** argv) { 51 | /* parse command line */ 52 | if (argc < 3) { 53 | std::cerr << "Usage: " << argv[0]; 54 | std::cerr << " \n"; 55 | return -1; 56 | } 57 | std::string dictionary_fname = argv[1]; 58 | unsigned long long n_lines = get_size(argv[2]); 59 | 60 | /* bring dictionary to memory */ 61 | std::ifstream dictionary_file(dictionary_fname); 62 | std::string word; 63 | while (dictionary_file.good()) { 64 | dictionary_file >> word; 65 | dictionary.push_back(word); 66 | } 67 | 68 | /* generate and emit random lines */ 69 | std::uniform_int_distribution ds(0, dictionary.size() - 1); 70 | for (unsigned line_i = 0; line_i < n_lines; ++line_i) { 71 | generate_line(ds); 72 | print_progress((float)(line_i + 1) / n_lines); 73 | } 74 | 75 | std::cerr << "\ndone\n"; 76 | 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/InOut/ReadFromStdInFFNode.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_INOUT_READFROMSTDINFFNODE_HPP_ 22 | #define INTERNALS_FFOPERATORS_INOUT_READFROMSTDINFFNODE_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include "pico/Internals/Microbatch.hpp" 31 | #include "pico/Internals/utils.hpp" 32 | 33 | #include "pico/ff_implementation/ff_config.hpp" 34 | 35 | #define CHUNK_SIZE 512 36 | 37 | /* 38 | * TODO only works with non-decorating token 39 | */ 40 | 41 | template 42 | class ReadFromStdInFFNode : public base_filter { 43 | public: 44 | ReadFromStdInFFNode(char delimiter_) : delimiter(delimiter_) {} 45 | 46 | void kernel(pico::base_microbatch *) { assert(false); } 47 | 48 | void begin_callback() { 49 | /* get a fresh tag */ 50 | tag = pico::base_microbatch::fresh_tag(); 51 | begin_cstream(tag); 52 | auto mb = NEW(tag, pico::global_params.MICROBATCH_SIZE); 53 | std::string str; 54 | 55 | while (std::getline(std::cin, str, delimiter)) { 56 | new (mb->allocate()) std::string(str); 57 | mb->commit(); 58 | if (mb->full()) { 59 | send_mb(mb); 60 | mb = NEW(tag, pico::global_params.MICROBATCH_SIZE); 61 | } 62 | } 63 | 64 | if (!mb->empty()) 65 | send_mb(mb); 66 | else 67 | DELETE(mb); 68 | 69 | end_cstream(tag); 70 | } 71 | 72 | private: 73 | typedef pico::Microbatch mb_t; 74 | char delimiter; 75 | pico::base_microbatch::tag_t tag = 0; // a tag for the generated collection 76 | 77 | void error(const char *msg) { 78 | perror(msg); 79 | exit(0); 80 | } 81 | }; 82 | 83 | #endif /* INTERNALS_FFOPERATORS_INOUT_READFROMSTDINFFNODE_HPP_ */ 84 | -------------------------------------------------------------------------------- /examples/stock-market/testdata/nasdaq_europe.txt: -------------------------------------------------------------------------------- 1 | ABB 2 | ACIU 3 | ACN 4 | ADAP 5 | AGRO 6 | AAAP 7 | ANW 8 | AEB 9 | AED 10 | AEG 11 | AEH 12 | AEK 13 | AER 14 | AFMD 15 | AIXG 16 | ALKS 17 | AWH 18 | ASPS 19 | AMRN 20 | DOX 21 | AMFW 22 | BUD 23 | MT 24 | ASND 25 | ASML 26 | AZN 27 | ABY 28 | TEAM 29 | EARS 30 | ALV 31 | AV 32 | BBVA 33 | SAN 34 | SAN^A 35 | SAN^B 36 | SAN^C 37 | SAN^I 38 | TAPR 39 | BCS 40 | BCS^A 41 | BCS^D 42 | BBL 43 | BP 44 | BTI 45 | BT 46 | CPLP 47 | CLLS 48 | CYAD 49 | CGG 50 | CBI 51 | CMPR 52 | CNV 53 | CSTM 54 | CLB 55 | CMRE 56 | CMRE^B 57 | CMRE^C 58 | CMRE^D 59 | GLDI 60 | CS 61 | SLVO 62 | CRH 63 | CRSP 64 | CRTO 65 | CB 66 | DAC 67 | DBVT 68 | DLPH 69 | DB 70 | DTK 71 | DXB 72 | DHT 73 | DEO 74 | DCIX 75 | DSX 76 | DSX^B 77 | DSXN 78 | DRYS 79 | DLNG 80 | DLNG^A 81 | ETN 82 | EDAP 83 | ENDP 84 | E 85 | ESV 86 | ERIC 87 | EURN 88 | ESEA 89 | GSM 90 | FCAM 91 | FCAU 92 | FLML 93 | FLY 94 | FWP 95 | FMS 96 | GLPG 97 | GRMN 98 | GSK 99 | GBLI 100 | GBLIZ 101 | GSL 102 | GSL^B 103 | GLBS 104 | GRFS 105 | GWPH 106 | HZNP 107 | HSBC 108 | HSBC^A 109 | HSEA 110 | HSEB 111 | ICLR 112 | ING 113 | INZ 114 | ISF 115 | ISG 116 | ISP 117 | IR 118 | INNL 119 | I 120 | IHG 121 | INXN 122 | DTYS 123 | DTYL 124 | DTUS 125 | DTUL 126 | DFVS 127 | DFVL 128 | FLAT 129 | DLBS 130 | DLBL 131 | STPP 132 | JHX 133 | JAZZ 134 | JCI 135 | KNOP 136 | PHG 137 | LBTYA 138 | LBTYB 139 | LBTYK 140 | LILA 141 | LILAK 142 | LIQT 143 | LIVN 144 | LYG 145 | EVAR 146 | LXFR 147 | LUX 148 | LYB 149 | MX 150 | MNK 151 | MAMS 152 | MANU 153 | MHG 154 | MTLS 155 | MRUS 156 | MTD 157 | MTP 158 | MIME 159 | MYL 160 | NBRV 161 | NGG 162 | NW^C 163 | NTZ 164 | NNA 165 | NMM 166 | NEON 167 | NVET 168 | NLSN 169 | NE 170 | NOK 171 | NVS 172 | NVO 173 | NVCR 174 | NXPI 175 | OASM 176 | OMAA 177 | OMAM 178 | OHGI 179 | ORAN 180 | OEC 181 | OXFD 182 | PACD 183 | PTHN 184 | PSO 185 | PNR 186 | PIRS 187 | LENS 188 | PRQR 189 | PUK 190 | PUK^ 191 | PUK^A 192 | PXS 193 | QGEN 194 | QTNT 195 | GOLD 196 | RENX 197 | RELX 198 | RBS 199 | RBS^F 200 | RBS^H 201 | RBS^L 202 | RBS^S 203 | RYAAY 204 | GCVRZ 205 | SNY 206 | SAP 207 | SLB 208 | SDLP 209 | STX 210 | SHIP 211 | ST 212 | SQNS 213 | SHPG 214 | SNN 215 | SBLK 216 | SBLKL 217 | STO 218 | GASS 219 | STLR 220 | STLRU 221 | STLRW 222 | STM 223 | SMMT 224 | SYT 225 | TLND 226 | TEL 227 | TI 228 | TEF 229 | TS 230 | TX 231 | TOPS 232 | TOT 233 | RIG 234 | RIGP 235 | TRIB 236 | TNP 237 | TNP^B 238 | TNP^C 239 | TNP^D 240 | UBS 241 | UN 242 | UL 243 | QURE 244 | DGLD 245 | DSLV 246 | UGLD 247 | USLV 248 | TVIZ 249 | TVIX 250 | ZIV 251 | XIV 252 | VIIZ 253 | VIIX 254 | VIP 255 | VOD 256 | VTTI 257 | WFT 258 | WLTW 259 | WPPGY 260 | WMGI 261 | WMGIZ 262 | XL 263 | YNDX 264 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/InputOperator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * InputOperator.hpp 22 | * 23 | * Created on: Aug 2, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef ACTORS_INPUTACTORNODE_HPP_ 28 | #define ACTORS_INPUTACTORNODE_HPP_ 29 | 30 | #include "pico/Operators/UnaryOperator.hpp" 31 | 32 | namespace pico { 33 | 34 | /** 35 | * Defines an operator performing the policy for generating input (i.e. read 36 | *from file). 37 | * 38 | * The input generation kernel is defined by the user and can be a lambda 39 | *function, a functor or a function. 40 | * 41 | * Furthermore, the user specifies the structure of the data type the Emitter 42 | *generates. It can be: 43 | * - Bag (spec: unordered, bounded) 44 | * - List (spec: ordered, bounded) 45 | * - Stream (spec: ordered, unbounded) 46 | * - Unbounded Bag (spec: unordered, unbounded) 47 | * 48 | * The operator is global and unique for the Pipe it refers to. 49 | */ 50 | template 51 | class InputOperator : public UnaryOperator { 52 | public: 53 | /** 54 | * Constructor. 55 | * Creates a new Emitter by defining its kernel function 56 | * genf: void -> Out 57 | * operating on a specified datatype. 58 | */ 59 | InputOperator(StructureType st_) { 60 | this->set_input_degree(0); 61 | this->set_output_degree(1); 62 | this->stype(st_, true); 63 | } 64 | 65 | /** 66 | * Copy constructor 67 | */ 68 | InputOperator(const InputOperator ©) : UnaryOperator(copy) {} 69 | 70 | /** 71 | * Returns the name of the operator, consisting in the name of the class. 72 | */ 73 | std::string name_short() { return "Emitter"; } 74 | 75 | virtual ~InputOperator() {} 76 | 77 | // protected: 78 | const OpClass operator_class() { return OpClass::INPUT; } 79 | }; 80 | 81 | } /* namespace pico */ 82 | 83 | #endif /* ACTORS_INPUTACTORNODE_HPP_ */ 84 | -------------------------------------------------------------------------------- /examples/stock-market/stock_pricing.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | // This code implements a pipeline for batch processing of stocks. 21 | // It first computes a price for each option from a text file, 22 | // then it extracts the maximum price for each stock name. 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "pico/pico.hpp" 31 | 32 | #include "black_scholes.hpp" 33 | #include "common.hpp" 34 | #include "defs.h" 35 | 36 | int main(int argc, char** argv) { 37 | // parse command line 38 | if (argc < 3) { 39 | std::cerr << "Usage: " << argv[0] << " \n"; 40 | return -1; 41 | } 42 | std::string in_fname(argv[1]), out_fname(argv[2]); 43 | 44 | /* 45 | * define a batch pipeline that: 46 | * 1. read options from file 47 | * 2. computes prices by means of the blackScholes pipeline 48 | * 3. extracts the maximum price for each stock name 49 | * 4. write prices to file 50 | */ 51 | pico::Map blackScholes([](const std::string& in) { 52 | OptionData opt; 53 | char otype, name[128]; 54 | parse_opt(opt, otype, name, in); 55 | opt.OptionType = (otype == 'P'); 56 | return StockAndPrice(std::string(name), black_scholes(opt)); 57 | }); 58 | 59 | auto stockPricing = pico::Pipe() // 60 | .add(pico::ReadFromFile(in_fname)) // 61 | .add(blackScholes) // 62 | .add(SPReducer()) // 63 | .add(pico::WriteToDisk(out_fname)); 64 | 65 | /* print the semantic graph and generate dot file */ 66 | stockPricing.print_semantics(); 67 | stockPricing.to_dotfile("stock_pricing.dot"); 68 | 69 | /* execute the pipeline */ 70 | stockPricing.run(); 71 | 72 | /* print execution time */ 73 | std::cout << "done in " << stockPricing.pipe_time() << " ms\n"; 74 | 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /examples/word-count/seq_wc.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | /** 22 | * This code implements a word-count (i.e., the Big Data "hello world!") 23 | * on top of the PiCo API. 24 | * 25 | * We use a mix of static functions and lambdas in order to show the support 26 | * of various user code styles provided by PiCo operators. 27 | */ 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include "pico/KeyValue.hpp" 38 | 39 | int main(int argc, char** argv) { 40 | // parse command line 41 | if (argc < 2) { 42 | std::cerr << "Usage: " << argv[0] << " \n"; 43 | return -1; 44 | } 45 | std::string filename = argv[1]; 46 | std::string outputfilename = argv[2]; 47 | 48 | /* prepare the output word-count map */ 49 | std::unordered_map word_cnt; 50 | 51 | /* start measurement */ 52 | auto t0 = std::chrono::high_resolution_clock::now(); 53 | 54 | /* read the input file line by line */ 55 | std::ifstream infile(filename); 56 | assert(infile.is_open()); 57 | std::string line; 58 | while (getline(infile, line)) { 59 | std::istringstream f(line); 60 | std::string s; 61 | 62 | /* tokenize the line and increment each word counter */ 63 | while (std::getline(f, s, ' ')) word_cnt[s]++; 64 | } 65 | 66 | /* write output */ 67 | std::ofstream outfile(outputfilename); 68 | assert(outfile.is_open()); 69 | for (auto it = word_cnt.begin(); it != word_cnt.end(); ++it) { 70 | assert(it->second != 0); 71 | pico::KeyValue kv(it->first, it->second); 72 | outfile << kv.to_string() << std::endl; 73 | } 74 | 75 | /* stop measurement */ 76 | auto t1 = std::chrono::high_resolution_clock::now(); 77 | auto d = std::chrono::duration_cast(t1 - t0); 78 | 79 | /* print the execution time */ 80 | std::cout << "done in " << d.count() << " s\n"; 81 | 82 | return 0; 83 | } 84 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/InOut/WriteToDiskFFNode.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_INOUT_WRITETODISKFFNODE_HPP_ 22 | #define INTERNALS_FFOPERATORS_INOUT_WRITETODISKFFNODE_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/Microbatch.hpp" 27 | #include "pico/Internals/Token.hpp" 28 | #include "pico/Internals/utils.hpp" 29 | 30 | #include "pico/ff_implementation/SupportFFNodes/base_nodes.hpp" 31 | 32 | /* 33 | * TODO only works with non-decorating token 34 | */ 35 | 36 | template 37 | class WriteToDiskFFNode : public base_filter { 38 | public: 39 | WriteToDiskFFNode(std::string fname, std::function kernel_) 40 | : wkernel(kernel_), outfile(fname) { 41 | if (!outfile.is_open()) { 42 | std::cerr << "Unable to open output file\n"; 43 | assert(false); 44 | } 45 | } 46 | 47 | /* sink node */ 48 | bool propagate_cstream_sync() { return false; } 49 | 50 | void kernel(pico::base_microbatch* in_mb) { 51 | auto mb = reinterpret_cast>*>(in_mb); 52 | for (In& in : *mb) outfile << wkernel(in) << std::endl; 53 | DELETE(mb); 54 | } 55 | 56 | private: 57 | std::function wkernel; 58 | std::ofstream outfile; 59 | }; 60 | 61 | template 62 | class WriteToDiskFFNode_ostream : public base_filter { 63 | public: 64 | WriteToDiskFFNode_ostream(std::string fname) : outfile(fname) { 65 | if (!outfile.is_open()) { 66 | std::cerr << "Unable to open output file\n"; 67 | assert(false); 68 | } 69 | } 70 | 71 | /* sink node */ 72 | bool propagate_cstream_sync() { return false; } 73 | 74 | void kernel(pico::base_microbatch* in_mb) { 75 | auto mb = reinterpret_cast>*>(in_mb); 76 | for (In& in : *mb) outfile << in << std::endl; 77 | DELETE(mb); 78 | } 79 | 80 | private: 81 | std::ofstream outfile; 82 | }; 83 | 84 | #endif /* INTERNALS_FFOPERATORS_INOUT_WRITETODISKFFNODE_HPP_ */ 85 | -------------------------------------------------------------------------------- /include/pico/Operators/FoldReduce.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef OPERATORS_FOLDREDUCE_HPP_ 22 | #define OPERATORS_FOLDREDUCE_HPP_ 23 | 24 | #include "UnaryOperator.hpp" 25 | 26 | #include "pico/Internals/Token.hpp" 27 | #include "pico/ff_implementation/OperatorsFFNodes/FoldReduceBatch.hpp" 28 | 29 | namespace pico { 30 | 31 | template 32 | class FoldReduce : public UnaryOperator { 33 | public: 34 | /** 35 | * \ingroup op-api 36 | * 37 | * FoldReduce Constructor 38 | * 39 | * Creates a new FoldReduce operator by defining its kernel function. 40 | */ 41 | FoldReduce(std::function foldf_, 42 | std::function reducef_, // 43 | unsigned par = def_par()) { 44 | foldf = foldf_; 45 | reducef = reducef_; 46 | this->set_input_degree(1); 47 | this->set_output_degree(1); 48 | this->stype(StructureType::BAG, true); 49 | this->stype(StructureType::STREAM, false); 50 | this->pardeg(par); 51 | } 52 | 53 | FoldReduce(const FoldReduce& copy) 54 | : UnaryOperator(copy), 55 | foldf(copy.foldf), 56 | reducef(copy.reducef) {} 57 | 58 | /** 59 | * Returns the name of the operator, consisting in the name of the class. 60 | */ 61 | std::string name_short() { return "FoldReduce"; } 62 | 63 | protected: 64 | FoldReduce* clone() { return new FoldReduce(*this); } 65 | 66 | const OpClass operator_class() { return OpClass::FOLDREDUCE; } 67 | 68 | ff::ff_node* node_operator(int parallelism, StructureType st) { 69 | assert(st == StructureType::BAG); 70 | return new FoldReduceBatch, 71 | Token>(parallelism, foldf, reducef); 72 | } 73 | 74 | private: 75 | std::function foldf; 76 | std::function reducef; 77 | }; 78 | 79 | } /* namespace pico */ 80 | 81 | #endif /* OPERATORS_FOLDREDUCE_HPP_ */ 82 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/ReadFromHDFS.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * ReadFromHDFS.hpp 22 | * 23 | * Created on: Feb 21, 2017 24 | * Author: misale 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_READFROMHDFS_HPP_ 28 | #define OPERATORS_INOUT_READFROMHDFS_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | //#include 34 | //"../../ff_implementation/OperatorsFFNodes/InOut/ReadFromHDFSFFNode.hpp" 35 | #include "InputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that reads data from HDFS file system and produces an 41 | * Ordered+Bounded collection (i.e. LIST). 42 | * 43 | * The operator returns a std::string to the user containing a single line read. 44 | * 45 | * The operator is global and unique for the Pipe it refers to. 46 | */ 47 | 48 | class ReadFromHDFS : public InputOperator { 49 | public: 50 | /** 51 | * \ingroup op-api 52 | * ReadFromHDFS Constructor 53 | * 54 | * Creates a new ReadFromHDFS. 55 | */ 56 | ReadFromHDFS(std::string fname_) 57 | : InputOperator(StructureType::BAG), fname(fname_) {} 58 | 59 | /** 60 | * Copy constructor. 61 | */ 62 | ReadFromHDFS(const ReadFromHDFS ©) 63 | : InputOperator(copy), fname(copy.fname) {} 64 | 65 | /** 66 | * Returns a unique name for the operator. 67 | */ 68 | std::string name() { 69 | std::string name("ReadFromHDFS"); 70 | std::ostringstream address; 71 | address << (void const *)this; 72 | return name + address.str().erase(0, 2); 73 | } 74 | 75 | /** 76 | * Returns the name of the operator, consisting in the name of the class. 77 | */ 78 | std::string name_short() { return "ReadFromHDFS\n[" + fname + "]"; } 79 | 80 | protected: 81 | ReadFromHDFS *clone() { return new ReadFromHDFS(*this); } 82 | 83 | ff::ff_node *node_operator(int parallelism, StructureType st) { 84 | assert(st == StructureType::BAG); 85 | // return new ReadFromHDFSFFNode(); 86 | assert(false); 87 | return nullptr; 88 | } 89 | 90 | private: 91 | std::string fname; 92 | }; 93 | 94 | } /* namespace pico */ 95 | 96 | #endif /* OPERATORS_INOUT_READFROMHDFS_HPP_ */ 97 | -------------------------------------------------------------------------------- /examples/stock-market/seq_stock_pricing.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | // This code implements a pipeline for batch processing of stocks. 22 | // It first computes a price for each option from a text file, 23 | // then it extracts the maximum price for each stock name. 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #include "black_scholes.hpp" 36 | #include "defs.h" 37 | 38 | int main(int argc, char** argv) { 39 | /* parse command line */ 40 | if (argc < 3) { 41 | std::cerr << "Usage: " << argv[0]; 42 | std::cerr << " \n"; 43 | return -1; 44 | } 45 | std::string in_fname = argv[1], out_fname = argv[2]; 46 | 47 | /* read options from the input file */ 48 | std::ifstream in_file(in_fname); 49 | assert(in_file.is_open()); 50 | std::chrono::seconds wt(0); 51 | std::unordered_map red; 52 | while (in_file.good()) { 53 | std::string opt_line; 54 | std::getline(in_file, opt_line, '\n'); 55 | 56 | /* 57 | * map + reduce 58 | */ 59 | auto t0 = std::chrono::high_resolution_clock::now(); 60 | 61 | std::string name; 62 | OptionData opt; 63 | char otype; 64 | std::stringstream ins(opt_line); 65 | 66 | /* read stock name */ 67 | ins >> name; 68 | 69 | /* read stock option data */ 70 | ins >> opt.s >> opt.strike >> opt.r >> opt.divq; 71 | ins >> opt.v >> opt.t >> otype >> opt.divs >> opt.DGrefval; 72 | opt.OptionType = (otype == 'P'); 73 | 74 | StockPrice res = black_scholes(opt); 75 | 76 | if (red.find(name) != red.end()) 77 | red[name] = std::max(StockAndPrice(name, res), red[name]); 78 | else 79 | red[name] = StockAndPrice(name, res); 80 | 81 | auto t1 = std::chrono::high_resolution_clock::now(); 82 | wt += std::chrono::duration_cast(t0 - t1); 83 | } 84 | 85 | /* print results */ 86 | std::ofstream out_file(out_fname); 87 | for (auto item : red) out_file << item.second.to_string() << std::endl; 88 | 89 | std::cerr << "map = " << wt.count() << " s" << std::endl; 90 | 91 | return 0; 92 | } 93 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/ReadFromFile.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * ReadFromFile.hpp 22 | * 23 | * Created on: Aug 30, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_READFROMFILE_HPP_ 28 | #define OPERATORS_INOUT_READFROMFILE_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | #include "pico/ff_implementation/OperatorsFFNodes/InOut/ReadFromFileFFNode.hpp" 34 | 35 | #include "InputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that reads data from a text file and produces an 41 | * Ordered+Bounded collection (i.e. LIST). 42 | * 43 | * The operator returns a std::string to the user containing a single line read. 44 | * 45 | * The operator is global and unique for the Pipe it refers to. 46 | */ 47 | 48 | class ReadFromFile : public InputOperator { 49 | public: 50 | /** 51 | * \ingroup op-api 52 | * 53 | * ReadFromFile Constructor 54 | * 55 | * Creates a new ReadFromFile operator, 56 | * yielding an unordered bounded collection. 57 | */ 58 | ReadFromFile(std::string fname_, unsigned par = def_par()) 59 | : InputOperator(StructureType::BAG), fname(fname_) { 60 | this->pardeg(par); 61 | } 62 | 63 | /** 64 | * Copy constructor. 65 | */ 66 | ReadFromFile(const ReadFromFile ©) 67 | : InputOperator(copy), fname(copy.fname) {} 68 | 69 | /** 70 | * Returns a unique name for the operator. 71 | */ 72 | std::string name() { 73 | std::string name("ReadFromFile"); 74 | std::ostringstream address; 75 | address << (void const *)this; 76 | return name + address.str().erase(0, 2); 77 | } 78 | 79 | /** 80 | * Returns the name of the operator, consisting in the name of the class. 81 | */ 82 | std::string name_short() { return "ReadFromFile\n[" + fname + "]"; } 83 | 84 | protected: 85 | ReadFromFile *clone() { return new ReadFromFile(*this); } 86 | 87 | ff::ff_node *node_operator(int parallelism, StructureType st) { 88 | assert(st == StructureType::BAG); 89 | return ReadFromFileFFNode(parallelism, fname); 90 | } 91 | 92 | private: 93 | std::string fname; 94 | }; 95 | 96 | } /* namespace pico */ 97 | 98 | #endif /* OPERATORS_INOUT_READFROMFILE_HPP_ */ 99 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: Google 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: true 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: All 15 | AllowShortIfStatementsOnASingleLine: true 16 | AllowShortLoopsOnASingleLine: true 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: None 36 | BreakBeforeBraces: Attach 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | BreakAfterJavaFieldAnnotations: false 40 | BreakStringLiterals: true 41 | ColumnLimit: 80 42 | CommentPragmas: '^ IWYU pragma:' 43 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 44 | ConstructorInitializerIndentWidth: 4 45 | ContinuationIndentWidth: 4 46 | Cpp11BracedListStyle: true 47 | DerivePointerAlignment: true 48 | DisableFormat: false 49 | ExperimentalAutoDetectBinPacking: false 50 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 51 | IncludeCategories: 52 | - Regex: '^<.*\.h>' 53 | Priority: 1 54 | - Regex: '^<.*' 55 | Priority: 2 56 | - Regex: '.*' 57 | Priority: 3 58 | IncludeIsMainRegex: '([-_](test|unittest))?$' 59 | IndentCaseLabels: true 60 | IndentWidth: 2 61 | IndentWrappedFunctionNames: false 62 | JavaScriptQuotes: Leave 63 | JavaScriptWrapImports: true 64 | KeepEmptyLinesAtTheStartOfBlocks: false 65 | MacroBlockBegin: '' 66 | MacroBlockEnd: '' 67 | MaxEmptyLinesToKeep: 1 68 | NamespaceIndentation: None 69 | ObjCBlockIndentWidth: 2 70 | ObjCSpaceAfterProperty: false 71 | ObjCSpaceBeforeProtocolList: false 72 | PenaltyBreakBeforeFirstCallParameter: 1 73 | PenaltyBreakComment: 300 74 | PenaltyBreakFirstLessLess: 120 75 | PenaltyBreakString: 1000 76 | PenaltyExcessCharacter: 1000000 77 | PenaltyReturnTypeOnItsOwnLine: 200 78 | PointerAlignment: Left 79 | ReflowComments: true 80 | SortIncludes: true 81 | SpaceAfterCStyleCast: false 82 | SpaceAfterTemplateKeyword: true 83 | SpaceBeforeAssignmentOperators: true 84 | SpaceBeforeParens: ControlStatements 85 | SpaceInEmptyParentheses: false 86 | SpacesBeforeTrailingComments: 2 87 | SpacesInAngles: false 88 | SpacesInContainerLiterals: true 89 | SpacesInCStyleCastParentheses: false 90 | SpacesInParentheses: false 91 | SpacesInSquareBrackets: false 92 | Standard: Auto 93 | TabWidth: 8 94 | UseTab: Never 95 | ... 96 | 97 | -------------------------------------------------------------------------------- /examples/word-count/pico_wc.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | /** 22 | * This code implements a word-count (i.e., the Big Data "hello world!") 23 | * on top of the PiCo API. 24 | * 25 | * We use a mix of static functions and lambdas in order to show the support 26 | * of various user code styles provided by PiCo operators. 27 | */ 28 | 29 | #include 30 | #include 31 | 32 | #include "pico/pico.hpp" 33 | 34 | int main(int argc, char** argv) { 35 | using KV = pico::KeyValue; 36 | 37 | // parse command line 38 | if (argc < 3) { 39 | std::cerr << "Usage: ./pico_wc \n"; 40 | return -1; 41 | } 42 | std::string in_fname(argv[1]), out_fname(argv[2]); 43 | 44 | /* define a generic word-count pipeline */ 45 | pico::FlatMap tokenizer( 46 | [](std::string& in, pico::FlatMapCollector& collector) { 47 | std::string::size_type i = 0, j; 48 | while ((j = in.find_first_of(' ', i)) != std::string::npos) { 49 | collector.add(KV(in.substr(i, j - i), 1)); 50 | i = j + 1; 51 | } 52 | if (i < in.size()) collector.add(KV(in.substr(i, in.size() - i), 1)); 53 | }); 54 | 55 | auto countWords = 56 | pico::Pipe() // the empty pipeline 57 | .add(tokenizer) // 58 | .add(pico::ReduceByKey([](int v1, int v2) { return v1 + v2; })); 59 | 60 | // countWords can now be used to build batch pipelines. 61 | // If we enrich the last combine operator with a windowing policy (i.e., 62 | // WPReduce combine operator), the pipeline can be used to build both batch 63 | // and streaming pipelines. 64 | 65 | /* define i/o operators from/to file */ 66 | pico::ReadFromFile reader(in_fname); 67 | pico::WriteToDisk writer(out_fname); 68 | 69 | /* compose the pipeline */ 70 | auto wc = pico::Pipe() // the empty pipeline 71 | .add(reader) // 72 | .to(countWords) // 73 | .add(writer); 74 | 75 | // generate the semantic dot 76 | wc.print_semantics(); 77 | wc.to_dotfile("word-count.dot"); 78 | 79 | /* execute the pipeline */ 80 | wc.run(); 81 | 82 | /* print the execution time */ 83 | std::cout << "done in " << wc.pipe_time() << " ms\n"; 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/MapBatch.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_MAPBATCH_HPP_ 22 | #define INTERNALS_FFOPERATORS_MAPBATCH_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/Microbatch.hpp" 27 | #include "pico/Internals/TimedToken.hpp" 28 | #include "pico/Internals/utils.hpp" 29 | 30 | #include "pico/ff_implementation/SupportFFNodes/emitters.hpp" 31 | #include "pico/ff_implementation/ff_config.hpp" 32 | 33 | template 35 | class MapBatch : public Farm { 36 | public: 37 | MapBatch(int par, std::function &mapf) { 38 | ff::ff_node *e; 39 | if (this->isOFarm()) 40 | e = new OrdForwardingEmitter(par); 41 | else 42 | e = new ForwardingEmitter(par); 43 | this->setEmitterF(e); 44 | this->setCollectorF(new ForwardingCollector(par)); 45 | std::vector w; 46 | for (int i = 0; i < par; ++i) w.push_back(new Worker(mapf)); 47 | this->add_workers(w); 48 | this->cleanup_all(); 49 | } 50 | 51 | private: 52 | class Worker : public base_filter { 53 | public: 54 | Worker(std::function kernel_) : mkernel(kernel_) {} 55 | 56 | void kernel(pico::base_microbatch *in_mb) { 57 | auto in_microbatch = reinterpret_cast(in_mb); 58 | auto tag = in_mb->tag(); 59 | auto out_mb = NEW(tag, pico::global_params.MICROBATCH_SIZE); 60 | // iterate over microbatch 61 | for (In &in : *in_microbatch) { 62 | /* build item and enable copy elision */ 63 | new (out_mb->allocate()) Out(mkernel(in)); 64 | out_mb->commit(); 65 | } 66 | ff_send_out(reinterpret_cast(out_mb)); 67 | DELETE(in_microbatch); 68 | } 69 | 70 | private: 71 | typedef pico::Microbatch mb_in; 72 | typedef pico::Microbatch mb_out; 73 | std::function mkernel; 74 | }; 75 | }; 76 | 77 | template 78 | using MapBatchStream = MapBatch; 79 | 80 | template 81 | using MapBatchBag = MapBatch; 82 | 83 | #endif /* INTERNALS_FFOPERATORS_MAPBATCH_HPP_ */ 84 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/PReduceCollector.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_FF_IMPLEMENTATION_SUPPORTFFNODES_PREDUCECOLLECTOR_HPP_ 22 | #define PICO_FF_IMPLEMENTATION_SUPPORTFFNODES_PREDUCECOLLECTOR_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/Microbatch.hpp" 27 | #include "pico/Internals/utils.hpp" 28 | 29 | #include "base_nodes.hpp" 30 | 31 | template 32 | class PReduceCollector : public base_sync_duplicate { 33 | typedef typename KV::keytype K; 34 | typedef typename KV::valuetype V; 35 | typedef pico::Microbatch mb_t; 36 | 37 | public: 38 | PReduceCollector(unsigned nworkers_, std::function &rk_) 39 | : base_sync_duplicate(nworkers_), rk(rk_) {} 40 | 41 | private: 42 | std::function rk; 43 | const int mb_size = pico::global_params.MICROBATCH_SIZE; 44 | 45 | struct key_state { 46 | std::unordered_map kvmap; 47 | }; 48 | std::unordered_map tag_state; 49 | 50 | void kernel(pico::base_microbatch *in) { 51 | auto in_microbatch = reinterpret_cast(in); 52 | auto tag = in->tag(); 53 | auto &s(tag_state[tag]); 54 | /* update the internal map */ 55 | for (KV &kv : *in_microbatch) { 56 | auto &k(kv.Key()); 57 | if (s.kvmap.find(k) != s.kvmap.end()) 58 | s.kvmap[k] = rk(kv.Value(), s.kvmap[k]); 59 | else 60 | s.kvmap[k] = kv.Value(); 61 | } 62 | DELETE(in_microbatch); 63 | } 64 | 65 | void cstream_end_callback(pico::base_microbatch::tag_t tag) { 66 | /* stream the internal map downstream */ 67 | auto &s(tag_state[tag]); 68 | auto out_microbatch = NEW(tag, mb_size); 69 | for (auto it = s.kvmap.begin(); it != s.kvmap.end(); ++it) { 70 | new (out_microbatch->allocate()) KV(it->first, it->second); 71 | out_microbatch->commit(); 72 | if (out_microbatch->full()) { 73 | ff_send_out(reinterpret_cast(out_microbatch)); 74 | out_microbatch = NEW(tag, mb_size); 75 | } 76 | } 77 | 78 | /* send or delete residual microbatch */ 79 | if (!out_microbatch->empty()) 80 | ff_send_out(reinterpret_cast(out_microbatch)); 81 | else 82 | DELETE(out_microbatch); 83 | } 84 | }; 85 | 86 | #endif /* PICO_FF_IMPLEMENTATION_SUPPORTFFNODES_PREDUCECOLLECTOR_HPP_ */ 87 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/SupportFFNodes/ByKeyEmitter.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_WINDOWFFNODES_BYKEYEMITTER_HPP_ 22 | #define INTERNALS_FFOPERATORS_WINDOWFFNODES_BYKEYEMITTER_HPP_ 23 | 24 | #include 25 | 26 | #include 27 | 28 | #include "../../Internals/Microbatch.hpp" 29 | #include "base_nodes.hpp" 30 | #include "farms.hpp" 31 | 32 | template 33 | class ByKeyEmitter : public base_emitter { 34 | public: 35 | ByKeyEmitter(unsigned nworkers_) 36 | : base_emitter(nworkers_), nworkers(nworkers_) {} 37 | 38 | void cstream_begin_callback(pico::base_microbatch::tag_t tag) { 39 | /* prepare a microbatch for each worker */ 40 | auto &s(tag_state[tag]); 41 | for (unsigned dst = 0; dst < nworkers; ++dst) 42 | s.worker_mb[dst] = NEW(tag, pico::global_params.MICROBATCH_SIZE); 43 | } 44 | 45 | void kernel(pico::base_microbatch *in_mb) { 46 | auto in_microbatch = reinterpret_cast(in_mb); 47 | auto tag = in_mb->tag(); 48 | auto &s(tag_state[tag]); 49 | for (auto tt : *in_microbatch) { 50 | auto dst = key_to_worker(tt.Key()); 51 | // add token to dst's microbatch 52 | new (s.worker_mb[dst]->allocate()) DataType(tt); 53 | s.worker_mb[dst]->commit(); 54 | if (s.worker_mb[dst]->full()) { 55 | send_mb_to(s.worker_mb[dst], dst); 56 | s.worker_mb[dst] = NEW(tag, pico::global_params.MICROBATCH_SIZE); 57 | } 58 | } 59 | DELETE(in_microbatch); 60 | } 61 | 62 | void cstream_end_callback(pico::base_microbatch::tag_t tag) { 63 | auto &s(tag_state[tag]); 64 | for (unsigned i = 0; i < nworkers; ++i) { 65 | if (!s.worker_mb[i]->empty()) 66 | send_mb_to(s.worker_mb[i], i); 67 | else 68 | DELETE(s.worker_mb[i]); // spurious microbatch 69 | } 70 | } 71 | 72 | private: 73 | typedef typename TokenType::datatype DataType; 74 | typedef typename DataType::keytype keytype; 75 | typedef pico::Microbatch mb_t; 76 | unsigned nworkers; 77 | 78 | struct w_state { 79 | std::unordered_map worker_mb; 80 | }; 81 | std::unordered_map tag_state; 82 | 83 | inline size_t key_to_worker(const keytype &k) { 84 | return std::hash{}(k) % nworkers; 85 | } 86 | }; 87 | 88 | #endif /* INTERNALS_FFOPERATORS_WINDOWFFNODES_BYKEYEMITTER_HPP_ */ 89 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/FMapBatch.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_FMAPBATCH_HPP_ 22 | #define INTERNALS_FFOPERATORS_FMAPBATCH_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/FlatMapCollector.hpp" 27 | #include "pico/Internals/TimedToken.hpp" 28 | #include "pico/Internals/utils.hpp" 29 | 30 | #include "pico/ff_implementation/SupportFFNodes/emitters.hpp" 31 | 32 | template 34 | class FMapBatch : public Farm { 35 | public: 36 | FMapBatch(int par, 37 | std::function &)> flatmapf) { 38 | ff::ff_node *e; 39 | if (this->isOFarm()) 40 | e = new OrdForwardingEmitter(par); 41 | else 42 | e = new ForwardingEmitter(par); 43 | auto c = new UnpackingCollector>(par); 44 | this->setEmitterF(e); 45 | this->setCollectorF(c); 46 | std::vector w; 47 | for (int i = 0; i < par; ++i) w.push_back(new Worker(flatmapf)); 48 | this->add_workers(w); 49 | this->cleanup_all(); 50 | } 51 | 52 | private: 53 | class Worker : public base_filter { 54 | typedef typename pico::TokenCollector::cnode cnode_t; 55 | 56 | public: 57 | Worker(std::function &)> kernel_) 58 | : mkernel(kernel_) {} 59 | 60 | void kernel(pico::base_microbatch *mb) { 61 | auto in_mb = reinterpret_cast *>(mb); 62 | auto tag = mb->tag(); 63 | collector.tag(tag); 64 | // iterate over microbatch 65 | for (In &tt : *in_mb) { 66 | mkernel(tt, collector); 67 | } 68 | if (collector.begin()) 69 | ff_send_out(NEW>(tag, collector.begin())); 70 | 71 | // clean up 72 | DELETE(in_mb); 73 | collector.clear(); 74 | } 75 | 76 | private: 77 | pico::TokenCollector collector; 78 | std::function &)> mkernel; 79 | }; 80 | }; 81 | 82 | template 83 | using FMapBatchStream = FMapBatch; 84 | 85 | template 86 | using FMapBatchBag = FMapBatch; 87 | 88 | #endif /* INTERNALS_FFOPERATORS_FMAPBATCH_HPP_ */ 89 | -------------------------------------------------------------------------------- /examples/stock-market/binomial_tree.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_STOCK_MARKET_BINOMIAL_TREE_HPP_ 22 | #define EXAMPLES_STOCK_MARKET_BINOMIAL_TREE_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #include "defs.h" 31 | 32 | /* Template code for the binomial tree 33 | * Instructions to build the code are here 34 | * http://www.maths.manchester.ac.uk/~pjohnson/Tutorials/node1.html 35 | */ 36 | 37 | // return the value of the binomial tree 38 | double binomialTree(double S0, // price 39 | double X, // strike price 40 | double T, // time to maturity or option expiration in years 41 | double r, // risk-free interest rate 42 | double sigma, // volatility 43 | int n) // tree paramaters (steps in tree) 44 | { 45 | // declare and initialise local variables (u,d,q) 46 | double dt, u, d, q; 47 | dt = T / n; 48 | u = exp(sigma * sqrt(dt)); 49 | d = exp(-sigma * sqrt(dt)); 50 | q = (exp(r * dt) - d) / (u - d); 51 | // create storage for the stock price tree and option price tree 52 | double stockTree[n + 1][n + 1]; 53 | // setup and initialise the stock price tree 54 | for (int i = 0; i <= n; i++) { 55 | for (int j = 0; j <= i; j++) { 56 | stockTree[i][j] = S0 * pow(u, j) * pow(d, i - j); 57 | } 58 | } 59 | double valueTree[n + 1][n + 1]; 60 | for (int j = 0; j <= n; j++) { 61 | valueTree[n][j] = payoff(stockTree[n][j], X); 62 | } 63 | for (int i = n - 1; i >= 0; i--) { 64 | for (int j = 0; j <= i; j++) { 65 | valueTree[i][j] = exp(-r * dt) * (q * valueTree[i + 1][j + 1] + 66 | (1 - q) * valueTree[i + 1][j]); 67 | } 68 | } 69 | return valueTree[0][0]; 70 | } 71 | 72 | StockPrice binomial_tree(const OptionData &opt, int steps) { 73 | return binomialTree(opt.s, opt.strike, opt.t, opt.r, opt.v, steps); 74 | } 75 | 76 | // int main() 77 | //{ 78 | // // declare and initialise Black Scholes parameters 79 | // // declare and initialise Black Scholes parameters 80 | // double S0=100.,X=100.,T=1.,r=0.06,sigma=0.2; 81 | // // declare and initialise tree paramaters (steps in tree) 82 | // int n=3; 83 | // cout << " V(S="<. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "../../common/utils.hpp" 28 | 29 | #define MIN_WPL 1 // min words per line 30 | #define MAX_WPL 128 // max words per line 31 | 32 | static std::vector dictionary; 33 | static std::default_random_engine rng; 34 | 35 | /* 36 | * return true if the emitted tweet does mention a unique stock name 37 | */ 38 | static inline void generate_line( // 39 | std::uniform_int_distribution &ds) { 40 | /* pick a random line length */ 41 | static std::uniform_int_distribution ll(MIN_WPL, MAX_WPL); 42 | unsigned line_len = ll(rng); 43 | 44 | for (; line_len > 1; --line_len) { 45 | std::cout << dictionary[ds(rng)] << " "; 46 | } 47 | std::cout << dictionary[ds(rng)] << std::endl; 48 | } 49 | 50 | /* parse a size string */ 51 | static long long unsigned get_size(char *str) { 52 | long long unsigned size; 53 | char mod[32]; 54 | 55 | switch (sscanf(str, "%llu%1[mMkK]", &size, mod)) { 56 | case 1: 57 | return (size); 58 | case 2: 59 | switch (*mod) { 60 | case 'm': 61 | case 'M': 62 | return (size << 20); 63 | case 'k': 64 | case 'K': 65 | return (size << 10); 66 | default: 67 | return (size); 68 | } 69 | break; // suppress warning 70 | default: 71 | return (-1); 72 | } 73 | } 74 | 75 | int main(int argc, char **argv) { 76 | /* parse command line */ 77 | if (argc < 3) { 78 | std::cerr << "Usage: " << argv[0]; 79 | std::cerr << " \n"; 80 | return -1; 81 | } 82 | std::string dictionary_fname = argv[1]; 83 | unsigned long long n_lines = get_size(argv[2]); 84 | 85 | /* bring dictionary to memory */ 86 | std::ifstream dictionary_file(dictionary_fname); 87 | std::string word; 88 | while (dictionary_file.good()) { 89 | dictionary_file >> word; 90 | dictionary.push_back(word); 91 | } 92 | 93 | /* generate and emit random lines */ 94 | std::uniform_int_distribution ds(0, dictionary.size() - 1); 95 | for (unsigned line_i = 0; line_i < n_lines; ++line_i) { 96 | generate_line(ds); 97 | print_progress((float)(line_i + 1) / n_lines); 98 | } 99 | 100 | std::cerr << "\ndone\n"; 101 | 102 | return 0; 103 | } 104 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/ReadFromStdIn.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * ReadFromStdIn.hpp 22 | * 23 | * Created on: Jan 29, 2018 24 | * Author: drocco 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_READFROMSTDIN_HPP_ 28 | #define OPERATORS_INOUT_READFROMSTDIN_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | #include "pico/ff_implementation/OperatorsFFNodes/InOut/ReadFromStdInFFNode.hpp" 34 | 35 | #include "InputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that reads data from a socket and produces a Stream. 41 | * 42 | * The user specifies the kernel function that operates on each item of the 43 | * stream, passed as a std::string. The delimiter is used to separate single 44 | * items of the stream. The kernel can be a lambda function, a functor or a 45 | * function. 46 | * 47 | * 48 | * The operator is global and unique for the Pipe it refers to. 49 | */ 50 | 51 | class ReadFromStdIn : public InputOperator { 52 | public: 53 | /** 54 | * \ingroup op-api 55 | * ReadFromSocket Constructor 56 | * 57 | * Creates a new ReadFromSocket operator by defining its kernel function, 58 | * operating on each token of the stream, delimited by the delimiter value. 59 | */ 60 | ReadFromStdIn(char delimiter_) 61 | : InputOperator(StructureType::STREAM) { 62 | delimiter = delimiter_; 63 | } 64 | 65 | /** 66 | * Copy constructor. 67 | */ 68 | ReadFromStdIn(const ReadFromStdIn ©) : InputOperator(copy) { 69 | delimiter = copy.delimiter; 70 | } 71 | 72 | /** 73 | * Returns a unique name for the operator. 74 | */ 75 | std::string name() { 76 | std::string name("ReadFromStdIn"); 77 | std::ostringstream address; 78 | address << (void const *)this; 79 | return name + address.str().erase(0, 2); 80 | } 81 | 82 | /** 83 | * Returns the name of the operator, consisting in the name of the class. 84 | */ 85 | std::string name_short() { return "ReadFromStdIn"; } 86 | 87 | protected: 88 | ReadFromStdIn *clone() { return new ReadFromStdIn(*this); } 89 | 90 | const OpClass operator_class() { return OpClass::INPUT; } 91 | 92 | ff::ff_node *node_operator(int parallelism, StructureType st) { 93 | assert(st == StructureType::STREAM); 94 | return new ReadFromStdInFFNode>(delimiter); 95 | } 96 | 97 | private: 98 | char delimiter; 99 | }; 100 | 101 | } /* namespace pico */ 102 | 103 | #endif /* OPERATORS_INOUT_READFROMSTDIN_HPP_ */ 104 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/ReadFromSocket.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * ReadFromSocket.hpp 22 | * 23 | * Created on: Dec 13, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_READFROMSOCKET_HPP_ 28 | #define OPERATORS_INOUT_READFROMSOCKET_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | #include "pico/ff_implementation/OperatorsFFNodes/InOut/ReadFromSocketFFNode.hpp" 34 | 35 | #include "InputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that reads a data stream from a socket, 41 | * yielding an ordered unbounded collection. 42 | * 43 | * The user specifies a delimiter to identify stream items. 44 | */ 45 | 46 | class ReadFromSocket : public InputOperator { 47 | public: 48 | /** 49 | * \ingroup op-api 50 | * ReadFromSocket Constructor 51 | * 52 | * Creates a new ReadFromSocket operator by defining its kernel function, 53 | * operating on each token of the stream, delimited by the delimiter value. 54 | */ 55 | ReadFromSocket(std::string server_, int port_, char delimiter_) 56 | : InputOperator(StructureType::STREAM) { 57 | server_name = server_; 58 | port = port_; 59 | delimiter = delimiter_; 60 | } 61 | 62 | /** 63 | * Copy constructor. 64 | */ 65 | ReadFromSocket(const ReadFromSocket ©) 66 | : InputOperator(copy) { 67 | server_name = copy.server_name; 68 | port = copy.port; 69 | delimiter = copy.delimiter; 70 | } 71 | 72 | /** 73 | * Returns a unique name for the operator. 74 | */ 75 | std::string name() { 76 | std::string name("ReadFromSocket"); 77 | std::ostringstream address; 78 | address << (void const *)this; 79 | return name + address.str().erase(0, 2); 80 | } 81 | 82 | /** 83 | * Returns the name of the operator, consisting in the name of the class. 84 | */ 85 | std::string name_short() { return "ReadFromSocket\n[" + server_name + "]"; } 86 | 87 | protected: 88 | ReadFromSocket *clone() { return new ReadFromSocket(*this); } 89 | 90 | const OpClass operator_class() { return OpClass::INPUT; } 91 | 92 | ff::ff_node *node_operator(int parallelism, StructureType st) { 93 | assert(st == StructureType::STREAM); 94 | return new ReadFromSocketFFNode(server_name, port, delimiter); 95 | } 96 | 97 | private: 98 | std::string server_name; 99 | int port; 100 | char delimiter; 101 | }; 102 | 103 | } /* namespace pico */ 104 | 105 | #endif /* OPERATORS_INOUT_READFROMSOCKET_HPP_ */ 106 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.8) 2 | 3 | if(${CMAKE_VERSION} VERSION_LESS 3.13) 4 | cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) 5 | else() 6 | cmake_policy(VERSION 3.13) 7 | endif() 8 | 9 | if (NOT DEFINED PICO_VERSION_MAJOR) 10 | set(PICO_VERSION_MAJOR 0) 11 | endif() 12 | if (NOT DEFINED PICO_VERSION_MINOR) 13 | set(PICO_VERSION_MINOR 0) 14 | endif() 15 | if (NOT DEFINED PICO_VERSION_PATCH) 16 | set(PICO_VERSION_PATCH 1) 17 | endif() 18 | 19 | 20 | # Add path for custom CMake modules 21 | set(CMAKE_MODULE_PATH 22 | ${CMAKE_MODULE_PATH} 23 | "${CMAKE_CURRENT_SOURCE_DIR}/cmake" 24 | ) 25 | 26 | include(GenerateVersionFromGit) 27 | add_version_from_git(GIT_VERSION) 28 | 29 | if (NOT PACKAGE_VERSION) 30 | set(PACKAGE_VERSION 31 | "${PICO_VERSION_MAJOR}.${PICO_VERSION_MINOR}.${PICO_VERSION_PATCH} (${GIT_VERSION})") 32 | endif() 33 | 34 | if (CMAKE_CXX_COMPILER STREQUAL "") 35 | set(CMAKE_CXX_COMPILER g++) 36 | endif() 37 | 38 | project(PICO VERSION ${PICO_VERSION_MAJOR}.${PICO_VERSION_MINOR}.${PICO_VERSION_PATCH} 39 | LANGUAGES CXX) 40 | 41 | if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) 42 | message(STATUS "No build type selected, default to Release") 43 | set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type (default Release)" FORCE) 44 | endif() 45 | 46 | 47 | # No in-tree build allowed. 48 | if (CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR) 49 | message(FATAL_ERROR 50 | "In-source build are not allowed. 51 | Please create a directory directory and run cmake from there, passing the path 52 | to this source directory as the last argumente. 53 | This process created the file `CMakeCache.txt' and the directory `CMakeFiles'. 54 | Please delete them.") 55 | endif() 56 | 57 | string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) 58 | 59 | if (CMAKE_BUILD_TYPE AND 60 | NOT uppercase_CMAKE_BUILD_TYPE MATCHES "^(DEBUG|RELEASE|RELWITHDEBINFO|MINSIZEREL)$") 61 | message(FATAL_ERROR "Invalid value for CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") 62 | endif() 63 | 64 | # Build options: 65 | option(PICO_ENABLE_CPPLINT "Enable the linting of source code" ON) 66 | option(PICO_ENABLE_DOXYGEN "Use doxygen to generate the shad API documentation" OFF) 67 | option(PICO_ENABLE_UNIT_TEST "Enable the compilation of Unit Tests" ON) 68 | 69 | set( 70 | PICO_RUNTIME_SYSTEM "FF" CACHE STRING 71 | "Runtime system to be used as backend (Default=FF, Supported=FF)") 72 | 73 | include(config) 74 | 75 | include_directories(./include) 76 | 77 | if (PICO_ENABLE_UNIT_TEST) 78 | # TODO Catch/CTest integration 79 | # Prepare "Catch" library for other executables 80 | include(CTest) 81 | enable_testing() 82 | set(CATCH_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests/include) 83 | add_library(Catch INTERFACE) 84 | target_include_directories(Catch INTERFACE ${YOUR_CATCH_INCLUDE_DIR}) 85 | add_subdirectory(tests) 86 | endif() 87 | if (PICO_ENABLE_DOXYGEN) 88 | add_subdirectory(Doxygen) 89 | endif() 90 | add_subdirectory(examples) 91 | 92 | # Adding clang-format target if executable is found 93 | if(CLANG_FORMAT_EXE) 94 | # Additional targets to perform clang-format/clang-tidy 95 | # Get all project files 96 | file(GLOB_RECURSE 97 | ALL_CXX_SOURCE_FILES 98 | *.[chi]pp *.[chi]xx *.cc *.hh *.ii *.[CHI] *.[ch] 99 | ) 100 | 101 | add_custom_target( 102 | clang-format 103 | COMMAND ${CLANG_FORMAT_EXE} 104 | -i 105 | ${ALL_CXX_SOURCE_FILES} 106 | ) 107 | endif() 108 | -------------------------------------------------------------------------------- /examples/stock-market/stock_pricing_stream.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | // This code implements a pipeline for batch processing of stocks. 22 | // It first computes a price for each option from a text file, 23 | // then it extracts the maximum price for each stock name. 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | #include "pico/pico.hpp" 32 | 33 | #include "binomial_tree.hpp" 34 | #include "black_scholes.hpp" 35 | #include "common.hpp" 36 | #include "defs.h" 37 | #include "explicit_finite_difference.hpp" 38 | 39 | int main(int argc, char** argv) { 40 | // parse command line 41 | if (argc < 3) { 42 | std::cerr << "Usage: " << argv[0] << " \n"; 43 | return -1; 44 | } 45 | std::string server(argv[1]); 46 | int port = atoi(argv[2]); 47 | 48 | /* 49 | * define a batch pipeline that: 50 | * 1. read options from file 51 | * 2. computes prices by means of the blackScholes pipeline 52 | * 3. extracts the maximum price for each stock name 53 | * 4. write prices to file 54 | */ 55 | pico::Map blackScholes([](const std::string& in) { 56 | OptionData opt; 57 | char otype, name[128]; 58 | parse_opt(opt, otype, name, in); 59 | opt.OptionType = (otype == 'P'); 60 | int iMax = 4, jMax = 4, steps = 10; 61 | int const size = 3; 62 | StockPrice res[size]; 63 | res[0] = black_scholes(opt); 64 | res[1] = binomial_tree(opt, steps); 65 | res[2] = explicit_finite_difference(opt, iMax, jMax); 66 | StockPrice mean = 0; 67 | for (int i = 0; i < size; ++i) { 68 | mean += res[i]; 69 | } 70 | mean /= size; 71 | StockPrice variance = 0; 72 | for (int i = 0; i < size; ++i) { 73 | variance += (res[i] - mean) * (res[i] - mean); 74 | } 75 | variance /= size; 76 | return StockAndPrice(std::string(name), variance); 77 | }); 78 | 79 | auto stockPricing = pico::Pipe() // 80 | .add(pico::ReadFromSocket(server, port, '\n')) // 81 | .add(blackScholes) 82 | . // 83 | add(SPReducer().window(8)) // batch-windowing reduce 84 | .add(pico::WriteToStdOut()); 85 | 86 | /* print the semantic graph and generate dot file */ 87 | stockPricing.print_semantics(); 88 | stockPricing.to_dotfile("stock_pricing_stream.dot"); 89 | 90 | /* execute the pipeline */ 91 | stockPricing.run(); 92 | 93 | /* print execution time */ 94 | std::cout << "done in " << stockPricing.pipe_time() << " ms\n"; 95 | 96 | return 0; 97 | } 98 | -------------------------------------------------------------------------------- /include/pico/Operators/Operator.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef ACTORNODE_HPP_ 22 | #define ACTORNODE_HPP_ 23 | 24 | #include 25 | #include 26 | 27 | #include 28 | 29 | #include "pico/Internals/utils.hpp" 30 | #include "pico/WindowPolicy.hpp" 31 | 32 | namespace pico { 33 | 34 | static unsigned def_par() { 35 | auto env = std::getenv("PARDEG"); 36 | return (unsigned)(env ? atoi(env) : (int)ff_realNumCores()); 37 | } 38 | 39 | /** 40 | * Base class defining a semantic dataflow operator. 41 | * An operator has an input and output cardinality , where 42 | * - I-degree = 0 if generating input, >1 otherwise 43 | * - O-degree = 0 if collecting output, 1 otherwise 44 | * 45 | * Operators have also the specification about the Structure Type they can 46 | * manage. 47 | */ 48 | 49 | class Operator { 50 | public: 51 | Operator() : in_deg(0), out_deg(0) { 52 | st_map[StructureType::BAG] = false; 53 | st_map[StructureType::STREAM] = false; 54 | } 55 | 56 | Operator(const Operator ©) { 57 | set_input_degree(copy.i_degree()); 58 | set_output_degree(copy.o_degree()); 59 | stype(StructureType::BAG, copy.st_map.at(StructureType::BAG)); 60 | stype(StructureType::STREAM, copy.st_map.at(StructureType::STREAM)); 61 | pardeg_ = copy.pardeg_; 62 | } 63 | 64 | virtual ~Operator() {} 65 | 66 | /* 67 | * naming 68 | */ 69 | virtual std::string name() { 70 | std::ostringstream address; 71 | address << (void const *)this; 72 | return name_short() + address.str().erase(0, 2); 73 | } 74 | virtual std::string name_short() = 0; 75 | virtual const OpClass operator_class() = 0; 76 | 77 | /* 78 | * structural properties 79 | */ 80 | virtual bool partitioning() const { return false; } 81 | 82 | virtual bool windowing() const { return false; } 83 | 84 | /* 85 | * syntax-related functions 86 | */ 87 | virtual Operator *clone() = 0; 88 | 89 | void set_input_degree(size_t degree) { in_deg = degree; } 90 | 91 | size_t i_degree() const { return in_deg; } 92 | 93 | void set_output_degree(size_t degree) { out_deg = degree; } 94 | 95 | size_t o_degree() const { return out_deg; } 96 | 97 | const st_map_t stype() const { return st_map; } 98 | 99 | void stype(StructureType s, bool v) { st_map[s] = v; } 100 | 101 | unsigned pardeg() const { return pardeg_; } 102 | 103 | void pardeg(unsigned pardeg__) { pardeg_ = pardeg__; } 104 | 105 | private: 106 | size_t in_deg, out_deg; 107 | st_map_t st_map; 108 | unsigned pardeg_ = def_par(); 109 | }; 110 | 111 | } /* namespace pico */ 112 | 113 | #endif /* ACTORNODE_HPP_ */ 114 | -------------------------------------------------------------------------------- /include/pico/Operators/JoinFlatMapByKey.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef OPERATORS_BINARYMAP_HPP_ 22 | #define OPERATORS_BINARYMAP_HPP_ 23 | 24 | #include "BinaryOperator.hpp" 25 | 26 | #include "pico/ff_implementation/OperatorsFFNodes/JoinFlatMapByKeyFarm.hpp" 27 | 28 | namespace pico { 29 | 30 | /** 31 | * Defines an operator performing a FlatMap over pairs produced by 32 | * key-partitioning two collections and joining elements from same-key 33 | * sub-partitions. 34 | * The FlatMap produces zero or more elements in output, for each input pair, 35 | * according to the callable kernel. 36 | */ 37 | template 38 | class JoinFlatMapByKey : public BinaryOperator { 39 | public: 40 | /** 41 | * \ingroup op-api 42 | * 43 | * JoinFlatMapByKey Constructor 44 | * 45 | * Creates a new JoinFlatMapByKey operator by defining its kernel function. 46 | */ 47 | JoinFlatMapByKey( 48 | std::function &)> kernel_, 49 | unsigned par = def_par()) { 50 | kernel = kernel_; 51 | this->set_input_degree(2); 52 | this->set_output_degree(1); 53 | this->stype(StructureType::BAG, true); 54 | this->stype(StructureType::STREAM, false); 55 | this->pardeg(par); 56 | } 57 | 58 | JoinFlatMapByKey(const JoinFlatMapByKey ©) 59 | : BinaryOperator(copy), kernel(copy.kernel) {} 60 | 61 | std::string name_short() { return "JoinFlatMapByKey"; } 62 | 63 | const OpClass operator_class() { return OpClass::BFMAP; } 64 | 65 | ff::ff_node *node_operator(int parallelism, bool left_input, // 66 | StructureType st) { 67 | assert(st == StructureType::BAG); 68 | using t = JoinFlatMapByKeyFarm, Token, Token>; 69 | return new t(parallelism, kernel, left_input); 70 | } 71 | 72 | ff::ff_node *opt_node(int pardeg, bool lin, PEGOptimization_t opt, 73 | StructureType st, opt_args_t a) { 74 | assert(opt == PJFMAP_PREDUCE); 75 | assert(st == StructureType::BAG); 76 | auto nextop = dynamic_cast *>(a.op); 77 | if (nextop->pardeg() == 1) { 78 | using t = JFMRBK_seq_red, Token, Token>; 79 | return new t(pardeg, lin, kernel, nextop->kernel()); 80 | } 81 | using t = JFMRBK_par_red, Token, Token>; 82 | return new t(pardeg, lin, kernel, nextop->pardeg(), nextop->kernel()); 83 | } 84 | 85 | protected: 86 | JoinFlatMapByKey *clone() { return new JoinFlatMapByKey(*this); } 87 | 88 | private: 89 | std::function &)> kernel; 90 | }; 91 | 92 | } /* namespace pico */ 93 | 94 | #endif /* OPERATORS_BINARYMAP_HPP_ */ 95 | -------------------------------------------------------------------------------- /examples/page-rank/testdata/prg-convert.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | typedef unsigned long long url_t; 30 | 31 | template 32 | void to_adj(std::ifstream &infile, T &adj, U &in_degree) { 33 | for (std::string line; std::getline(infile, line);) { 34 | std::istringstream iss(line); 35 | std::string src, dst; 36 | 37 | iss >> src >> dst; 38 | if (src[0] != '#') { 39 | adj[src].push_back(dst); 40 | in_degree[dst]++; 41 | } 42 | } 43 | } 44 | 45 | template 46 | void write_nodes(const char *out_prefix, T &adj, U &in_degree) { 47 | /* open output file */ 48 | std::ofstream of; 49 | std::stringstream nodes_outfname; 50 | nodes_outfname << out_prefix << "-nodes"; 51 | of.open(nodes_outfname.str()); 52 | if (!of.is_open()) { 53 | std::cerr << "could not open nodes output file\n"; 54 | exit(1); 55 | } 56 | 57 | for (auto a : adj) { 58 | /* emit src node if both input and output degrees are non-zero */ 59 | if (!a.second.empty() && in_degree[a.first] > 0) of << a.first << std::endl; 60 | } 61 | } 62 | 63 | template 64 | void write_edges(const char *out_prefix, T &adj, U &in_degree) { 65 | unsigned long long dropped = 0; 66 | 67 | /* open output file */ 68 | std::ofstream of; 69 | std::stringstream nodes_outfname; 70 | nodes_outfname << out_prefix << "-edges"; 71 | of.open(nodes_outfname.str()); 72 | if (!of.is_open()) { 73 | std::cerr << "could not open edges output file\n"; 74 | exit(1); 75 | } 76 | 77 | /* edges */ 78 | for (auto a : adj) { 79 | for (auto dst : a.second) { 80 | /* print edge if dst has non-zero input/output degree */ 81 | if (in_degree[dst] > 0) 82 | of << a.first << " " << dst << std::endl; 83 | else 84 | ++dropped; 85 | } 86 | } 87 | 88 | /* print dropped edges */ 89 | std::cerr << "> n. dropped edges: " << dropped << std::endl; 90 | } 91 | 92 | int main(int argc, char *argv[]) { 93 | if (argc < 3) { 94 | std::cout << "Usage: " << std::endl; 95 | printf("%s \n", argv[0]); 96 | return 1; 97 | } 98 | 99 | /* open input file */ 100 | std::ifstream infile(argv[1]); 101 | if (!infile.is_open()) { 102 | std::cerr << "could not open input file\n"; 103 | return 1; 104 | } 105 | 106 | /* build adjacency lists */ 107 | std::map> adj; 108 | std::map in_degree; 109 | to_adj(infile, adj, in_degree); 110 | 111 | /* write output */ 112 | write_nodes(argv[2], adj, in_degree); 113 | write_edges(argv[2], adj, in_degree); 114 | } 115 | -------------------------------------------------------------------------------- /tests/wordcount.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "pico/pico.hpp" 26 | 27 | #include "common/io.hpp" 28 | 29 | typedef pico::KeyValue KV; 30 | 31 | /* 32 | * sequential wordcount 33 | */ 34 | 35 | std::unordered_map seq_wc(std::vector lines) { 36 | std::unordered_map wc; 37 | for (auto line : lines) { 38 | std::string::size_type i = 0, j; 39 | while ((j = line.find_first_of(' ', i)) != std::string::npos) { 40 | ++wc[line.substr(i, j - i)]; 41 | i = j + 1; 42 | } 43 | if (i < line.size()) { 44 | ++wc[line.substr(i, line.size() - i)]; 45 | } 46 | } 47 | return wc; 48 | } 49 | 50 | /* static tokenizer function */ 51 | static auto tokenizer = [](std::string& in, 52 | pico::FlatMapCollector& collector) { 53 | std::string::size_type i = 0, j; 54 | while ((j = in.find_first_of(' ', i)) != std::string::npos) { 55 | collector.add(KV(in.substr(i, j - i), 1)); 56 | i = j + 1; 57 | } 58 | if (i < in.size()) { 59 | collector.add(KV(in.substr(i, in.size() - i), 1)); 60 | } 61 | }; 62 | 63 | /* 64 | * convert an unorderd map to a vector of strings 65 | * If (k, v) is a element of un_map then will be an element of the vector 66 | */ 67 | 68 | std::vector to_vec_str( 69 | std::unordered_map un_map) { 70 | std::vector vec; 71 | for (auto pair : un_map) { 72 | vec.push_back(KV(pair.first, pair.second).to_string()); 73 | } 74 | return vec; 75 | } 76 | 77 | TEST_CASE("wordcount", "wordcount tag") { 78 | std::string input_file = "./testdata/lines.txt"; 79 | std::string output_file = "output.txt"; 80 | 81 | /* define a generic word-count pipeline */ 82 | auto countWords = 83 | pico::Pipe() // the empty pipeline 84 | .add(pico::FlatMap(tokenizer)) // 85 | .add(pico::ReduceByKey([](int v1, int v2) { return v1 + v2; })); 86 | 87 | pico::ReadFromFile reader(input_file); 88 | pico::WriteToDisk writer(output_file, 89 | [](KV in) { return in.to_string(); }); 90 | 91 | /* compose the pipeline */ 92 | auto wc = pico::Pipe().add(reader).to(countWords).add(writer); 93 | 94 | /* execute the pipeline */ 95 | wc.run(); 96 | 97 | /* forget the order and compare */ 98 | auto observed = read_lines(output_file); 99 | std::sort(observed.begin(), observed.end()); 100 | 101 | auto input_lines = read_lines(input_file); 102 | std::unordered_map expected_map = seq_wc(input_lines); 103 | 104 | auto expected = to_vec_str(expected_map); 105 | std::sort(expected.begin(), expected.end()); 106 | 107 | REQUIRE(expected == observed); 108 | } 109 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/WriteToStdOut.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * WriteToStdOut.hpp 22 | * 23 | * Created on: Dec 23, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_WRITETOSTDOUT_HPP_ 28 | #define OPERATORS_INOUT_WRITETOSTDOUT_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | #include "pico/ff_implementation/OperatorsFFNodes/InOut/WriteToStdOutFFNode.hpp" 34 | 35 | #include "OutputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that writes data to standard output. 41 | * 42 | * The user specifies the kernel function that operates on each item before 43 | * being printed to standard output. The kernel can be a lambda function, a 44 | * functor or a function. 45 | * 46 | * 47 | * The operator is global and unique for the Pipe it refers to. 48 | */ 49 | 50 | template 51 | class WriteToStdOut : public OutputOperator { 52 | public: 53 | /** 54 | * \ingroup op-api 55 | * WriteToStdOut Constructor 56 | * 57 | * Creates a new WriteToStdOut operator by defining its kernel function. 58 | */ 59 | WriteToStdOut(std::function func_) 60 | : OutputOperator(StructureType::STREAM) { 61 | usr_func = true; 62 | func = func_; 63 | } 64 | 65 | /** 66 | * \ingroup op-api 67 | * WriteToStdOut Constructor 68 | * 69 | * Creates a new WriteToStdOut operator writing by operator<<. 70 | */ 71 | WriteToStdOut() : OutputOperator(StructureType::STREAM) {} 72 | 73 | /** 74 | * Copy constructor. 75 | */ 76 | WriteToStdOut(const WriteToStdOut& copy) : OutputOperator(copy) { 77 | func = copy.func; 78 | usr_func = copy.usr_func; 79 | } 80 | 81 | /** 82 | * Returns a unique name for the operator. 83 | */ 84 | std::string name() { 85 | std::string name("WriteToStdOut"); 86 | std::ostringstream address; 87 | address << (void const*)this; 88 | return name + address.str().erase(0, 2); 89 | } 90 | 91 | /** 92 | * Returns the name of the operator, consisting in the name of the class. 93 | */ 94 | std::string name_short() { return "WriteToStdOut"; } 95 | 96 | protected: 97 | /** 98 | * Duplicates a WriteToStdOut with a copy of the kernel function. 99 | * @return new WriteToStdOut pointer 100 | */ 101 | WriteToStdOut* clone() { return new WriteToStdOut(*this); } 102 | 103 | const OpClass operator_class() { return OpClass::OUTPUT; } 104 | 105 | ff::ff_node* node_operator(int parallelism, StructureType st) { 106 | assert(st == StructureType::STREAM); 107 | if (usr_func) return new WriteToStdOutFFNode>(func); 108 | return new WriteToStdOutFFNode_ostream>(); 109 | } 110 | 111 | private: 112 | bool usr_func = false; 113 | std::function func; 114 | }; 115 | 116 | } /* namespace pico */ 117 | 118 | #endif /* OPERATORS_INOUT_WRITETOSTDOUT_HPP_ */ 119 | -------------------------------------------------------------------------------- /include/pico/Operators/InOut/WriteToDisk.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | /* 21 | * WriteToDisk.hpp 22 | * 23 | * Created on: Sep 30, 2016 24 | * Author: misale 25 | */ 26 | 27 | #ifndef OPERATORS_INOUT_WRITETODISK_HPP_ 28 | #define OPERATORS_INOUT_WRITETODISK_HPP_ 29 | 30 | #include 31 | #include 32 | 33 | #include "pico/ff_implementation/OperatorsFFNodes/InOut/WriteToDiskFFNode.hpp" 34 | 35 | #include "OutputOperator.hpp" 36 | 37 | namespace pico { 38 | 39 | /** 40 | * Defines an operator that writes data to a text file. 41 | * 42 | * The user specifies the kernel function that operates on each line written to 43 | * the text file, passed as a std::string. The kernel can be a lambda function, 44 | * a functor or a function. 45 | * 46 | * 47 | * The operator is global and unique for the Pipe it refers to. 48 | */ 49 | 50 | template 51 | class WriteToDisk : public OutputOperator { 52 | public: 53 | /** 54 | * \ingroup op-api 55 | * 56 | * WritetoDisk Constructor 57 | * 58 | * Creates a new WriteToDisk operator by defining its kernel function. 59 | */ 60 | WriteToDisk(std::string fname_, std::function func_) 61 | : OutputOperator(StructureType::BAG), 62 | fname(fname_), // 63 | usr_func(true), 64 | func(func_) {} 65 | 66 | /** 67 | * \ingroup op-api 68 | * 69 | * WritetoDisk Constructor 70 | * 71 | * Creates a new WriteToDisk writing by ostream. 72 | */ 73 | WriteToDisk(std::string fname_) 74 | : OutputOperator(StructureType::BAG), fname(fname_) {} 75 | 76 | /** 77 | * Copy constructor. 78 | */ 79 | WriteToDisk(const WriteToDisk& copy) 80 | : OutputOperator(copy), fname(copy.fname), func(copy.func) {} 81 | 82 | /** 83 | * Returns a unique name for the operator. 84 | */ 85 | std::string name() { 86 | std::string name("WriteToDisk"); 87 | std::ostringstream address; 88 | address << (void const*)this; 89 | return name + address.str().erase(0, 2); 90 | } 91 | 92 | /** 93 | * Returns the name of the operator, consisting in the name of the class. 94 | */ 95 | std::string name_short() { return "WriteToDisk\n[" + fname + "]"; } 96 | 97 | protected: 98 | /** 99 | * Duplicates a WriteToDisk with a copy of the kernel function. 100 | * @return new WriteToDisk pointer 101 | */ 102 | WriteToDisk* clone() { return new WriteToDisk(*this); } 103 | 104 | ff::ff_node* node_operator(int parallelism, StructureType st) { 105 | assert(st == StructureType::BAG); 106 | if (usr_func) 107 | return new WriteToDiskFFNode(fname, func); 108 | else 109 | return new WriteToDiskFFNode_ostream(fname); 110 | } 111 | 112 | private: 113 | std::string fname; 114 | bool usr_func = false; 115 | std::function func; 116 | }; 117 | 118 | } /* namespace pico */ 119 | 120 | #endif /* ACTORS_INOUT_READFROMFILE_HPP_ */ 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PiCo: High-Performance Data-Analytics Pipelines in C++ 2 | =============== 3 | [![Travis Build Status](https://travis-ci.org/alpha-unito/pico.svg?branch=master)](https://travis-ci.org/alpha-unito/pico) 4 | [![License: LGPL v3](https://img.shields.io/badge/License-LGPL%20v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) 5 | [![GitHub tag](https://img.shields.io/github/tag/alpha-unito/pico.svg)](http://github.com/alpha-unito/pico/releases) 6 | [![GitHub Issues](https://img.shields.io/github/issues/alpha-unito/pico.svg)](http://github.com/alpha-unito/pico/issues) 7 | 8 | PiCo (**Pi**peline **Co**mposition) is an open-source C++11 header-only DSL for *high-performance data analytics*, featuring low latency, high throughput, and minimal memory footprint on multi-core platforms. 9 | 10 | Take a look to the [word-count](examples/word-count/pico_wc.cpp) code to see how easy is writing a PiCo pipeline! 11 | 12 | ## Report bugs + get help 13 | https://github.com/alpha-unito/pico/issues/new 14 | 15 | ## Build and run tests 16 | The following steps require `cmake >=3.1` as build system. 17 | 18 | Get PiCo: 19 | ```bash 20 | git clone https://github.com/alpha-unito/pico.git 21 | ``` 22 | The current implementation is based on [FastFlow](https://github.com/fastflow/fastflow) as runtime system. 23 | Get it and a link it: 24 | ```bash 25 | cd pico 26 | git clone https://github.com/fastflow/fastflow.git 27 | ln -s fastflow/ff . 28 | ``` 29 | :rescue_worker_helmet: A better solution for including FastFlow as dependency is under development! 30 | 31 | Build and run tests and examples: (from `pico` root directory) 32 | ```bash 33 | mkdir build && cd build 34 | cmake .. -DPICO_ENABLE_UNIT_TEST=ON 35 | cmake --build . 36 | ctest 37 | ``` 38 | 39 | ## Use PiCo in your code 40 | Good news! PiCo is header-only, you do not need to build/link any library to use it in your code. 41 | Just include PiCo headers at the beginning of your source file: 42 | ```c++ 43 | #include "pico/pico.hpp" 44 | ``` 45 | and use good ol' compiler flags to include PiCo (and FastFlow) when compiling your `app`: 46 | ```bash 47 | git clone https://github.com/alpha-unito/pico.git 48 | cd pico 49 | git clone https://github.com/fastflow/fastflow.git 50 | ln -s fastflow/ff . 51 | g++ -I/path/to/pico/include -Iff app.cc 52 | ``` 53 | :rescue_worker_helmet: A modern CMake-based solution for linking PiCo (with its dependencies) is under development! 54 | 55 | ## Examples 56 | The [examples](examples) folder contains some proof-of-concept applications, showing the PiCo user experience: 57 | - [word-count](examples/word-count): PiCo pipelines 101 + visualizing application graphs 58 | - [stock-market](examples/stock-market): batch vs stream pipelines 59 | - [page-rank](examples/page-rank): iterative pipelines 60 | 61 | 62 | ## PiCo Team 63 | Maurizio Drocco (maintainer) 64 | Claudia Misale (creator + co-maintainer) 65 | Alberto Riccardo Martinelli (co-maintainer) 66 | 67 | #### Contributors 68 | Marco Aldinucci (boss) 69 | Massimo Torquati (FastFlow maintainer) 70 | Guy Tremblay (DSL wizard) 71 | 72 | ## How to cite PiCo 73 | C. Misale, M. Drocco, G. Tremblay, A. R. Martinelli, and M. Aldinucci, "PiCo: High-Performance Data Analytics Pipelines in Modern C++," Future Generation Computer Systems, Volume 87, 2018. 74 | [![PiCo_DOI_badge](https://img.shields.io/badge/DOI-https%3A%2F%2Fdoi.org%2F10.1016%2Fj.future.2018.05.030-blue.svg)](https://doi.org/10.1016/j.future.2018.05.030) 75 | [![PiCo_BibTexview](https://img.shields.io/badge/BibTex-view-blue.svg)](https://dblp.uni-trier.de/rec/bibtex/journals/fgcs/MisaleDTMA18) 76 | [![PiCo_BibTexdownload](https://img.shields.io/badge/BibTex-download-blue.svg)](https://dblp.uni-trier.de/rec/bib2/journals/fgcs/MisaleDTMA18.bib) 77 | [![PiCo_RISdownload](https://img.shields.io/badge/RIS-download-blue.svg)](https://dblp.uni-trier.de/rec/ris/journals/fgcs/MisaleDTMA18.ris) 78 | -------------------------------------------------------------------------------- /tests/streaming_reduce_by_key.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #include 22 | 23 | #include 24 | 25 | #include "pico/pico.hpp" 26 | 27 | #include "common/io.hpp" 28 | 29 | typedef pico::KeyValue KV; 30 | 31 | /* 32 | * simple reduce by key with windowing that sums pairs. 33 | */ 34 | 35 | TEST_CASE("streaming reduce by key", "streaming reduce by key tag") { 36 | /* 37 | * - read-from-stdin emulated by streaming a file to a socket on port 4000 38 | * - output inspected by redirecting stdout to a file 39 | */ 40 | std::string input_file = "./testdata/pairs.txt"; 41 | std::string output_file = "output.txt"; 42 | constexpr unsigned wsize = 2; 43 | 44 | /* redirect stdout to output file */ 45 | auto coutbuf = std::cout.rdbuf(); // save old buf 46 | std::ofstream out(output_file); 47 | std::cout.rdbuf(out.rdbuf()); // redirect 48 | 49 | /* build the pipeline */ 50 | pico::ReadFromSocket reader("localhost", 4000, '\n'); 51 | pico::WriteToStdOut writer([](KV in) { return in.to_string(); }); 52 | 53 | auto test_pipe = pico::Pipe() 54 | .add(reader) 55 | .add(pico::Map([](std::string line) { 56 | return KV::from_string(line); 57 | })) 58 | .add(pico::ReduceByKey([](int v1, int v2) { 59 | return v1 + v2; 60 | }).window(wsize)) // 61 | .add(writer); 62 | 63 | /* execute the pipeline */ 64 | test_pipe.run(); 65 | 66 | /* undo stdout redirection */ 67 | std::cout.rdbuf(coutbuf); 68 | 69 | /* parse output into grouped char-int pairs */ 70 | std::unordered_map> observed; 71 | auto output_pairs_str = read_lines(output_file); 72 | for (auto kv_str : output_pairs_str) { 73 | auto kv = KV::from_string(kv_str); 74 | observed[kv.Key()].push_back(kv); 75 | } 76 | 77 | /* 78 | * emulate by-key grouping and windowing reduce 79 | */ 80 | std::unordered_map> expected; 81 | auto input_pairs_str = read_lines(input_file); 82 | 83 | /* make groups */ 84 | std::unordered_map> groups; 85 | for (auto kv_str : input_pairs_str) { 86 | auto kv = KV::from_string(kv_str); 87 | groups[kv.Key()].push_back(kv); 88 | } 89 | 90 | /* reduce each group per-window */ 91 | for (auto kgroup : groups) { 92 | auto k(kgroup.first); 93 | auto group(kgroup.second); 94 | auto it = group.begin(); 95 | for (; it + wsize <= group.end(); it += wsize) { 96 | // full window 97 | int wres = 0; 98 | for (auto wit = it; wit < it + wsize; ++wit) wres += wit->Value(); 99 | expected[k].push_back(KV(k, wres)); 100 | } 101 | if (it < group.end()) { 102 | // remaining partial window 103 | int wres = 0; 104 | for (; it < group.end(); ++it) wres += it->Value(); 105 | expected[k].push_back(KV(k, wres)); 106 | } 107 | } 108 | 109 | REQUIRE(expected == observed); 110 | } 111 | -------------------------------------------------------------------------------- /include/pico/Operators/ReduceByKey.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef REDUCEBYKEY_HPP_ 22 | #define REDUCEBYKEY_HPP_ 23 | 24 | #include "pico/Internals/TimedToken.hpp" 25 | #include "pico/Internals/Token.hpp" 26 | #include "pico/ff_implementation/OperatorsFFNodes/PReduceWin.hpp" 27 | 28 | #include "UnaryOperator.hpp" 29 | 30 | namespace pico { 31 | 32 | /** 33 | * Defines a PReduce operator performing a tree reduce function on partitioned 34 | * input (i.e. reduce by key). The reduce kernel is defined by the user and can 35 | * be a lambda function, a functor or a function. 36 | * 37 | * The reduce kernel function operates on windows and/or groups if defined on 38 | * the Pipe. 39 | * 40 | * It implements a tree reduce operator where input and output value are the 41 | * same. 42 | */ 43 | template 44 | class ReduceByKey : public UnaryOperator { 45 | typedef typename In::keytype K; 46 | typedef typename In::valuetype V; 47 | 48 | public: 49 | /** 50 | * \ingroup op-api 51 | * ReduceByKey Constructor 52 | * 53 | * Creates a ReduceByKey operator by defining its kernel function. 54 | */ 55 | ReduceByKey(std::function reducef_, unsigned par = def_par()) 56 | : reducef(reducef_) { 57 | this->set_input_degree(1); 58 | this->set_output_degree(1); 59 | this->stype(StructureType::BAG, true); 60 | this->stype(StructureType::STREAM, false); 61 | this->pardeg(par); 62 | } 63 | 64 | /** 65 | * \ingroup op-api 66 | * ReduceByKey copy Constructor 67 | */ 68 | ReduceByKey(const ReduceByKey& copy) : UnaryOperator(copy) { 69 | reducef = copy.reducef; 70 | win = copy.win ? copy.win->clone() : nullptr; 71 | } 72 | 73 | ~ReduceByKey() { 74 | if (win) delete win; 75 | } 76 | 77 | /** 78 | * Returns the name of the operator, consisting in the name of the class. 79 | */ 80 | std::string name_short() { return "PReduce"; } 81 | 82 | /* 83 | * Sets batch windowing of fixed size. 84 | * Windowing is applied on partitioning basis: 85 | * each window contains only items belonging to a given partition. 86 | */ 87 | ReduceByKey window(size_t size) { 88 | ReduceByKey res(*this); 89 | res.win = new ByKeyWindow>(size); 90 | res.stype(StructureType::STREAM, true); 91 | return res; 92 | } 93 | 94 | std::function kernel() { return reducef; } 95 | 96 | protected: 97 | ReduceByKey* clone() { return new ReduceByKey(*this); } 98 | 99 | const OpClass operator_class() { return OpClass::REDUCE; } 100 | 101 | bool windowing() const { return win != nullptr; } 102 | 103 | bool partitioning() const { return true; } 104 | 105 | ff::ff_node* node_operator(int pardeg, StructureType st) { 106 | // todo assert unique stype 107 | if (st == StructureType::STREAM) { 108 | assert(win); 109 | return new PReduceWin>(pardeg, reducef, win); 110 | } 111 | // todo 112 | return nullptr; 113 | } 114 | 115 | private: 116 | std::function reducef; 117 | WindowPolicy* win = nullptr; 118 | }; 119 | 120 | } /* namespace pico */ 121 | 122 | #endif /* PREDUCE_HPP_ */ 123 | -------------------------------------------------------------------------------- /examples/stock-market/explicit_finite_difference.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef EXAMPLES_STOCK_MARKET_EXPLICIT_FINITE_DIFFERENCE_HPP_ 22 | #define EXAMPLES_STOCK_MARKET_EXPLICIT_FINITE_DIFFERENCE_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | /* Template code for the Explicit Finite Difference 31 | */ 32 | double explicitCallOption( 33 | double S0, // price 34 | double X, // strike price 35 | double T, // time to maturity or option expiration in years 36 | double r, // risk-free interest rate 37 | double sigma, // volatility 38 | int iMax, // grid paramaters 39 | int jMax) { 40 | // declare and initialise local variables (ds,dt) 41 | double S_max = 2 * X; 42 | double dS = S_max / jMax; 43 | double dt = T / iMax; 44 | // create storage for the stock price and option price (old and new) 45 | double S[jMax + 1], vOld[jMax + 1], vNew[jMax + 1]; 46 | // setup and initialise the stock price 47 | for (int j = 0; j <= jMax; j++) { 48 | S[j] = j * dS; 49 | } 50 | // setup and initialise the final conditions on the option price 51 | for (int j = 0; j <= jMax; j++) { 52 | vOld[j] = std::max(S[j] - X, 0.); 53 | vNew[j] = std::max(S[j] - X, 0.); 54 | } 55 | // loop through time levels, setting the option price at each grid point, and 56 | // also on the boundaries 57 | for (int i = iMax - 1; i >= 0; i--) { 58 | // apply boundary condition at S=0 59 | vNew[0] = 0.; 60 | for (int j = 1; j <= jMax - 1; j++) { 61 | double A, B, C; 62 | A = 0.5 * sigma * sigma * j * j * dt + 0.5 * r * j * dt; 63 | B = 1. - sigma * sigma * j * j * dt; 64 | C = 0.5 * sigma * sigma * j * j * dt - 0.5 * r * j * dt; 65 | vNew[j] = 1. / (1. + r * dt) * 66 | (A * vOld[j + 1] + B * vOld[j] + C * vOld[j - 1]); 67 | } 68 | // apply boundary condition at S=S_max 69 | vNew[jMax] = S[jMax] - X * exp(-r * (T - i * dt)); 70 | // set old values to new 71 | for (int j = 0; j <= jMax; j++) { 72 | vOld[j] = vNew[j]; 73 | } 74 | } 75 | // get j* such that S_0 \in [ j*dS , (j*+1)dS ] 76 | int jstar; 77 | jstar = S0 / dS; 78 | double sum = 0.; 79 | // run 2 point Lagrange polynomial interpolation 80 | sum = sum + (S0 - S[jstar + 1]) / (S[jstar] - S[jstar + 1]) * vNew[jstar]; 81 | sum = sum + (S0 - S[jstar]) / (S[jstar + 1] - S[jstar]) * vNew[jstar + 1]; 82 | return sum; 83 | } 84 | 85 | StockPrice explicit_finite_difference(const OptionData &opt, 86 | int iMax, // grid paramaters 87 | int jMax) { 88 | return explicitCallOption(opt.s, opt.strike, opt.t, opt.r, opt.v, iMax, jMax); 89 | } 90 | 91 | // int main() 92 | //{ 93 | // // declare and initialise Black Scholes parameters 94 | // double S0=1.639,X=2.,T=1.,r=0.05,sigma=0.4; 95 | // // declare and initialise grid paramaters 96 | // int iMax=4,jMax=4; 97 | // cout << explicitCallOption(S0,X,T,r,sigma,iMax,jMax) << endl; 98 | // /* OUTPUT 99 | // 0.194858 100 | // */ 101 | //} 102 | 103 | #endif /* EXAMPLES_STOCK_MARKET_EXPLICIT_FINITE_DIFFERENCE_HPP_ */ 104 | -------------------------------------------------------------------------------- /include/pico/KeyValue.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef KEYVALUE_HPP_ 22 | #define KEYVALUE_HPP_ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace pico { 29 | 30 | /** 31 | * \ingroup op-api 32 | */ 33 | template 34 | class KeyValue { 35 | public: 36 | typedef K keytype; 37 | typedef V valuetype; 38 | 39 | KeyValue() {} 40 | 41 | /** 42 | * Explicit constructor 43 | */ 44 | KeyValue(K key_, V val_) : key(key_), val(val_) {} 45 | 46 | /** 47 | * Copy constructor 48 | */ 49 | KeyValue(const KeyValue& kv) : key(kv.key), val(kv.val) {} 50 | 51 | /** 52 | * Move constructor 53 | */ 54 | KeyValue(KeyValue&& kv) : key(std::move(kv.key)), val(std::move(kv.val)) {} 55 | 56 | /** 57 | * Copy assignment 58 | */ 59 | KeyValue& operator=(const KeyValue& kv) { 60 | key = kv.key; 61 | val = kv.val; 62 | return *this; 63 | } 64 | 65 | /** 66 | * Move assignment 67 | */ 68 | KeyValue& operator=(KeyValue&& kv) { 69 | key = std::move(kv.key); 70 | val = std::move(kv.val); 71 | return *this; 72 | } 73 | 74 | /** 75 | * Getter methods. 76 | */ 77 | const K& Key() const { return key; } 78 | 79 | const V& Value() const { return val; } 80 | 81 | V& Value() { return val; } 82 | 83 | /** 84 | * Setter methods. 85 | */ 86 | void Key(K key_) { key = key_; } 87 | void Value(V val_) { val = val_; } 88 | 89 | bool operator==(const KeyValue& kv) const { 90 | return key == kv.Key() && val == kv.Value(); 91 | } 92 | 93 | friend bool operator<(const KeyValue& l, const KeyValue& r) { 94 | return l.Value() < r.Value(); 95 | } 96 | 97 | friend std::ostream& operator<<(std::ostream& os, const KeyValue& kv) { 98 | os << "<" << kv.key << ", " << kv.val << ">"; 99 | return os; 100 | } 101 | 102 | KeyValue& operator+=(const KeyValue& rhs) { 103 | val += rhs.val; // reuse compound assignment 104 | return *this; // return the result by value (uses move constructor) 105 | } 106 | 107 | // friends defined inside class body are inline and are hidden from non-ADL 108 | // lookup 109 | friend KeyValue operator+( 110 | KeyValue lhs, // passing lhs by value helps optimize chained a+b+c 111 | const KeyValue& 112 | rhs) // otherwise, both parameters may be const references 113 | { 114 | lhs.val += rhs.val; // reuse compound assignment 115 | return lhs; // return the result by value (uses move constructor) 116 | } 117 | 118 | void sumValue(const KeyValue& kv) { val += kv.val; } 119 | 120 | bool sameKey(const KeyValue& kv) const { return key == kv.key; } 121 | 122 | std::string to_string() const { 123 | std::stringstream res; 124 | res << *this; 125 | return res.str(); 126 | } 127 | 128 | static KeyValue from_string(std::string s) { 129 | KeyValue res; 130 | std::stringstream in(s); 131 | assert(in.get() == '<'); 132 | in >> res.key; 133 | assert(in.get() == ','); 134 | assert(in.get() == ' '); 135 | in >> res.val; 136 | assert(in.get() == '>'); 137 | return res; 138 | } 139 | 140 | private: 141 | K key; 142 | V val; 143 | }; 144 | 145 | } /* namespace pico */ 146 | 147 | #endif /* KEYVALUE_HPP_ */ 148 | -------------------------------------------------------------------------------- /include/pico/FlatMapCollector.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_TYPES_FLATMAPCOLLECTOR_HPP_ 22 | #define INTERNALS_TYPES_FLATMAPCOLLECTOR_HPP_ 23 | 24 | #include "pico/Internals/Microbatch.hpp" 25 | #include "pico/Internals/Token.hpp" 26 | #include "pico/defines/Global.hpp" 27 | #include "pico/ff_implementation/ff_config.hpp" 28 | 29 | namespace pico { 30 | 31 | /** 32 | * \ingroup op-api 33 | * 34 | * Collector for FlatMap user kernels. 35 | */ 36 | template 37 | class FlatMapCollector { 38 | public: 39 | virtual void add(const Out &) = 0; 40 | virtual void add(Out &&) = 0; 41 | 42 | virtual ~FlatMapCollector() {} 43 | 44 | void tag(base_microbatch::tag_t tag__) { tag_ = tag__; } 45 | 46 | protected: 47 | base_microbatch::tag_t tag() const { return tag_; } 48 | 49 | private: 50 | base_microbatch::tag_t tag_ = 0; 51 | }; 52 | 53 | /** 54 | * A FlatMapCollector for non-decorated collection items 55 | */ 56 | template 57 | class TokenCollector : public FlatMapCollector { 58 | typedef Microbatch> mb_t; 59 | 60 | public: 61 | TokenCollector() { clear(); } 62 | 63 | /** 64 | * Add a token by copying a DataType value 65 | */ 66 | inline void add(const DataType &x) { 67 | make_room(); 68 | 69 | /* insert the element and its token decoration */ 70 | new (head->mb->allocate()) DataType(x); 71 | // TokenType *t = new (mb_t::token_desc(item)) TokenType(*item); 72 | head->mb->commit(); 73 | } 74 | 75 | /** 76 | * Add a token by moving a DataType value 77 | */ 78 | inline void add(DataType &&x) { 79 | make_room(); 80 | 81 | /* insert the element and its token decoration */ 82 | new (head->mb->allocate()) DataType(std::move(x)); 83 | // TokenType *t = new (mb_t::token_desc(item)) TokenType(*item); 84 | head->mb->commit(); 85 | } 86 | 87 | /** 88 | * Clear the list without destroying it. 89 | */ 90 | inline void clear() { 91 | first = head = nullptr; 92 | this->tag(0); 93 | } 94 | 95 | /** 96 | * A collector is stored as a list of Microbatch objects. 97 | * They travel together in order to guarantee the semantics of FlatMap 98 | * when processed. 99 | * 100 | * The list type is exposed since it is freed externally. 101 | */ 102 | struct cnode { 103 | mb_t *mb; 104 | struct cnode *next; 105 | }; 106 | 107 | cnode *begin() { return first; } 108 | 109 | private: 110 | cnode *first, *head; 111 | 112 | /* 113 | * ensure there is an available slot in the head node 114 | */ 115 | inline void make_room() { 116 | if (first) { 117 | if (head->mb->full()) { 118 | assert(head->next == nullptr); 119 | head->next = allocate(); 120 | head = head->next; 121 | } 122 | } 123 | 124 | else { 125 | /* initializes the list */ 126 | first = allocate(); 127 | head = first; 128 | } 129 | } 130 | 131 | inline cnode *allocate() { 132 | assert(this->tag()); 133 | cnode *res = (cnode *)MALLOC(sizeof(cnode)); 134 | res->next = nullptr; 135 | res->mb = NEW(this->tag(), global_params.MICROBATCH_SIZE); 136 | return res; 137 | } 138 | }; 139 | 140 | } /* namespace pico */ 141 | 142 | #endif /* INTERNALS_TYPES_FLATMAPCOLLECTOR_HPP_ */ 143 | -------------------------------------------------------------------------------- /include/pico/PEGOptimizations.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef PICO_PEGOPTIMIZATIONS_HPP_ 22 | #define PICO_PEGOPTIMIZATIONS_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/PEGOptimization/defs.hpp" 27 | #include "pico/Operators/BinaryOperator.hpp" 28 | #include "pico/Operators/UnaryOperator.hpp" 29 | #include "pico/Pipe.hpp" 30 | #include "pico/ff_implementation/SupportFFNodes/PairFarm.hpp" 31 | 32 | namespace pico { 33 | 34 | /* apply PE optimization over two adjacent pipes */ 35 | static bool opt_match(base_UnaryOperator *op1, base_UnaryOperator *op2, // 36 | PEGOptimization_t opt) { 37 | bool res = true; 38 | 39 | if (res) { 40 | auto opc1 = op1->operator_class(), opc2 = op2->operator_class(); 41 | switch (opt) { 42 | case MAP_PREDUCE: 43 | res = res && (opc1 == OpClass::MAP && opc2 == OpClass::REDUCE); 44 | res = res && (!op2->windowing() && op2->partitioning()); 45 | break; 46 | case FMAP_PREDUCE: 47 | res = res && (opc1 == OpClass::FMAP && opc2 == OpClass::REDUCE); 48 | res = res && (!op2->windowing() && op2->partitioning()); 49 | break; 50 | default: 51 | assert(false); 52 | } 53 | } 54 | 55 | return res; 56 | } 57 | 58 | static bool opt_match_binary(const Pipe &p, base_UnaryOperator &op2, 59 | PEGOptimization_t opt) { 60 | bool res = true; 61 | 62 | if (res) { 63 | auto opc1 = p.get_operator_ptr()->operator_class(); 64 | auto opc2 = op2.operator_class(); 65 | switch (opt) { 66 | case PJFMAP_PREDUCE: 67 | // TODO check batch 68 | res = res && (opc1 == OpClass::BFMAP && opc2 == OpClass::REDUCE); 69 | res = res && (!op2.windowing() && op2.partitioning()); 70 | break; 71 | default: 72 | assert(false); 73 | } 74 | } 75 | 76 | return res; 77 | 78 | return false; 79 | } 80 | 81 | template 82 | static bool add_optimized(ff::ff_pipeline *p, ItType it1, ItType it2, // 83 | StructureType st) { 84 | auto &p1 = **it1, &p2 = **it2; 85 | auto p1t = p1.term_node_type(), p2t = p2.term_node_type(); 86 | 87 | if (p1t == Pipe::OPERATOR && p2t == Pipe::OPERATOR) { 88 | /* unary-unary chain */ 89 | auto op1 = dynamic_cast(p1.get_operator_ptr()); 90 | auto op2 = dynamic_cast(p2.get_operator_ptr()); 91 | auto args = opt_args_t{op2}; 92 | 93 | if (opt_match(op1, op2, MAP_PREDUCE)) 94 | p->add_stage(op1->opt_node(op1->pardeg(), MAP_PREDUCE, st, args)); 95 | else if (opt_match(op1, op2, FMAP_PREDUCE)) 96 | p->add_stage(op1->opt_node(op1->pardeg(), FMAP_PREDUCE, st, args)); 97 | else 98 | return false; 99 | } else if (p1t == Pipe::PAIR && p2t == Pipe::OPERATOR) { 100 | /* binary-unary chain */ 101 | auto op2 = dynamic_cast(p2.get_operator_ptr()); 102 | bool lin = p1.in_deg(); // has left-input 103 | auto bop = dynamic_cast(p1.get_operator_ptr()); 104 | auto &children = p1.children(); 105 | assert(children.size() == 2); 106 | auto args = opt_args_t{op2}; 107 | 108 | if (opt_match_binary(p1, *op2, PJFMAP_PREDUCE)) { 109 | p->add_stage(make_pair_farm(*children[0], *children[1], st)); 110 | auto bpar = bop->pardeg(); 111 | p->add_stage(bop->opt_node(bpar, lin, PJFMAP_PREDUCE, st, args)); 112 | } else 113 | return false; 114 | } else 115 | return false; 116 | 117 | return true; 118 | } 119 | 120 | } /* namespace pico */ 121 | 122 | #endif /* PICO_PEGOPTIMIZATIONS_HPP_ */ 123 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Common names for build folders 2 | build/ 3 | Debug/ 4 | 5 | 6 | # Created by https://www.gitignore.io/api/c,c++,cmake,eclipse,linux,windows,macos,vim,emacs 7 | 8 | ### C ### 9 | # Prerequisites 10 | *.d 11 | 12 | # Object files 13 | *.o 14 | *.ko 15 | *.obj 16 | *.elf 17 | 18 | # Linker output 19 | *.ilk 20 | *.map 21 | *.exp 22 | 23 | # Precompiled Headers 24 | *.gch 25 | *.pch 26 | 27 | # Libraries 28 | *.lib 29 | *.a 30 | *.la 31 | *.lo 32 | 33 | # Shared objects (inc. Windows DLLs) 34 | *.dll 35 | *.so 36 | *.so.* 37 | *.dylib 38 | 39 | # Executables 40 | *.exe 41 | *.out 42 | *.app 43 | *.i*86 44 | *.x86_64 45 | *.hex 46 | 47 | # Debug files 48 | *.dSYM/ 49 | *.su 50 | *.idb 51 | *.pdb 52 | 53 | # Kernel Module Compile Results 54 | *.mod* 55 | *.cmd 56 | modules.order 57 | Module.symvers 58 | Mkfile.old 59 | dkms.conf 60 | 61 | 62 | ### C++ ### 63 | # Prerequisites 64 | 65 | # Compiled Object files 66 | *.slo 67 | 68 | # Precompiled Headers 69 | 70 | # Compiled Dynamic libraries 71 | 72 | # Fortran module files 73 | *.mod 74 | *.smod 75 | 76 | # Compiled Static libraries 77 | *.lai 78 | 79 | # Executables 80 | 81 | 82 | ### CMake ### 83 | CMakeCache.txt 84 | CMakeFiles 85 | CMakeScripts 86 | Makefile 87 | cmake_install.cmake 88 | install_manifest.txt 89 | CTestTestfile.cmake 90 | 91 | 92 | ### Eclipse ### 93 | 94 | .metadata 95 | bin/ 96 | tmp/ 97 | *.tmp 98 | *.bak 99 | *.swp 100 | *~.nib 101 | local.properties 102 | .settings/ 103 | .loadpath 104 | .recommenders 105 | 106 | # Eclipse Core 107 | .project 108 | 109 | # External tool builders 110 | .externalToolBuilders/ 111 | 112 | # Locally stored "Eclipse launch configurations" 113 | *.launch 114 | 115 | # PyDev specific (Python IDE for Eclipse) 116 | *.pydevproject 117 | 118 | # CDT-specific (C/C++ Development Tooling) 119 | .cproject 120 | 121 | # JDT-specific (Eclipse Java Development Tools) 122 | .classpath 123 | 124 | # Java annotation processor (APT) 125 | .factorypath 126 | 127 | # PDT-specific (PHP Development Tools) 128 | .buildpath 129 | 130 | # sbteclipse plugin 131 | .target 132 | 133 | # Tern plugin 134 | .tern-project 135 | 136 | # TeXlipse plugin 137 | .texlipse 138 | 139 | # STS (Spring Tool Suite) 140 | .springBeans 141 | 142 | # Code Recommenders 143 | .recommenders/ 144 | 145 | 146 | ### Linux ### 147 | *~ 148 | 149 | # temporary files which can be created if a process still has a handle open of a deleted file 150 | .fuse_hidden* 151 | 152 | # KDE directory preferences 153 | .directory 154 | 155 | # Linux trash folder which might appear on any partition or disk 156 | .Trash-* 157 | 158 | # .nfs files are created when an open file is removed but is still being accessed 159 | .nfs* 160 | 161 | 162 | ### Windows ### 163 | # Windows image file caches 164 | Thumbs.db 165 | ehthumbs.db 166 | 167 | # Folder config file 168 | Desktop.ini 169 | 170 | # Recycle Bin used on file shares 171 | $RECYCLE.BIN/ 172 | 173 | # Windows Installer files 174 | *.cab 175 | *.msi 176 | *.msm 177 | *.msp 178 | 179 | # Windows shortcuts 180 | *.lnk 181 | 182 | 183 | ### macOS ### 184 | *.DS_Store 185 | .AppleDouble 186 | .LSOverride 187 | 188 | # Icon must end with two \r 189 | Icon 190 | # Thumbnails 191 | ._* 192 | # Files that might appear in the root of a volume 193 | .DocumentRevisions-V100 194 | .fseventsd 195 | .Spotlight-V100 196 | .TemporaryItems 197 | .Trashes 198 | .VolumeIcon.icns 199 | .com.apple.timemachine.donotpresent 200 | # Directories potentially created on remote AFP share 201 | .AppleDB 202 | .AppleDesktop 203 | Network Trash Folder 204 | Temporary Items 205 | .apdisk 206 | 207 | 208 | ### Vim ### 209 | # swap 210 | [._]*.s[a-w][a-z] 211 | [._]s[a-w][a-z] 212 | # session 213 | Session.vim 214 | # temporary 215 | .netrwhist 216 | # auto-generated tag files 217 | tags 218 | 219 | 220 | ### Emacs ### 221 | # -*- mode: gitignore; -*- 222 | \#*\# 223 | /.emacs.desktop 224 | /.emacs.desktop.lock 225 | *.elc 226 | auto-save-list 227 | tramp 228 | .\#* 229 | 230 | # Org-mode 231 | .org-id-locations 232 | *_archive 233 | 234 | # flymake-mode 235 | *_flymake.* 236 | 237 | # eshell files 238 | /eshell/history 239 | /eshell/lastdir 240 | 241 | # elpa packages 242 | /elpa/ 243 | 244 | # reftex files 245 | *.rel 246 | 247 | # AUCTeX auto folder 248 | /auto/ 249 | 250 | # cask packages 251 | .cask/ 252 | dist/ 253 | 254 | # Flycheck 255 | flycheck_*.el 256 | 257 | # server auth directory 258 | /server/ 259 | 260 | # projectiles files 261 | .projectile 262 | 263 | # directory configuration 264 | .dir-locals.el 265 | /Debug/ 266 | -------------------------------------------------------------------------------- /examples/stock-market/stock_tweets.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | // This code implements a pipeline for online processing of tweets. 22 | // Given a tweet feed (from standard input), it extracts those tweets that 23 | // mention exactly one stock name. 24 | // For each of them, the word count is computed to exemplify an arbitrary 25 | // processing defined by the user. 26 | // The resulting StockName-WordCount pairs are written to the standard output. 27 | // 28 | // This example shows the processing of a (potentially unbounded) stream. 29 | // Moreover, it shows how a FlatMap operator can be exploited to implement a 30 | // filtering operator. 31 | 32 | #include 33 | #include 34 | #include 35 | 36 | #include "pico/pico.hpp" 37 | 38 | #include "defs.h" 39 | 40 | /* the set of stock names to match tweets against */ 41 | static std::set stock_names; 42 | 43 | /* 44 | * define a generic operator that: 45 | * - filters out all those tweets mentioning zero or multiple stock names 46 | * - count the number of words for each remaining tweet 47 | */ 48 | auto filterTweets = pico::FlatMap( // 49 | [](std::string& tweet, pico::FlatMapCollector& collector) { 50 | StockName stock; 51 | bool single_stock = false; 52 | unsigned long long count = 0; 53 | 54 | /* tokenize the tweet */ 55 | std::istringstream f(tweet); 56 | std::string s; 57 | while (std::getline(f, s, ' ')) { 58 | /* count token length + blank space */ 59 | count += s.size() + 1; 60 | 61 | /* stock name occurrence */ 62 | if (stock_names.find(s) != stock_names.end()) { 63 | if (!single_stock) { 64 | /* first stock name occurrence */ 65 | stock = s; 66 | single_stock = true; 67 | } else if (s != stock) { 68 | /* multiple stock names, invalid record */ 69 | single_stock = false; 70 | break; 71 | } 72 | } 73 | } 74 | 75 | /* emit result if valid record */ 76 | if (single_stock) { 77 | collector.add(StockAndCount(stock, count)); 78 | } 79 | }); 80 | 81 | int main(int argc, char** argv) { 82 | /* parse command line */ 83 | if (argc < 4) { 84 | std::cerr << "Usage: " << argv[0] << " "; 85 | std::cerr << " \n"; 86 | return -1; 87 | } 88 | std::string stock_fname(argv[1]); 89 | std::string server(argv[2]); 90 | int port = atoi(argv[3]); 91 | 92 | /* bring tags to memory */ 93 | std::ifstream stocks_file(stock_fname); 94 | std::string stock_name; 95 | while (stocks_file.good()) { 96 | stocks_file >> stock_name; 97 | stock_names.insert(stock_name); 98 | } 99 | 100 | /* define i/o operators from/to standard input/output */ 101 | pico::ReadFromSocket readTweets(server, port, '-'); 102 | 103 | pico::WriteToStdOut writeCounts( 104 | [](StockAndCount c) { return c.to_string(); }); 105 | 106 | /* compose the main pipeline */ 107 | auto stockTweets = 108 | pico::Pipe(readTweets) // 109 | .add(filterTweets) // 110 | .add(pico::ReduceByKey([](unsigned p1, unsigned p2) { 111 | return std::max(p1, p2); 112 | }).window(8)) 113 | .add(writeCounts); 114 | 115 | /* print the semantic graph and generate dot file */ 116 | stockTweets.print_semantics(); 117 | stockTweets.to_dotfile("stock_tweets.dot"); 118 | 119 | /* execute the pipeline */ 120 | stockTweets.run(); 121 | 122 | /* print execution time */ 123 | std::cout << "done in " << stockTweets.pipe_time() << " ms\n"; 124 | 125 | return 0; 126 | } 127 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/FoldReduceBatch.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FOLDREDUCEBATCH_HPP_ 22 | #define INTERNALS_FOLDREDUCEBATCH_HPP_ 23 | 24 | #include 25 | 26 | #include "pico/Internals/Microbatch.hpp" 27 | #include "pico/Internals/Token.hpp" 28 | #include "pico/Internals/utils.hpp" 29 | #include "pico/WindowPolicy.hpp" 30 | 31 | #include "pico/ff_implementation/ff_config.hpp" 32 | 33 | template 35 | class FoldReduceBatch : public Farm { 36 | public: 37 | FoldReduceBatch(int parallelism, 38 | std::function &foldf_, 39 | std::function &reducef_) { 40 | this->setEmitterF(new ByKeyEmitter( 41 | parallelism, this->getlb(), pico::global_params.MICROBATCH_SIZE)); 42 | this->setCollectorF(new Collector(parallelism, reducef_)); 43 | std::vector w; 44 | for (int i = 0; i < parallelism; ++i) { 45 | w.push_back(new Worker(foldf_)); 46 | } 47 | this->add_workers(w); 48 | this->cleanup_all(); 49 | } 50 | 51 | private: 52 | class Worker : public base_filter { 53 | public: 54 | Worker(std::function &foldf_) : foldf(foldf_) {} 55 | 56 | void cstream_begin_callback(pico::base_microbatch::tag_t tag) { 57 | tag_state[tag].state_ = NEW(); 58 | } 59 | 60 | void kernel(pico::base_microbatch *in_mb) { 61 | mb_t *in_microbatch = reinterpret_cast(in_mb); 62 | auto &s(tag_state[in_mb->tag()]); 63 | // iterate over microbatch 64 | for (In &in : *in_microbatch) { 65 | /* build item and enable copy elision */ 66 | foldf(in, *s.state_); 67 | } 68 | DELETE(in_microbatch); 69 | } 70 | 71 | void cstream_end_callback(pico::base_microbatch::tag_t tag) { 72 | /* wrap state into a microbatch and send out */ 73 | auto &s(tag_state[tag]); 74 | ff_send_out(NEW>(tag, s.state_)); 75 | } 76 | 77 | private: 78 | typedef pico::Microbatch mb_t; 79 | std::function &foldf; 80 | struct state { 81 | State *state_; 82 | }; 83 | std::unordered_map tag_state; 84 | }; 85 | 86 | class Collector : public base_sync_duplicate { 87 | public: 88 | Collector(int nworkers_, 89 | std::function &reducef_) 90 | : reducef(reducef_) {} 91 | 92 | void cstream_begin_callback(pico::base_microbatch::tag_t tag) { 93 | tag_state[tag].state_ = NEW(); 94 | } 95 | 96 | void kernel(pico::base_microbatch *in_mb) { 97 | auto wmb = reinterpret_cast *>(in_mb); 98 | auto &s(tag_state[in_mb->tag()]); 99 | auto in = reinterpret_cast(wmb->get()); 100 | reducef(*in, *s.state_); 101 | DELETE(in); 102 | DELETE(wmb); 103 | } 104 | 105 | void cstream_end_callback(pico::base_microbatch::tag_t tag) { 106 | auto &s(tag_state[tag]); 107 | auto out_mb = NEW(tag, 1); 108 | new (out_mb->allocate()) State(s.state_); 109 | out_mb->commit(); 110 | ff_send_out(out_mb); 111 | } 112 | 113 | private: 114 | std::function &reducef; 115 | typedef pico::Microbatch mb_out; 116 | struct state { 117 | State *state_; 118 | }; 119 | std::unordered_map tag_state; 120 | }; 121 | }; 122 | 123 | #endif /* INTERNALS_FOLDREDUCEBATCH_HPP_ */ 124 | -------------------------------------------------------------------------------- /include/pico/ff_implementation/OperatorsFFNodes/PReduceWin.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019 alpha group, CS department, University of Torino. 3 | * 4 | * This file is part of pico 5 | * (see https://github.com/alpha-unito/pico). 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Lesser General Public License as published by 9 | * the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Lesser General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Lesser General Public License 18 | * along with this program. If not, see . 19 | */ 20 | 21 | #ifndef INTERNALS_FFOPERATORS_PREDUCEWIN_HPP_ 22 | #define INTERNALS_FFOPERATORS_PREDUCEWIN_HPP_ 23 | 24 | #include 25 | 26 | #include 27 | 28 | #include "pico/Internals/utils.hpp" 29 | #include "pico/KeyValue.hpp" 30 | #include "pico/WindowPolicy.hpp" 31 | 32 | #include "pico/ff_implementation/SupportFFNodes/ByKeyEmitter.hpp" 33 | #include "pico/ff_implementation/SupportFFNodes/collectors.hpp" 34 | #include "pico/ff_implementation/SupportFFNodes/farms.hpp" 35 | #include "pico/ff_implementation/ff_config.hpp" 36 | 37 | /* 38 | * Partitions input stream by key and reduces sub-streams on per-window basis. 39 | * A non-ordering farm is sufficient for keeping intra-key ordering. 40 | * Only batching windowing is supported by now, windowing is performed by 41 | * workers. 42 | */ 43 | template 44 | class PReduceWin : public NonOrderingFarm { 45 | typedef typename In::keytype K; 46 | typedef typename In::valuetype V; 47 | 48 | public: 49 | PReduceWin(int parallelism, std::function &preducef, 50 | pico::WindowPolicy *win) { 51 | auto e = new ByKeyEmitter(parallelism); 52 | this->setEmitterF(e); 53 | this->setCollectorF(new ForwardingCollector( 54 | parallelism)); // collects and emits single items 55 | std::vector w; 56 | for (int i = 0; i < parallelism; ++i) { 57 | w.push_back(new PReduceWinWorker(preducef, win->win_size())); 58 | } 59 | this->add_workers(w); 60 | this->cleanup_all(); 61 | } 62 | 63 | private: 64 | class PReduceWinWorker : public base_filter { 65 | public: 66 | PReduceWinWorker(std::function &reducef_, size_t win_size_) 67 | : rkernel(reducef_), win_size(win_size_) {} 68 | 69 | void kernel(pico::base_microbatch *in_mb_) { 70 | auto in_mb = reinterpret_cast(in_mb_); 71 | auto tag = in_mb_->tag(); 72 | auto &s(tag_state[tag]); 73 | for (In &kv : *in_mb) { 74 | auto k(kv.Key()); 75 | if (s.kvmap.find(k) != s.kvmap.end() && s.kvcountmap[k]) { 76 | ++s.kvcountmap[k]; 77 | s.kvmap[k] = rkernel(s.kvmap[k], kv.Value()); 78 | } else { 79 | s.kvcountmap[k] = 1; 80 | s.kvmap[k] = kv.Value(); 81 | } 82 | if (s.kvcountmap[k] == win_size) { 83 | mb_t *out_mb; 84 | out_mb = NEW(tag, 1); 85 | new (out_mb->allocate()) In(k, s.kvmap[k]); 86 | out_mb->commit(); 87 | ff_send_out(reinterpret_cast(out_mb)); 88 | s.kvcountmap[k] = 0; 89 | } 90 | } 91 | DELETE(in_mb); 92 | } 93 | 94 | void cstream_end_callback(pico::base_microbatch::tag_t tag) { 95 | auto &s(tag_state[tag]); 96 | /* stream out incomplete windows */ 97 | for (auto kc : s.kvcountmap) { 98 | auto k(kc.first); 99 | if (kc.second) { 100 | mb_t *out_mb; 101 | out_mb = NEW(tag, 1); 102 | new (out_mb->allocate()) In(k, s.kvmap[k]); 103 | out_mb->commit(); 104 | ff_send_out(reinterpret_cast(out_mb)); 105 | s.kvcountmap[k] = 0; 106 | } 107 | } 108 | } 109 | 110 | private: 111 | typedef pico::Microbatch mb_t; 112 | std::function rkernel; 113 | struct key_state { 114 | std::unordered_map kvmap; // partial per-window/key reduced value 115 | std::unordered_map kvcountmap; // per-window/key counter 116 | }; 117 | std::unordered_map tag_state; 118 | size_t win_size; 119 | }; 120 | }; 121 | 122 | #endif /* INTERNALS_FFOPERATORS_PREDUCEWIN_HPP_ */ 123 | --------------------------------------------------------------------------------