├── ci ├── email-test │ ├── start.out │ ├── rules-mail.cpp │ ├── run.sh │ ├── send-events.py │ ├── run-memory-check-stop-postfix.sh │ └── run-memory-check.sh ├── graphite-test │ ├── graphite-read-events.sh │ ├── rules-forward.cpp │ ├── rules-graphite.cpp │ ├── send-events.py │ └── run.sh ├── ws-test │ ├── run.sh │ ├── test-ws.py │ ├── ws-read-events.py │ └── generate-events.py ├── send-test │ └── run.py └── setup-drone.sh ├── debian ├── compat ├── conffiles ├── dirs ├── install ├── README ├── README.Debian ├── README.source ├── cavalieri.postinst ├── cavalieri.preinst ├── control ├── rules ├── changelog └── copyright ├── src ├── core │ ├── core.cpp │ ├── mock_core.cpp │ ├── real_core.cpp │ └── real_core_helper.cpp ├── query │ ├── grammar.sh │ ├── scanner.h │ ├── driver.cpp │ ├── stack.hh │ └── parser.yy ├── cavalieri.cpp ├── index │ └── mock_index.cpp ├── os │ ├── os_functions.cpp │ ├── real_os_functions.cpp │ └── mock_os_functions.cpp ├── instrumentation │ ├── gauge.cpp │ ├── rate.cpp │ ├── mem.cpp │ └── reservoir.cpp ├── riemann_udp_pool.cpp ├── external │ ├── graphite.cpp │ ├── riemann_tcp_client.cpp │ ├── graphite_pool.cpp │ ├── riemann_tcp_client_pool.cpp │ ├── mock_external.cpp │ ├── pagerduty_pool.cpp │ ├── real_external.cpp │ └── mailer_pool.cpp ├── rules │ └── util.cpp ├── pub_sub │ └── pub_sub.cpp ├── proto.proto ├── transport │ ├── listen_tcp_socket.cpp │ ├── udp_pool.cpp │ └── tcp_connection.cpp ├── streams │ ├── lib.cpp │ └── stream_infra.cpp ├── websocket │ ├── worker_threads.cpp │ └── worker_pool.cpp ├── scheduler │ └── mock_scheduler.cpp ├── rule_tester.cpp ├── riemann_tcp_pool.cpp ├── pool │ ├── executor_thread_pool.cpp │ └── async_thread_pool.cpp ├── folds │ └── folds.cpp ├── config │ └── config.cpp ├── predicates │ └── predicates.cpp ├── riemann_tcp_connection.cpp └── rule_tester_util.cpp ├── docs └── images │ ├── split-1.odg │ ├── split-1.png │ ├── split-2.png │ └── split-3.png ├── include ├── instrumentation │ ├── mem.h │ ├── gauge.h │ ├── rate.h │ ├── reservoir.h │ └── instrumentation.h ├── transport │ ├── listen_tcp_socket.h │ ├── tcp_connection.h │ ├── udp_pool.h │ ├── ws_connection.h │ ├── ws_util.h │ ├── tcp_client_pool.h │ ├── tcp_pool.h │ └── curl_pool.h ├── rules_loader.h ├── external │ ├── mailer.h │ ├── external_event.h │ ├── graphite_pool.h │ ├── graphite.h │ ├── rieman_tcp_client_pool.h │ ├── rieman_tcp_client.h │ ├── mailer_pool.h │ ├── pagerduty_pool.h │ ├── mock_external.h │ ├── external.h │ └── real_external.h ├── websocket │ ├── common.h │ ├── worker_threads.h │ ├── worker_pool.h │ └── websocket_pool.h ├── real_os_functions.h ├── rules │ ├── util.h │ └── common.h ├── os │ ├── mock_os_functions.h │ └── os_functions.h ├── riemann_udp_pool.h ├── index │ ├── mock_index.h │ ├── index.h │ └── real_index.h ├── rule_tester_util.h ├── folds │ └── folds.h ├── config │ └── config.h ├── core │ ├── core.h │ ├── mock_core.h │ ├── real_core_helper.h │ └── real_core.h ├── pub_sub │ └── pub_sub.h ├── query │ ├── driver.h │ └── expression.h ├── riemann_tcp_connection.h ├── riemann_tcp_pool.h ├── scheduler │ ├── scheduler.h │ ├── mock_scheduler.h │ └── real_scheduler.h ├── pool │ ├── executor_thread_pool.h │ └── async_thread_pool.h ├── streams │ ├── stream_functions_lock.h │ ├── lib.h │ └── stream_infra.h ├── predicates │ └── predicates.h ├── util │ └── util.h ├── common │ └── event.h └── async │ └── async_loop.h ├── conf └── cavalieri.conf.in ├── tests ├── COVERAGE ├── basic_test_case.h ├── mock_async_fd.h ├── pubsub_test_case.h ├── test_plan.cpp ├── folds_test_case.h ├── index_test_case.h ├── tcp_connection_test_case.h ├── mock_scheduler_test_case.h ├── riemann_tcp_connection_test_case.h ├── ws_connection_test_case.h └── CMakeLists.txt ├── .cmake └── Modules │ ├── FindJsonCpp.cmake │ ├── FindCrypto.cmake │ ├── FindGlog.cmake │ ├── FindLibEv.cmake │ ├── FindGFlags.cmake │ └── FindTBB.cmake ├── thirdparty └── CMakeLists.txt ├── extra ├── client.py └── client-udp.py ├── LICENSE └── CHANGELOG.md /ci/email-test/start.out: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /debian/conffiles: -------------------------------------------------------------------------------- 1 | /etc/cavalieri/cavalieri.conf 2 | -------------------------------------------------------------------------------- /debian/dirs: -------------------------------------------------------------------------------- 1 | usr/lib/cavalieri 2 | var/log/cavalieri 3 | -------------------------------------------------------------------------------- /src/core/core.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | std::shared_ptr g_core; 4 | -------------------------------------------------------------------------------- /debian/install: -------------------------------------------------------------------------------- 1 | debian/cavalieri/usr/share/doc/cavalieri/examples/cavalieri.conf etc/cavalieri 2 | -------------------------------------------------------------------------------- /docs/images/split-1.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juruen/cavalieri/HEAD/docs/images/split-1.odg -------------------------------------------------------------------------------- /docs/images/split-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juruen/cavalieri/HEAD/docs/images/split-1.png -------------------------------------------------------------------------------- /docs/images/split-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juruen/cavalieri/HEAD/docs/images/split-2.png -------------------------------------------------------------------------------- /docs/images/split-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/juruen/cavalieri/HEAD/docs/images/split-3.png -------------------------------------------------------------------------------- /ci/graphite-test/graphite-read-events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tcpserver 0 2003 sh -c 'cat > graphite-$TCPREMOTEPORT.out' 4 | -------------------------------------------------------------------------------- /src/query/grammar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | yacc -o parser.cpp --defines=parser.h parser.yy 4 | flex -o scanner.cpp scanner.ll 5 | 6 | 7 | -------------------------------------------------------------------------------- /ci/graphite-test/rules-forward.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | streams_t* rules() { 4 | 5 | return new streams_t(forward("localhost", 15555)); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /ci/graphite-test/rules-graphite.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | streams_t* rules() { 4 | 5 | return new streams_t(send_graphite("localhost", 2003)); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/cavalieri.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main(int argc, char **argv) 4 | { 5 | 6 | start_core(argc, argv); 7 | 8 | return 0; 9 | 10 | } 11 | -------------------------------------------------------------------------------- /include/instrumentation/mem.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INSTRUMENTATION_MEM_H 2 | #define CAVALIERI_INSTRUMENTATION_MEM_H 3 | 4 | void process_mem_usage(double & vm, double & rss); 5 | 6 | #endif 7 | -------------------------------------------------------------------------------- /debian/README: -------------------------------------------------------------------------------- 1 | The Debian Package miniriemann 2 | ---------------------------- 3 | 4 | Comments regarding the Package 5 | 6 | -- Javier Uruen Sat, 22 Mar 2014 22:31:42 +0100 7 | -------------------------------------------------------------------------------- /include/transport/listen_tcp_socket.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_LISTEN_TCP_SOCKET_H 2 | #define CAVALIERI_TRANSPORT_LISTEN_TCP_SOCKET_H 3 | 4 | int create_tcp_listen_socket(int port); 5 | 6 | #endif 7 | -------------------------------------------------------------------------------- /conf/cavalieri.conf.in: -------------------------------------------------------------------------------- 1 | -log_dir=/var/log/cavalieri/ 2 | -v=1 3 | -rules_directory=${CMAKE_INSTALL_PREFIX}/lib/cavalieri 4 | -enable_internal_metrics=true 5 | -index_expire_interval=60 6 | -events_port=5555 7 | -ws_port=5556 8 | -------------------------------------------------------------------------------- /debian/README.Debian: -------------------------------------------------------------------------------- 1 | miniriemann for Debian 2 | ---------------------- 3 | 4 | 5 | 6 | -- Javier Uruen Sat, 22 Mar 2014 22:31:42 +0100 7 | -------------------------------------------------------------------------------- /include/rules_loader.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RULES_LOADER_H 2 | #define CAVALIERI_RULES_LOADER_H 3 | 4 | #include 5 | 6 | void load_rules(const std::string file, std::vector & streams); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /ci/email-test/rules-mail.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | streams_t* rules() { 4 | 5 | auto mail_stream = email("localhost", "cavalieri@localhost", 6 | "ubuntu@localhost"); 7 | 8 | return new streams_t(mail_stream); 9 | } 10 | -------------------------------------------------------------------------------- /debian/README.source: -------------------------------------------------------------------------------- 1 | miniriemann for Debian 2 | ---------------------- 3 | 4 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /include/external/mailer.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_MAILER_H 2 | #define CAVALIERI_MAILER_H 3 | 4 | #include 5 | 6 | streams_t email(const std::string & server, const std::string & from, 7 | const std::string & to); 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /src/index/mock_index.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void mock_index::add_event(const Event & e) { 5 | events_.push_back({g_core->sched().unix_time(), e}); 6 | } 7 | 8 | std::vector mock_index::events() const { 9 | return events_; 10 | } 11 | -------------------------------------------------------------------------------- /debian/cavalieri.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Fakeroot and lein don't get along, so we set ownership after the fact. 3 | #test -d /var/log/cavalieri || mkdir /var/log/cavalieri 4 | chown cavalieri:cavalieri /var/log/cavalieri 5 | chown -R cavalieri:cavalieri /etc/cavalieri 6 | #test -d /usr/lib/cavalieri || mkdir /usr/lib/cavalieri 7 | 8 | -------------------------------------------------------------------------------- /include/websocket/common.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_WEBSOCKET_COMMON_H 2 | #define CAVALIERI_WEBSOCKET_COMMON_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using event_filters_t = std::vector>; 9 | 10 | const size_t k_max_ws_queue_size = 20E4; 11 | 12 | #endif 13 | -------------------------------------------------------------------------------- /tests/COVERAGE: -------------------------------------------------------------------------------- 1 | Code Coverage 2 | ------------- 3 | 4 | Dependences: lcov 5 | 6 | With a Debug build type (cmake -DCMAKE_BUILD_TYPE=Debug ..) call: 7 | 8 | make test_plan_coverage 9 | 10 | This command will create a "coverage" directory with an HTML output (index.html) 11 | with coverage information, open your favorite browser and enjoy it. 12 | -------------------------------------------------------------------------------- /debian/cavalieri.preinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Create cavalieri user and group 3 | USERNAME="cavalieri" 4 | GROUPNAME="cavalieri" 5 | getent group "$GROUPNAME" >/dev/null || groupadd -r "$GROUPNAME" 6 | getent passwd "$USERNAME" >/dev/null || \ 7 | useradd -r -g "$GROUPNAME" -d /usr/lib/cavalieri -s /bin/false \ 8 | -c "Cavalieri monitoring system" "$USERNAME" 9 | exit 0 10 | -------------------------------------------------------------------------------- /include/real_os_functions.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_REAL_OS_FUNCTIONS_H 2 | #define CAVALIERI_REAL_OS_FUNCTIONS_H 3 | 4 | #include 5 | 6 | class real_os_functions : public os_functions_interface { 7 | public: 8 | ssize_t recv(int fd, void *buf, size_t len, int flags); 9 | ssize_t write(int fd, void *buff, size_t count); 10 | }; 11 | 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /include/external/external_event.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_EXTERNAL_EVENT_H 2 | #define CAVALIERI_EXTERNAL_EXTERNAL_EVENT_H 3 | 4 | #include 5 | #include 6 | 7 | typedef struct { 8 | const std::string external_method; 9 | const std::string message; 10 | const std::string extra; 11 | const time_t time; 12 | const Event e; 13 | } external_event_t; 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /src/os/os_functions.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | os_functions::os_functions(os_functions_interface & impl) : impl_(impl) 4 | { 5 | } 6 | 7 | ssize_t os_functions::recv(int fd, void *buf, size_t len, int flags) { 8 | return impl_.recv(fd, buf, len, flags); 9 | } 10 | 11 | ssize_t os_functions::write(int fd, void *buf, size_t count) { 12 | return impl_.write(fd, buf, count); 13 | } 14 | -------------------------------------------------------------------------------- /include/rules/util.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RULES_UTIL_H 2 | #define CAVALIERI_RULES_UTIL_H 3 | 4 | #include 5 | 6 | streams_t max_critical_hosts(size_t n); 7 | 8 | struct target_t { 9 | streams_t pagerduty; 10 | streams_t email; 11 | streams_t index; 12 | streams_t all; 13 | }; 14 | 15 | target_t create_targets(const std::string pd_key, const std::string email); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /src/instrumentation/gauge.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | gauge::gauge() : gauge_(0) {}; 4 | 5 | void gauge::update(unsigned int ticks) { 6 | gauge_.store(ticks); 7 | } 8 | 9 | void gauge::incr(unsigned int ticks) { 10 | gauge_ += ticks; 11 | } 12 | 13 | void gauge::decr(unsigned int ticks) { 14 | gauge_ -= ticks; 15 | } 16 | 17 | unsigned int gauge::snapshot() { 18 | return gauge_.load(); 19 | } 20 | -------------------------------------------------------------------------------- /src/riemann_udp_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | riemann_udp_pool::riemann_udp_pool(uint32_t port, raw_msg_fn_t raw_msg_fn) 5 | : 6 | udp_pool_(1, port, raw_msg_fn) 7 | { 8 | udp_pool_.start_threads(); 9 | } 10 | 11 | void riemann_udp_pool::stop() { 12 | VLOG(3) << "stop()"; 13 | 14 | udp_pool_.stop_threads(); 15 | } 16 | 17 | riemann_udp_pool::~riemann_udp_pool() {} 18 | -------------------------------------------------------------------------------- /tests/basic_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef BASIC_TEST_CASE 2 | #define BASIC_TEST_CASE 3 | 4 | #include 5 | #include "util/util.h" 6 | 7 | TEST(basic_test_case, test) 8 | { 9 | std::vector str; 10 | 11 | str.push_back('H'); 12 | str.push_back('E'); 13 | str.push_back('L'); 14 | str.push_back('L'); 15 | str.push_back('O'); 16 | 17 | EXPECT_EQ("SEVMTE8=", base64Encode(str)); 18 | } 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /include/os/mock_os_functions.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_MOCK_OS_FUNCTIONS_H 2 | #define CAVALIERI_MOCK_OS_FUNCTIONS_H 3 | 4 | #include 5 | #include 6 | 7 | class mock_os_functions : public os_functions_interface { 8 | public: 9 | ssize_t recv(int fd, void *buf, size_t len, int flags); 10 | ssize_t write(int fd, void *buf, size_t count); 11 | 12 | std::vector buffer; 13 | }; 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /include/instrumentation/gauge.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INSTRUMENTATION_GAUGE_H 2 | #define CAVALIERI_INSTRUMENTATION_GAUGE_H 3 | 4 | #include 5 | 6 | class gauge { 7 | public: 8 | 9 | gauge(); 10 | 11 | void update(unsigned int ticks); 12 | void incr(unsigned int ticks); 13 | void decr(unsigned int ticks); 14 | 15 | unsigned int snapshot(); 16 | 17 | private: 18 | 19 | std::atomic gauge_; 20 | 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /include/instrumentation/rate.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INSTRUMENTATION_RATE_H 2 | #define CAVALIERI_INSTRUMENTATION_RATE_H 3 | 4 | #include 5 | #include 6 | 7 | class rate { 8 | public: 9 | 10 | rate(); 11 | void add(unsigned int ticks); 12 | double snapshot(); 13 | void reset(); 14 | 15 | private: 16 | std::atomic counter_; 17 | std::chrono::high_resolution_clock::time_point start_; 18 | 19 | }; 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /include/riemann_udp_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RIEMANN_UDP_POOL 2 | #define CAVALIERI_RIEMANN_UDP_POOL 3 | 4 | #include 5 | 6 | typedef std::function)> raw_msg_fn_t; 7 | 8 | class riemann_udp_pool { 9 | public: 10 | riemann_udp_pool(uint32_t port, raw_msg_fn_t raw_msg_fn); 11 | void stop(); 12 | ~riemann_udp_pool(); 13 | 14 | private: 15 | udp_pool udp_pool_; 16 | }; 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/os/real_os_functions.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | ssize_t real_os_functions::recv(int fd, void *buf, size_t len, int flags) { 7 | return ::recv(fd, buf, len, flags); 8 | } 9 | 10 | ssize_t real_os_functions::write(int fd, void *buf, size_t count) { 11 | return ::write(fd, buf, count); 12 | } 13 | 14 | real_os_functions real_os; 15 | os_functions g_os_functions(real_os); 16 | -------------------------------------------------------------------------------- /include/index/mock_index.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INDEX_MOCK_INDEX_H 2 | #define CAVALIERI_INDEX_MOCK_INDEX_H 3 | 4 | #include 5 | #include 6 | 7 | typedef std::pair mock_index_events_t; 8 | 9 | class mock_index : public index_interface { 10 | public: 11 | void add_event(const Event& e); 12 | std::vector events() const; 13 | 14 | private: 15 | std::vector events_; 16 | }; 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /include/rule_tester_util.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RULE_TESTER_UTIL_H 2 | #define CAVALIERI_RULE_TESTER_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | typedef std::pair mock_index_events_t; 9 | 10 | std::vector json_to_events(const std::string json, bool & ok); 11 | 12 | std::string results(std::vector index_events, 13 | std::vector external_events); 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /include/folds/folds.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_FOLDS_H 2 | #define CAVALIERI_FOLDS_H 3 | 4 | #include 5 | 6 | Event sum(const std::vector & events); 7 | 8 | Event product(const std::vector & events); 9 | 10 | Event difference(const std::vector & events); 11 | 12 | Event mean(const std::vector & events); 13 | 14 | Event minimum(const std::vector & events); 15 | 16 | Event maximum(const std::vector & events); 17 | 18 | Event count(const std::vector & events); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /.cmake/Modules/FindJsonCpp.cmake: -------------------------------------------------------------------------------- 1 | include(LibFindMacros) 2 | 3 | libfind_pkg_check_modules(JsonCpp_PKGCONF jsoncpp) 4 | 5 | find_path(JsonCpp_INCLUDE_DIR 6 | NAMES json/features.h 7 | PATH_SUFFIXES jsoncpp 8 | PATHS ${JsonCpp_PKGCONF_INCLUDE_DIRS} # /usr/include/jsoncpp/json 9 | ) 10 | 11 | find_library(JsonCpp_LIBRARY 12 | NAMES jsoncpp 13 | PATHS ${JsonCpp_PKGCONF_LIBRARY_DIRS} 14 | ) 15 | 16 | set(JsonCpp_PROCESS_INCLUDES JsonCpp_INCLUDE_DIR) 17 | set(JsonCpp_PROCESS_LIBS JsonCpp_LIBRARY) 18 | libfind_process(JsonCpp) 19 | -------------------------------------------------------------------------------- /tests/mock_async_fd.h: -------------------------------------------------------------------------------- 1 | #ifndef MOCK_ASYNC_FD_H 2 | #define MOCK_ASYNC_FD_H 3 | 4 | #include 5 | #include 6 | 7 | class mock_async_fd : public async_fd { 8 | public: 9 | MOCK_CONST_METHOD0(fd, int()); 10 | MOCK_CONST_METHOD0(error, bool()); 11 | MOCK_METHOD0(stop, void()); 12 | MOCK_CONST_METHOD0(ready_read, bool()); 13 | MOCK_CONST_METHOD0(ready_write, bool()); 14 | MOCK_METHOD1(set_mode, void(const mode&)); 15 | MOCK_METHOD0(loop, async_loop&()); 16 | }; 17 | 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /include/external/graphite_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_GRAPHITE_POOL_H 2 | #define CAVALIERI_EXTERNAL_GRAPHITE_POOL_H 3 | 4 | #include 5 | 6 | class graphite_pool { 7 | public: 8 | graphite_pool(const size_t thread_num, const std::string host, 9 | const int port); 10 | void push_event(const Event & event); 11 | 12 | private: 13 | std::vector output_events(const std::vector events); 14 | 15 | private: 16 | tcp_client_pool tcp_client_pool_; 17 | }; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/os/mock_os_functions.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | ssize_t mock_os_functions::recv(int, void *buf, size_t len, int) { 5 | auto min = std::min(len, buffer.size()); 6 | std::copy(buffer.begin(), buffer.begin() + min, static_cast(buf)); 7 | return min; 8 | } 9 | 10 | ssize_t mock_os_functions::write(int, void *buf, size_t count) { 11 | auto min = std::min(count, buffer.capacity()); 12 | auto pbuf = static_cast(buf); 13 | buffer.insert(buffer.end(), pbuf, pbuf + min); 14 | return min; 15 | } 16 | -------------------------------------------------------------------------------- /ci/ws-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | set -e 4 | 5 | # Build rules to index everything 6 | git clone https://github.com/juruen/cavalieri-rules.git 7 | cd cavalieri-rules 8 | git checkout -b index-everything origin/index-everything 9 | mkdir build && cd build && cmake .. && make 10 | 11 | # Run cavalieri in background 12 | cavalieri 2>/dev/null & 13 | 14 | # Run integration test 15 | cd ../../ 16 | if ! timeout 30 ./test-ws.py; then 17 | echo "websocket test failed" 18 | pkill cavalieri 19 | exit 1 20 | fi 21 | 22 | # Kill cavalieri 23 | pkill cavalieri 24 | -------------------------------------------------------------------------------- /include/websocket/worker_threads.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_WEBSOCKET_WORKER_THREADS_H 2 | #define CAVALIERI_WEBSOCKET_WORKER_THREADS_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class worker_threads { 9 | public: 10 | using task_t = std::function; 11 | 12 | worker_threads(const size_t threads, const task_t task); 13 | void stop(); 14 | 15 | void run_tasks(const int i); 16 | 17 | private: 18 | 19 | task_t task_; 20 | std::vector finished_threads_; 21 | std::vector threads_; 22 | 23 | }; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /include/external/graphite.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_GRAPHITE_H 2 | #define CAVALIERI_EXTERNAL_GRAPHITE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | class graphite { 10 | public: 11 | graphite(const config conf); 12 | void push_event(const std::string host, const int port, const Event & event); 13 | 14 | private: 15 | const config config_; 16 | std::unordered_map> pool_map_; 18 | std::mutex mutex_; 19 | 20 | }; 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /include/instrumentation/reservoir.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INSTRUMENTATION_RESERVOIR_H 2 | #define CAVALIERI_INSTRUMENTATION_RESERVOIR_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class reservoir { 9 | public: 10 | typedef std::vector samples_t; 11 | 12 | reservoir(); 13 | reservoir(const size_t size); 14 | void add_sample(const double sample); 15 | samples_t snapshot(); 16 | 17 | private: 18 | 19 | const size_t reservoir_size_; 20 | samples_t samples_; 21 | std::atomic n_; 22 | std::mutex mutex_; 23 | 24 | }; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /include/index/index.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INDEX_INDEX_H 2 | #define CAVALIERI_INDEX_INDEX_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using push_event_fn_t = std::function; 11 | using match_fn_t = std::function; 12 | using spwan_thread_fn_t = std::function)>; 13 | 14 | class index_interface { 15 | public: 16 | virtual void add_event(const Event & e) = 0; 17 | virtual ~index_interface() {}; 18 | }; 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /include/external/rieman_tcp_client_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_RIEMANN_TCP_CLIENT_POOL_H 2 | #define CAVALIERI_EXTERNAL_RIEMANN_TCP_CLIENT_POOL_H 3 | 4 | #include 5 | 6 | class riemann_tcp_client_pool { 7 | public: 8 | riemann_tcp_client_pool(const size_t thread_num, const std::string host, 9 | const int port); 10 | void push_event(const Event & event); 11 | 12 | private: 13 | std::vector output_events(const std::vector events); 14 | 15 | private: 16 | tcp_client_pool tcp_client_pool_; 17 | }; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /.cmake/Modules/FindCrypto.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Glog, Google Log Library 2 | # Once done, this will define 3 | # 4 | # Glog_FOUND - system has Glog 5 | # Glog_INCLUDE_DIRS - the Glog include directories 6 | # Glog_LIBRARIES - link these to use Glog 7 | 8 | INCLUDE(LibFindMacros) 9 | 10 | FIND_LIBRARY(Crypto_LIBRARY 11 | NAMES 12 | crypto 13 | ) 14 | 15 | 16 | # Set the include dir variables and the libraries and let libfind_process do the rest. 17 | # NOTE: Singular variables for this library, plural for libraries this this lib depends on. 18 | set(Glog_PROCESS_LIBS Crypto_LIBRARY) 19 | 20 | LIBFIND_PROCESS(Crypto) 21 | -------------------------------------------------------------------------------- /include/os/os_functions.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_OS_FUNCTIONS_H 2 | #define CAVALIERI_OS_FUNCTIONS_H 3 | 4 | #include 5 | 6 | class os_functions_interface { 7 | public: 8 | virtual ssize_t recv(int fd, void *buf, size_t len, int flags) = 0; 9 | virtual ssize_t write(int fd, void *buf, size_t count) = 0; 10 | }; 11 | 12 | class os_functions { 13 | public: 14 | os_functions(os_functions_interface&); 15 | ssize_t recv(int fd, void *buf, size_t len, int flags); 16 | ssize_t write(int fd, void *buf, size_t count); 17 | 18 | private: 19 | os_functions_interface & impl_; 20 | }; 21 | 22 | extern os_functions g_os_functions; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /include/external/rieman_tcp_client.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_RIEMANN_TCP_CLIENT_H 2 | #define CAVALIERI_EXTERNAL_RIEMANN_TCP_CLIENT_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | class riemann_tcp_client { 10 | public: 11 | riemann_tcp_client(const config conf); 12 | void push_event(const std::string host, const int port, const Event & event); 13 | 14 | private: 15 | const config config_; 16 | std::unordered_map> pool_map_; 18 | std::mutex mutex_; 19 | 20 | }; 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /include/external/mailer_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_MAILER_POOL_H 2 | #define CAVALIERI_MAILER_POOL_H 3 | 4 | #include 5 | 6 | class mailer_pool { 7 | public: 8 | mailer_pool(const size_t thread_num, const bool enable_debug); 9 | void push_event(const std::string server, const std::string from, 10 | const std::vector to, const Event & event); 11 | void stop(); 12 | 13 | private: 14 | void curl_event(const queued_event_t, 15 | const std::shared_ptr, 16 | std::function&); 17 | 18 | private: 19 | curl_pool curl_pool_; 20 | bool enable_debug_; 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /include/config/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_CONFIG_CONFIG_H 2 | #define CAVALIERI_CONFIG_CONFIG_H 3 | 4 | #include 5 | 6 | struct config { 7 | uint32_t events_port; 8 | size_t riemann_tcp_pool_size; 9 | uint32_t ws_port; 10 | size_t ws_pool_size; 11 | size_t executor_pool_size; 12 | uint64_t index_expire_interval; 13 | std::string rules_directory; 14 | size_t pagerduty_pool_size; 15 | size_t mail_pool_size; 16 | size_t graphite_pool_size; 17 | size_t forward_pool_size; 18 | bool enable_mail_debug; 19 | bool enable_pagerduty_debug; 20 | bool enable_internal_metrics; 21 | }; 22 | 23 | config create_config(); 24 | 25 | void log_config(config); 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /src/external/graphite.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | graphite::graphite(const config conf) : config_(conf) {} 4 | 5 | void graphite::push_event(const std::string host, const int port, 6 | const Event & event) 7 | { 8 | 9 | std::shared_ptr pool; 10 | 11 | { 12 | std::lock_guard lock(mutex_); 13 | auto it = pool_map_.find(host); 14 | if (it == pool_map_.end()) { 15 | pool = std::make_shared(config_.graphite_pool_size, 16 | host, port); 17 | pool_map_.insert({host, pool}); 18 | } else { 19 | pool = it->second; 20 | } 21 | } 22 | 23 | pool->push_event(event); 24 | 25 | } 26 | -------------------------------------------------------------------------------- /.cmake/Modules/FindGlog.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Glog, Google Log Library 2 | # Once done, this will define 3 | # 4 | # Glog_FOUND - system has Glog 5 | # Glog_INCLUDE_DIRS - the Glog include directories 6 | # Glog_LIBRARIES - link these to use Glog 7 | 8 | INCLUDE(LibFindMacros) 9 | 10 | FIND_PATH(Glog_INCLUDE_DIR 11 | NAMES glog/logging.h 12 | HINTS /usr/include 13 | ) 14 | 15 | FIND_LIBRARY(Glog_LIBRARY 16 | NAMES glog 17 | ) 18 | 19 | # Set the include dir variables and the libraries and let libfind_process do the rest. 20 | # NOTE: Singular variables for this library, plural for libraries this this lib depends on. 21 | set(Glog_PROCESS_INCLUDES Glog_INCLUDE_DIR) 22 | set(Glog_PROCESS_LIBS Glog_LIBRARY) 23 | 24 | LIBFIND_PROCESS(Glog) 25 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: cavalieri 2 | Section: universe/net 3 | Priority: extra 4 | Maintainer: Javier Uruen Val 5 | Build-Depends: debhelper (>= 8.0.0), cmake, subversion, protobuf-compiler, 6 | libprotobuf-dev, libev-dev, libgflags-dev, libgoogle-glog-dev, 7 | libcurl4-openssl-dev, libssl-dev, libtbb-dev, libjsoncpp-dev, lcov, 8 | flex, bison, libgoogle-glog-dev, libboost-filesystem-dev, libboost-system-dev 9 | Standards-Version: 3.9.4 10 | Homepage: http://github.com/juruen/cavalieri 11 | 12 | Package: cavalieri 13 | Architecture: any 14 | Depends: ${shlibs:Depends}, ${misc:Depends} 15 | Description: A C++ event stream processing tool to monitor and alert 16 | A C++ event stream processing tool to monitor and alert 17 | -------------------------------------------------------------------------------- /src/rules/util.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | target_t create_targets(const std::string pagerduty_key, const std::string to) { 7 | target_t target; 8 | 9 | auto pg_stream = (state("ok") >> pd_resolve(pagerduty_key), 10 | state("critical") >> pd_trigger(pagerduty_key)); 11 | 12 | auto mail_stream = email("localhost", "cavalieri@localhost", to); 13 | 14 | target.pagerduty = changed_state("ok") >> pg_stream; 15 | target.email = changed_state("ok") >> mail_stream; 16 | target.index = send_index(); 17 | target.all = (target.pagerduty, target.email, target.index); 18 | 19 | return target; 20 | } 21 | -------------------------------------------------------------------------------- /.cmake/Modules/FindLibEv.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find LibEv, Google Log Library 2 | # Once done, this will define 3 | # 4 | # LibEv_FOUND - system has LibEv 5 | # LibEv_INCLUDE_DIRS - the LibEv include directories 6 | # LibEv_LIBRARIES - link these to use LibEv 7 | 8 | INCLUDE(LibFindMacros) 9 | 10 | FIND_PATH(LibEv_INCLUDE_DIR 11 | NAMES ev++.h 12 | HINTS /usr/include 13 | ) 14 | 15 | FIND_LIBRARY(LibEv_LIBRARY 16 | NAMES ev 17 | ) 18 | 19 | # Set the include dir variables and the libraries and let libfind_process do the rest. 20 | # NOTE: Singular variables for this library, plural for libraries this this lib depends on. 21 | set(LibEv_PROCESS_INCLUDES LibEv_INCLUDE_DIR) 22 | set(LibEv_PROCESS_LIBS LibEv_LIBRARY) 23 | 24 | LIBFIND_PROCESS(LibEv) 25 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | # Sample debian/rules that uses debhelper. 4 | # 5 | # This file was originally written by Joey Hess and Craig Small. 6 | # As a special exception, when this file is copied by dh-make into a 7 | # dh-make output file, you may use that output file without restriction. 8 | # This special exception was added by Craig Small in version 0.37 of dh-make. 9 | # 10 | # Modified to make a template file for a multi-binary package with separated 11 | # build-arch and build-indep targets by Bill Allombert 2001 12 | 13 | # Uncomment this to turn on verbose mode. 14 | #export DH_VERBOSE=1 15 | 16 | # This has to be exported to make some magic below work. 17 | export DH_OPTIONS 18 | 19 | 20 | %: 21 | dh $@ 22 | -------------------------------------------------------------------------------- /include/core/core.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_CORE_CORE_H 2 | #define CAVALIERI_CORE_CORE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class core_interface { 11 | public: 12 | virtual void start() = 0; 13 | virtual void add_stream(std::shared_ptr stream) = 0; 14 | virtual void reload_rules() = 0; 15 | virtual index_interface & idx() = 0; 16 | virtual scheduler_interface & sched() = 0; 17 | virtual external_interface & externals() = 0; 18 | virtual ~core_interface() {} ; 19 | }; 20 | 21 | extern std::shared_ptr g_core; 22 | 23 | void start_core(int argv, char **argc); 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /include/external/pagerduty_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_PAGERDUTY_POOL_H 2 | #define CAVALIERI_PAGERDUTY_POOL_H 3 | 4 | #include 5 | 6 | class pagerduty_pool { 7 | public: 8 | enum class pd_action { 9 | trigger, 10 | acknowledge, 11 | resolve 12 | }; 13 | 14 | public: 15 | pagerduty_pool(const size_t thread_num, const bool enable_debug); 16 | void push_event(const pd_action action, const std::string pd_key, 17 | const Event & event); 18 | void stop(); 19 | 20 | private: 21 | void curl_event(const queued_event_t, const std::shared_ptr, 22 | std::function& clean_fn); 23 | 24 | private: 25 | curl_pool curl_pool_; 26 | bool enable_debug_; 27 | }; 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/instrumentation/rate.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | rate::rate() : counter_(0), start_(std::chrono::high_resolution_clock::now()) {} 4 | 5 | 6 | 7 | void rate::add(unsigned int ticks) { 8 | counter_ += ticks; 9 | } 10 | 11 | double rate::snapshot() { 12 | 13 | auto counter = counter_.exchange(0); 14 | auto now = std::chrono::high_resolution_clock::now(); 15 | 16 | auto dt = std::chrono::duration_cast(now -start_); 17 | auto rate = static_cast(counter) / static_cast(dt.count()); 18 | 19 | start_ = now; 20 | 21 | return rate * 1000; 22 | 23 | } 24 | 25 | void rate::reset() { 26 | 27 | counter_.exchange(0); 28 | start_ = std::chrono::high_resolution_clock::now(); 29 | 30 | 31 | } 32 | -------------------------------------------------------------------------------- /include/transport/tcp_connection.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_TCP_CONNECTION_H 2 | #define CAVALIERI_TRANSPORT_TCP_CONNECTION_H 3 | 4 | #include 5 | #include 6 | 7 | class tcp_connection { 8 | public: 9 | tcp_connection(int socket_fd); 10 | tcp_connection(int socket_fd, size_t buff_size); 11 | 12 | bool read(); 13 | bool queue_write(const char *src, size_t length); 14 | bool write(); 15 | bool pending_write() const; 16 | bool pending_read() const; 17 | size_t read_bytes() const; 18 | 19 | const size_t buff_size; 20 | int sfd; 21 | bool close_connection; 22 | boost::circular_buffer r_buffer; 23 | boost::circular_buffer w_buffer; 24 | 25 | }; 26 | 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/external/riemann_tcp_client.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | riemann_tcp_client::riemann_tcp_client(const config conf) : config_(conf) { } 5 | 6 | void riemann_tcp_client::push_event(const std::string host, const int port, 7 | const Event & event) 8 | { 9 | std::shared_ptr pool; 10 | 11 | { 12 | std::lock_guard lock(mutex_); 13 | auto it = pool_map_.find(host); 14 | if (it == pool_map_.end()) { 15 | pool = std::make_shared( 16 | config_.graphite_pool_size, host, port); 17 | pool_map_.insert({host, pool}); 18 | } else { 19 | pool = it->second; 20 | } 21 | } 22 | 23 | pool->push_event(event); 24 | } 25 | -------------------------------------------------------------------------------- /include/pub_sub/pub_sub.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_PUB_SUB_H 2 | #define CAVALIERI_PUB_SUB_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | typedef std::function notify_event_fn_t; 13 | typedef std::unordered_map publishers_t; 14 | 15 | 16 | class pub_sub { 17 | public: 18 | 19 | void add_publisher(const std::string & topic); 20 | 21 | void publish(const std::string & topic, const Event & event); 22 | 23 | void subscribe(const std::string & topic, 24 | const notify_event_fn_t notify_event); 25 | 26 | private: 27 | publishers_t publishers_; 28 | }; 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /ci/ws-test/test-ws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import multiprocessing 4 | import subprocess 5 | import sys 6 | import time 7 | 8 | client_processes = 5 9 | ws_processes = 20; 10 | total_processes = client_processes + ws_processes 11 | 12 | events = "5000" 13 | ws_client = "./ws-read-events.py" 14 | riemann_client = "./generate-events.py" 15 | 16 | def work(i): 17 | if i < client_processes: 18 | return subprocess.call([riemann_client], shell=False) 19 | else: 20 | time.sleep(10) 21 | return subprocess.call([ws_client, "localhost:5556", events], 22 | shell=False) 23 | 24 | pool = multiprocessing.Pool(processes=total_processes) 25 | result = pool.map(work, range(total_processes)) 26 | 27 | print result 28 | 29 | sys.exit(len([i for i in result if i > 0])) 30 | -------------------------------------------------------------------------------- /thirdparty/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Enable ExternalProject CMake module 2 | include(ExternalProject) 3 | 4 | # Add gmock 5 | ExternalProject_Add( 6 | googlemock 7 | SVN_REPOSITORY http://googlemock.googlecode.com/svn/tags/release-1.7.0 8 | UPDATE_COMMAND "" 9 | TIMEOUT 10 10 | CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} 11 | INSTALL_COMMAND "" 12 | LOG_DOWNLOAD ON 13 | LOG_CONFIGURE ON 14 | LOG_BUILD ON) 15 | 16 | set (ThirdParty_INCLUDE_DIRS 17 | ${CMAKE_BINARY_DIR}/thirdparty/googlemock-prefix/src/googlemock/gtest/include 18 | ${CMAKE_BINARY_DIR}/thirdparty/googlemock-prefix/src/googlemock/include 19 | PARENT_SCOPE) 20 | 21 | set (ThirdParty_LIBRARIES 22 | ${CMAKE_BINARY_DIR}/thirdparty/googlemock-prefix/src/googlemock-build/libgmock.a 23 | PARENT_SCOPE) 24 | -------------------------------------------------------------------------------- /include/transport/udp_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_UDP_POOL_H 2 | #define CAVALIERI_TRANSPORT_UDP_POOL_H 3 | 4 | #include 5 | #include 6 | 7 | typedef std::vector udp_buffer_t; 8 | typedef std::function udp_read_fn_t; 9 | 10 | class udp_pool { 11 | public: 12 | udp_pool( 13 | size_t thread_num, 14 | uint32_t port, 15 | udp_read_fn_t udp_ready_fn_t 16 | ); 17 | void start_threads(); 18 | void stop_threads(); 19 | virtual ~udp_pool(); 20 | 21 | private: 22 | void run_hook(async_loop & loop); 23 | void socket_callback(async_fd & async); 24 | 25 | private: 26 | async_thread_pool async_thread_pool_; 27 | uint32_t port_; 28 | udp_read_fn_t udp_read_fn_; 29 | }; 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /ci/ws-test/ws-read-events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import websocket 4 | import urllib 5 | import json 6 | import sys 7 | 8 | 9 | def ws_url(host): 10 | ws_host = "ws://%s/index?" % host 11 | q = {'subscribe' : 'true', 'query' : '(metric>0.5)'} 12 | return ws_host + urllib.urlencode(q) 13 | 14 | def main(): 15 | if len(sys.argv) < 3: 16 | sys.stderr.write("usage %s: host num_events\n" % sys.argv[0]) 17 | sys.exit(1) 18 | 19 | ws = websocket.create_connection(ws_url(sys.argv[1])) 20 | 21 | events_read = 0 22 | try: 23 | for i in range(int(sys.argv[2])): 24 | ws.recv() 25 | events_read += 1 26 | except: 27 | print "events_read: %i" % events_read 28 | sys.exit(1) 29 | 30 | ws.close() 31 | 32 | sys.exit(0) 33 | 34 | main() 35 | -------------------------------------------------------------------------------- /include/core/mock_core.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_CORE_MOCK_CORE_H 2 | #define CAVALIERI_CORE_MOCK_CORE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | class mock_core : public core_interface { 10 | public: 11 | mock_core(); 12 | 13 | void start(); 14 | 15 | void add_stream(std::shared_ptr stream); 16 | void reload_rules(); 17 | 18 | index_interface & idx(); 19 | scheduler_interface & sched(); 20 | external_interface & externals(); 21 | 22 | mock_external & mock_external_impl(); 23 | mock_index & mock_index_impl(); 24 | 25 | private: 26 | std::unique_ptr sched_; 27 | std::unique_ptr mock_index_; 28 | std::unique_ptr externals_; 29 | }; 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /include/query/driver.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_QUERY_DRIVER_H 2 | #define CAVALIERI_QUERY_DRIVER_H 3 | 4 | #include 5 | #include 6 | 7 | class query_context; 8 | 9 | namespace queryparser { 10 | 11 | class driver 12 | { 13 | public: 14 | driver(query_context & query); 15 | 16 | bool trace_scanning; 17 | 18 | bool trace_parsing; 19 | 20 | std::string streamname; 21 | 22 | bool parse_stream(std::istream& in, 23 | const std::string& sname = "stream input"); 24 | 25 | bool parse_string(const std::string& input, 26 | const std::string& sname = "string stream"); 27 | 28 | void error(const class location& l, const std::string& m); 29 | 30 | void error(const std::string& m); 31 | 32 | class Scanner* lexer; 33 | 34 | class query_context & query; 35 | }; 36 | 37 | } 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /tests/pubsub_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef PUBSUB_TEST_CASE 2 | #define PUBSUB_TEST_CASE 3 | 4 | #include 5 | 6 | TEST(pubsub_test_case, test) 7 | { 8 | pub_sub pubsub; 9 | 10 | pubsub.add_publisher("topic1"); 11 | pubsub.add_publisher("topic2"); 12 | 13 | std::vector vec1; 14 | std::vector vec2; 15 | 16 | pubsub.subscribe("topic1", [&](const Event & e){ vec1.push_back(e); }); 17 | pubsub.subscribe("topic2", [&](const Event & e){ vec2.push_back(e); }); 18 | 19 | Event e; 20 | 21 | e.set_host("topic1 1"); 22 | pubsub.publish("topic1", e); 23 | 24 | e.set_host("topic2 1"); 25 | pubsub.publish("topic2", e); 26 | 27 | ASSERT_EQ(1u, vec1.size()); 28 | ASSERT_EQ(1u, vec2.size()); 29 | 30 | ASSERT_EQ("topic1 1", vec1[0].host()); 31 | ASSERT_EQ("topic2 1", vec2[0].host()); 32 | 33 | } 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /.cmake/Modules/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | FIND_PATH(GFLAGS_INCLUDE_DIR gflags/gflags.h 2 | /usr/local/include 3 | /usr/include 4 | /opt/local/include 5 | ) 6 | 7 | FIND_LIBRARY(GFLAGS_LIBRARY NAMES gflags) 8 | 9 | INCLUDE(FindPackageHandleStandardArgs) 10 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(GFLAGS DEFAULT_MESSAGE GFLAGS_INCLUDE_DIR) 11 | 12 | IF(GFLAGS_FOUND) 13 | SET( GFLAGS_LIBRARIES ${GFLAGS_LIBRARY} ) 14 | IF(NOT GFLAGS_FIND_QUIETLY) 15 | MESSAGE(STATUS "Found GFLAGS (GFLAGS_LIBRARIES = ${GFLAGS_LIBRARIES})") 16 | MESSAGE(STATUS "Found GFLAGS (GFLAGS_INCLUDE_DIR = ${GFLAGS_INCLUDE_DIR})") 17 | ENDIF(NOT GFLAGS_FIND_QUIETLY) 18 | ELSE(GFLAGS_FOUND) 19 | IF(GFLAGS_FIND_REQUIRED) 20 | MESSAGE(FATAL_ERROR "Could not find GFLAGS") 21 | ENDIF(GFLAGS_FIND_REQUIRED) 22 | ENDIF(GFLAGS_FOUND) 23 | 24 | MARK_AS_ADVANCED(GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 25 | 26 | -------------------------------------------------------------------------------- /ci/graphite-test/send-events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import bernhard 4 | import datetime 5 | import sys 6 | import random 7 | 8 | batch_size = 1 9 | 10 | class BatchClient(bernhard.Client): 11 | def send(self, events): 12 | message = bernhard.Message(events=[bernhard.Event(params=e) for e in events]) 13 | response = self.transmit(message) 14 | return response.ok 15 | 16 | def create_event(i): 17 | global services 18 | return { 19 | 'host': "bar.com", 20 | 'service': "foo-%i" %i, 21 | 'metric': random.random(), 22 | 'description': str(datetime.datetime.now()), 23 | 'tags': ["stress-test"], 24 | 'attributes' : { 'foo' : 'bar'} 25 | } 26 | 27 | c = BatchClient(host='localhost') 28 | 29 | for i in range(0, 1000): 30 | event = create_event(i) 31 | c.send([event]) 32 | -------------------------------------------------------------------------------- /src/pub_sub/pub_sub.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void pub_sub::add_publisher(const std::string & topic) 5 | { 6 | VLOG(3) << "add_publisher() topic: " << topic; 7 | 8 | publishers_.insert({topic, {}}); 9 | } 10 | 11 | void pub_sub::publish(const std::string & topic, const Event & event) { 12 | 13 | VLOG(3) << "publish() topic: " << topic; 14 | 15 | auto it = publishers_.find(topic); 16 | 17 | CHECK(it != end(publishers_)) << "topic not found"; 18 | 19 | it->second(event); 20 | 21 | } 22 | 23 | void pub_sub::subscribe(const std::string & topic, 24 | const notify_event_fn_t notify_fn) 25 | { 26 | VLOG(3) << "subscribe() topic: " << topic; 27 | 28 | auto it = publishers_.find(topic); 29 | 30 | CHECK(it != end(publishers_)) << "topic not found"; 31 | 32 | it->second = notify_fn; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /include/riemann_tcp_connection.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RIEMANN_TCP_CONNECTION_H 2 | #define CAVALIERI_RIEMANN_TCP_CONNECTION_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | typedef std::function)> raw_msg_fn_t; 11 | 12 | class riemann_tcp_connection { 13 | public: 14 | riemann_tcp_connection( 15 | tcp_connection & tcp_connection_, 16 | raw_msg_fn_t raw_msg_fn 17 | ); 18 | void callback(async_fd &); 19 | 20 | private: 21 | void read_cb(); 22 | void write_cb(); 23 | void read_header(); 24 | void read_message(); 25 | 26 | private: 27 | tcp_connection & tcp_connection_; 28 | raw_msg_fn_t raw_msg_fn_; 29 | bool reading_header_; 30 | size_t protobuf_size_; 31 | }; 32 | 33 | #endif 34 | 35 | -------------------------------------------------------------------------------- /ci/email-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cleanup() { 6 | pkill cavalieri || true 7 | pkill send-events.py || true 8 | 9 | rm -rf cavalieri-rules || true 10 | 11 | sudo sh -c "cat > /var/mail/ubuntu" 12 | } 13 | 14 | git clone https://github.com/juruen/cavalieri-rules.git 15 | 16 | cp rules-mail.cpp cavalieri-rules/rules.cpp 17 | cd cavalieri-rules && mkdir build && cd build && cmake .. && make 18 | 19 | cavalieri -v 0 -rules_directory . >/dev/null 2>&1 & 20 | 21 | cd ../.. 22 | 23 | sleep 5 24 | 25 | ./send-events.py 500 & 26 | 27 | for i in $(seq 0 120); do 28 | 29 | EMAILS=$(sudo sh -c 'grep "^To:" /var/mail/ubuntu | wc -l') || true 30 | 31 | echo "EMAILS: ${EMAILS}" 32 | 33 | if [ "$EMAILS" == "500" ]; then 34 | cleanup 35 | exit 0 36 | fi 37 | 38 | sleep 1 39 | 40 | done 41 | 42 | echo "email test failed" 43 | 44 | cleanup 45 | 46 | exit 1 47 | -------------------------------------------------------------------------------- /include/riemann_tcp_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RIEMANN_TCP_POOL 2 | #define CAVALIERI_RIEMANN_TCP_POOL 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class riemann_tcp_pool { 9 | public: 10 | riemann_tcp_pool(size_t thread_num, raw_msg_fn_t raw_msg_fn, 11 | instrumentation::instrumentation & instr); 12 | void add_client (int fd); 13 | void stop(); 14 | ~riemann_tcp_pool(); 15 | 16 | private: 17 | void create_conn(int fd, async_loop & loop, tcp_connection & conn); 18 | void data_ready(async_fd & async, tcp_connection & conn); 19 | 20 | private: 21 | tcp_pool tcp_pool_; 22 | raw_msg_fn_t raw_msg_fn_; 23 | instrumentation::update_gauge_t connection_gauge_; 24 | std::vector> connections_; 25 | }; 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /ci/ws-test/generate-events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import bernhard 4 | import datetime 5 | import sys 6 | import os 7 | import random 8 | 9 | batch_size = 1000 10 | 11 | class BatchClient(bernhard.Client): 12 | def send(self, events): 13 | message = bernhard.Message(events=[bernhard.Event(params=e) for e in events]) 14 | response = self.transmit(message) 15 | return response.ok 16 | 17 | def create_event(i): 18 | global services 19 | return { 20 | 'host': "foo%i-%i.com" % (i, os.getpid()) , 21 | 'service': "requests_rate", 22 | 'metric': 100, 23 | 'description': str(datetime.datetime.now()), 24 | 'tags': ["stress-test"], 25 | 'ttl': 60, 26 | 'attributes' : { 'foo' : 'bar'} 27 | } 28 | 29 | c = BatchClient(host='localhost') 30 | 31 | c.send([create_event(i) for i in range(batch_size)]) 32 | 33 | sys.exit(0) 34 | -------------------------------------------------------------------------------- /src/core/mock_core.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | mock_core::mock_core() 6 | : 7 | sched_(new mock_scheduler()), 8 | mock_index_(new mock_index()), 9 | externals_(new mock_external()) 10 | { 11 | } 12 | 13 | void mock_core::start() { } 14 | 15 | void mock_core::add_stream(std::shared_ptr) {} 16 | 17 | void mock_core::reload_rules() {} 18 | 19 | class index_interface & mock_core::idx() { 20 | return *mock_index_; 21 | } 22 | 23 | mock_index & mock_core::mock_index_impl() { 24 | return *mock_index_; 25 | } 26 | 27 | scheduler_interface & mock_core::sched() { 28 | return *sched_; 29 | } 30 | 31 | external_interface & mock_core::externals() { 32 | return *externals_; 33 | } 34 | 35 | mock_external & mock_core::mock_external_impl() { 36 | return *externals_; 37 | } 38 | 39 | void start_core(int, char**) { } 40 | -------------------------------------------------------------------------------- /src/query/scanner.h: -------------------------------------------------------------------------------- 1 | #ifndef SCANNER_H 2 | #define SCANNER_H 3 | 4 | #ifndef YY_DECL 5 | 6 | #define YY_DECL \ 7 | queryparser::Parser::token_type \ 8 | queryparser::Scanner::lex( \ 9 | queryparser::Parser::semantic_type* yylval, \ 10 | queryparser::Parser::location_type* yylloc \ 11 | ) 12 | #endif 13 | 14 | #ifndef __FLEX_LEXER_H 15 | #define yyFlexLexer QueryparserFlexLexer 16 | #include "FlexLexer.h" 17 | #undef yyFlexLexer 18 | #endif 19 | 20 | #include "parser.h" 21 | 22 | namespace queryparser { 23 | 24 | class Scanner : public QueryparserFlexLexer 25 | { 26 | public: 27 | Scanner(std::istream* arg_yyin = 0, 28 | std::ostream* arg_yyout = 0); 29 | 30 | virtual ~Scanner(); 31 | 32 | virtual Parser::token_type lex( 33 | Parser::semantic_type* yylval, 34 | Parser::location_type* yylloc 35 | ); 36 | void set_debug(bool b); 37 | }; 38 | 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /ci/email-test/send-events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import bernhard 4 | import datetime 5 | import sys 6 | import random 7 | 8 | services = ["test1," "test2", "test3", "foo", "bar", "baz", "xyzzy", "attack", "cat", "treat"] 9 | 10 | class BatchClient(bernhard.Client): 11 | def send(self, events): 12 | message = bernhard.Message(events=[bernhard.Event(params=e) for e in events]) 13 | response = self.transmit(message) 14 | return response.ok 15 | 16 | def create_event(i): 17 | global services 18 | return { 19 | 'host': "host%i" %i , 20 | 'service': services[i % len(services)], 21 | 'metric': random.random(), 22 | 'description': str(datetime.datetime.now()), 23 | 'tags': ["stress-test"], 24 | 'attributes' : { 'foo' : 'bar'} 25 | } 26 | 27 | c = BatchClient(host='localhost') 28 | 29 | c.send([create_event(i) for i in range(int(sys.argv[1]))]) 30 | -------------------------------------------------------------------------------- /include/external/mock_external.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_MOCK_EXTERNAL_H 2 | #define CAVALIERI_EXTERNAL_MOCK_EXTERNAL_H 3 | 4 | #include 5 | #include 6 | 7 | class mock_external : public external_interface { 8 | public: 9 | void forward(const std::string server, const int port, const Event event); 10 | void graphite(const std::string server, const int port, const Event event); 11 | void pager_duty_trigger(const std::string pg_key, const Event event); 12 | void pager_duty_resolve(const std::string pg_key, const Event event); 13 | void pager_duty_acknowledge(const std::string pg_key, const Event event); 14 | void email(const std::string server, const std::string from, 15 | const std::vector to, const Event event); 16 | 17 | std::vector calls() const; 18 | 19 | private: 20 | std::vector calls_; 21 | 22 | }; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /extra/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import bernhard 4 | import datetime 5 | import sys 6 | import random 7 | 8 | batch_size = 100 9 | services = ["test1," "test2", "test3", "foo", "bar", "baz", "xyzzy", "attack", "cat", "treat"] 10 | 11 | class BatchClient(bernhard.Client): 12 | def send(self, events): 13 | message = bernhard.Message(events=[bernhard.Event(params=e) for e in events]) 14 | response = self.transmit(message) 15 | return response.ok 16 | 17 | def create_event(i): 18 | global services 19 | return { 20 | 'host': "host%i" %i , 21 | 'service': services[i % len(services)], 22 | 'metric': random.random(), 23 | 'description': str(datetime.datetime.now()), 24 | 'tags': ["stress-test"], 25 | 'attributes' : { 'foo' : 'bar'} 26 | } 27 | 28 | c = BatchClient(host='hw3578.spotify.net',port=15555) 29 | 30 | c.send([create_event(i) for i in range(batch_size)]) 31 | -------------------------------------------------------------------------------- /include/scheduler/scheduler.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_SCHEDULER_SCHEDULER_H 2 | #define CAVALIERI_SCHEDULER_SCHEDULER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using task_fn_t = std::function; 10 | using remove_task_fn_t = std::function; 11 | using remove_task_future_t = std::future; 12 | using remove_ns_tasks_future_t = std::future; 13 | 14 | class scheduler_interface { 15 | public: 16 | virtual remove_task_future_t add_periodic_task(task_fn_t task, 17 | float interval) = 0; 18 | virtual remove_task_future_t add_once_task(task_fn_t task, float dt) = 0; 19 | virtual remove_ns_tasks_future_t remove_ns_tasks(const std::string nm) = 0; 20 | virtual time_t unix_time() = 0; 21 | virtual void set_time(const time_t t) = 0; 22 | virtual void clear() = 0; 23 | virtual ~scheduler_interface() {}; 24 | }; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /extra/client-udp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import bernhard 4 | import datetime 5 | import sys 6 | import random 7 | 8 | batch_size = 1 9 | services = ["test1," "test2", "test3", "foo", "bar", "baz", "xyzzy", "attack", "cat", "treat"] 10 | 11 | class BatchClient(bernhard.Client): 12 | def send(self, events): 13 | message = bernhard.Message(events=[bernhard.Event(params=e) for e in events]) 14 | response = self.transmit(message) 15 | return response.ok 16 | 17 | def create_event(i): 18 | global services 19 | return { 20 | 'host': "host%i" %i , 21 | 'service': services[i % len(services)], 22 | 'metric': random.random(), 23 | 'description': str(datetime.datetime.now()), 24 | 'tags': ["stress-test"], 25 | 'attributes' : { 'foo' : 'bar'} 26 | } 27 | 28 | c = BatchClient(host='localhost', transport=bernhard.UDPTransport) 29 | 30 | #while True: 31 | c.send([create_event(i) for i in range(batch_size)]) 32 | -------------------------------------------------------------------------------- /include/websocket/worker_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_WEBSOCKET_WORKER_POOL_H 2 | #define CAVALIERI_WEBSOCKET_WORKER_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | class worker_pool { 11 | public: 12 | worker_pool(); 13 | worker_pool(worker_pool const&) = delete; 14 | void add_event(const Event & event); 15 | void update_filters(const size_t id, const event_filters_t); 16 | void stop(); 17 | 18 | public: 19 | using event_t = struct { 20 | Event event; 21 | bool stop; 22 | }; 23 | 24 | using event_queue_t = tbb::concurrent_bounded_queue; 25 | 26 | using filters_map_t = std::unordered_map; 27 | 28 | private: 29 | 30 | bool stop_, has_clients_; 31 | filters_map_t filters_; 32 | event_queue_t event_queue_; 33 | std::mutex mutex_; 34 | worker_threads worker_threads_; 35 | 36 | }; 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /.cmake/Modules/FindTBB.cmake: -------------------------------------------------------------------------------- 1 | FIND_PATH(TBB_INCLUDE_DIR tbb/tbb.h 2 | /usr/local/include 3 | /usr/include 4 | /opt/local/include 5 | ) 6 | 7 | FIND_LIBRARY(TBB_CORE_LIBRARY NAMES tbb) 8 | FIND_LIBRARY(TBB_MALLOC_LIBRARY NAMES tbbmalloc) 9 | 10 | # handle the QUIETLY and REQUIRED arguments and set TBB_FOUND to TRUE if 11 | # all listed variables are TRUE 12 | INCLUDE(FindPackageHandleStandardArgs) 13 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(TBB DEFAULT_MSG TBB_CORE_LIBRARY TBB_INCLUDE_DIR) 14 | 15 | IF(TBB_FOUND) 16 | SET( TBB_LIBRARIES ${TBB_CORE_LIBRARY} ${TBB_MALLOC_LIBRARY} ) 17 | IF(NOT TBB_FIND_QUIETLY) 18 | MESSAGE(STATUS "Found TBB (TBB_LIBRARIES = ${TBB_LIBRARIES})") 19 | MESSAGE(STATUS "Found TBB (TBB_INCLUDE_DIR = ${TBB_INCLUDE_DIR})") 20 | ENDIF(NOT TBB_FIND_QUIETLY) 21 | ELSE(TBB_FOUND) 22 | IF(TBB_FIND_REQUIRED) 23 | MESSAGE(FATAL_ERROR "Could not find TBB") 24 | ENDIF(TBB_FIND_REQUIRED) 25 | ENDIF(TBB_FOUND) 26 | 27 | MARK_AS_ADVANCED(TBB_INCLUDE_DIR TBB_LIBRARY) 28 | -------------------------------------------------------------------------------- /include/pool/executor_thread_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_POOL_EXECUTOR_THREAD_POOL_H 2 | #define CAVALIERI_POOL_EXECUTOR_THREAD_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | typedef std::function task_fn_t; 12 | 13 | class executor_thread_pool { 14 | public: 15 | executor_thread_pool(instrumentation::instrumentation & instr, 16 | const config & conf); 17 | void add_task(const task_fn_t & task); 18 | void stop(); 19 | 20 | private: 21 | void run_tasks(const int i); 22 | 23 | private: 24 | typedef struct { 25 | task_fn_t fn; 26 | bool stop; 27 | } task_t; 28 | 29 | instrumentation::update_gauge_t task_guague_; 30 | std::vector finished_threads_; 31 | std::vector threads_; 32 | size_t next_thread_; 33 | std::vector> tasks_; 34 | 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /include/streams/stream_functions_lock.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_STREAMS_STREAM_FUNCTIONS_LOCK_H 2 | #define CAVALIERI_STREAMS_STREAM_FUNCTIONS_LOCK_H 3 | 4 | #include 5 | 6 | streams_t by_lock(const by_keys_t & keys, const streams_t stream); 7 | 8 | streams_t by_lock(const by_keys_t & keys); 9 | 10 | streams_t coalesce_lock(fold_fn_t fold); 11 | 12 | streams_t project_lock(const predicates_t predicates, fold_fn_t fold); 13 | 14 | streams_t changed_state_lock(std::string initial); 15 | 16 | streams_t moving_event_window_lock(size_t n, fold_fn_t fold); 17 | 18 | streams_t fixed_event_window_lock(size_t n, fold_fn_t fold); 19 | 20 | streams_t moving_time_window_lock(time_t dt, fold_fn_t fold); 21 | 22 | streams_t fixed_time_window_lock(time_t dt, fold_fn_t fold); 23 | 24 | streams_t stable_lock(time_t dt); 25 | 26 | streams_t throttle_lock(size_t n, time_t dt); 27 | 28 | streams_t percentiles_lock(time_t interval, std::vector percentiles); 29 | 30 | streams_t ddt_lock(); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /include/streams/lib.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_STREAM_LIB_H 2 | #define CAVALIERI_STREAM_LIB_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | const std::string k_global_ns("global"); 11 | 12 | /* This thread_local variable is used to track the lib namespace that is using 13 | * the thread. It is useful to unload the library completely and remove 14 | * all code that belongs to the library 15 | */ 16 | extern thread_local std::string * thread_ns; 17 | 18 | void free_thread_ns(); 19 | 20 | void set_thread_ns(const std::string ns); 21 | 22 | void set_thread_global_ns(); 23 | 24 | std::string get_thread_ns(); 25 | 26 | struct stream_lib { 27 | 28 | std::string file; 29 | std::time_t last_write_time; 30 | void * handle{nullptr}; 31 | std::shared_ptr stream; 32 | std::atomic ref_counter{0}; 33 | 34 | bool used() const; 35 | void set_used(bool); 36 | bool inc(); 37 | void dec(); 38 | 39 | }; 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /include/transport/ws_connection.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_WS_CONNECTION_H 2 | #define CAVALIERI_TRANSPORT_WS_CONNECTION_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class ws_connection { 9 | public: 10 | static const uint32_t k_read_http_header = 0x1; 11 | static const uint32_t k_write_http_header = 0x2; 12 | static const uint32_t k_read_frame_header = 0x4; 13 | static const uint32_t k_write_frame = 0x08; 14 | 15 | ws_connection(tcp_connection & tcp_connection); 16 | bool send_frame(const std::string & payload); 17 | uint32_t state() const; 18 | std::string uri() const; 19 | void callback(async_fd &); 20 | 21 | private: 22 | void read_cb(); 23 | void write_cb(); 24 | void read_header(); 25 | void read_frame(); 26 | void write_response_header(); 27 | 28 | private: 29 | tcp_connection & tcp_connection_; 30 | ws_util ws_util_; 31 | uint32_t state_; 32 | }; 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /include/transport/ws_util.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_WS_UTIL_H 2 | #define CAVALIERI_TRANSPORT_WS_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | class ws_util { 10 | public: 11 | 12 | struct header_values { 13 | std::string method; 14 | std::string uri; 15 | std::string version; 16 | std::unordered_map headers; 17 | 18 | } header_vals; 19 | 20 | typedef struct { 21 | bool malformed_header; 22 | bool pending_bytes; 23 | size_t header_length; 24 | uint64_t payload_length; 25 | std::string decoded; 26 | } frame_length_t; 27 | 28 | bool find_header_end(const std::string& header); 29 | 30 | bool parse_header(const std::string& header); 31 | 32 | std::pair make_header_response(); 33 | 34 | std::string create_frame(const std::string payload); 35 | 36 | frame_length_t decode_frame(const std::vector, size_t); 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/query/driver.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | #include "driver.h" 6 | #include "scanner.h" 7 | 8 | namespace queryparser { 9 | 10 | driver::driver(class query_context& query) 11 | : trace_scanning(false), 12 | trace_parsing(false), 13 | query(query) 14 | { 15 | } 16 | 17 | bool driver::parse_stream(std::istream& in, const std::string& sname) 18 | { 19 | streamname = sname; 20 | 21 | Scanner scanner(&in); 22 | scanner.set_debug(trace_scanning); 23 | this->lexer = &scanner; 24 | 25 | Parser parser(*this); 26 | parser.set_debug_level(trace_parsing); 27 | return (parser.parse() == 0); 28 | } 29 | 30 | 31 | bool driver::parse_string(const std::string &input, const std::string& sname) 32 | { 33 | std::istringstream iss(input); 34 | return parse_stream(iss, sname); 35 | } 36 | 37 | void driver::error(const class location& l, 38 | const std::string& m) 39 | { 40 | std::cerr << l << ": " << m << std::endl; 41 | } 42 | 43 | void driver::error(const std::string& m) 44 | { 45 | std::cerr << m << std::endl; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /include/core/real_core_helper.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_CORE_REAL_CORE_HELPER_H 2 | #define CAVALIERI_CORE_REAL_CORE_HELPER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | void detach_thread(std::function fn); 9 | 10 | std::shared_ptr make_real_core(const config conf); 11 | 12 | std::unique_ptr init_tcp_server( 13 | const config & conf, 14 | main_async_loop_interface & loop, 15 | streams & streams, 16 | executor_thread_pool & executor_pool, 17 | instrumentation::instrumentation & instr); 18 | 19 | std::unique_ptr init_udp_server( 20 | const config & conf, 21 | std::shared_ptr streams); 22 | 23 | std::unique_ptr init_ws_server( 24 | const config & conf, 25 | main_async_loop_interface & loop, 26 | pub_sub & pubsub, 27 | real_index & index); 28 | 29 | void start_instrumentation(scheduler_interface & sched, 30 | instrumentation::instrumentation & instrumentation, 31 | streams & streams); 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /tests/test_plan.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "gtest/gtest.h" 3 | #include "gmock/gmock.h" 4 | 5 | #include "basic_test_case.h" 6 | #include "query_grammar_test_case.h" 7 | #include "mock_scheduler_test_case.h" 8 | #include "streams_test_case.h" 9 | #include "folds_test_case.h" 10 | #include "rules_common_test_case.h" 11 | #include "tcp_connection_test_case.h" 12 | #include "ws_connection_test_case.h" 13 | #include "riemann_tcp_connection_test_case.h" 14 | #include "pubsub_test_case.h" 15 | #include "index_test_case.h" 16 | #include 17 | #include 18 | #include "os/os_functions.h" 19 | #include "os/mock_os_functions.h" 20 | 21 | mock_os_functions mock_os; 22 | os_functions g_os_functions(mock_os); 23 | 24 | int main(int argc, char **argv) 25 | { 26 | int ret; 27 | 28 | ::testing::InitGoogleTest(&argc, argv); 29 | google::InitGoogleLogging(argv[0]); 30 | 31 | ::testing::InitGoogleMock(&argc, argv); 32 | 33 | auto m_core = std::make_shared(); 34 | g_core = std::dynamic_pointer_cast(m_core); 35 | 36 | ret = RUN_ALL_TESTS(); 37 | 38 | return ret; 39 | } 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Javier Uruen Val 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /ci/email-test/run-memory-check-stop-postfix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cleanup() { 6 | pkill send-events.py || true 7 | 8 | rm -rf cavalieri-rules || true 9 | 10 | sudo sh -c "cat > /var/mail/ubuntu" 11 | 12 | } 13 | 14 | git clone https://github.com/juruen/cavalieri-rules.git 15 | 16 | cp rules-mail.cpp cavalieri-rules/rules.cpp 17 | cd cavalieri-rules && mkdir build && cd build && cmake .. && make 18 | 19 | sudo /etc/init.d/postfix stop || true 20 | 21 | export LD_LIBRARY_PATH=$(pwd) 22 | 23 | valgrind --error-exitcode=1 --show-possibly-lost=no \ 24 | --workaround-gcc296-bugs=yes --log-file=../../valgrind.out \ 25 | cavalieri -v 0 -rules_directory . -index_expire_interval 10000 & 26 | 27 | valgrind_pid=$! 28 | 29 | cd ../.. 30 | 31 | sleep 5 32 | 33 | ./send-events.py 100 & 34 | 35 | sleep 30 36 | 37 | kill -INT $valgrind_pid 38 | 39 | if ! wait $valgrind_pid; then 40 | 41 | echo "valgrind reported an erro" 42 | cat valgrind.out 43 | 44 | exit 1 45 | 46 | fi 47 | 48 | grep -q "definitely lost: 0 bytes" valgrind.out \ 49 | || (cat valgrind.out && false) 50 | 51 | cleanup 52 | 53 | exit 0 54 | -------------------------------------------------------------------------------- /include/pool/async_thread_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_POOL_THREAD_POOL_H 2 | #define CAVALIERI_POOL_THREAD_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | typedef std::function hook_fn_t; 11 | 12 | class async_thread_pool { 13 | public: 14 | async_thread_pool(size_t thread_num); 15 | void set_run_hook(hook_fn_t); 16 | void set_async_hook(hook_fn_t); 17 | void start_threads(); 18 | void stop_threads(); 19 | void signal_thread(size_t tid); 20 | size_t next_thread(); 21 | async_loop & loop(const size_t id); 22 | virtual ~async_thread_pool(); 23 | 24 | private: 25 | virtual void run(const size_t thread_id); 26 | void async_callback(async_loop&); 27 | 28 | protected: 29 | bool stop_; 30 | size_t thread_num_; 31 | size_t next_thread_; 32 | std::vector threads_; 33 | std::vector finished_threads_; 34 | std::unique_ptr async_events_; 35 | hook_fn_t run_hook_fn_; 36 | hook_fn_t async_hook_fn_; 37 | std::mutex mutex_; 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /include/streams/stream_infra.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_STREAMS_STREAM_INFRA_H 2 | #define CAVALIERI_STREAMS_STREAM_INFRA_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | typedef std::vector next_events_t; 10 | 11 | typedef std::function forward_fn_t; 12 | typedef std::function fwd_new_stream_fn_t; 13 | 14 | typedef std::function on_event_fn_t; 15 | typedef std::function on_init_fn_t; 16 | typedef std::function on_init_simple_fn_t; 17 | 18 | 19 | typedef struct { 20 | on_init_fn_t on_init; 21 | on_event_fn_t on_event; 22 | } stream_node_t; 23 | 24 | typedef std::vector streams_t; 25 | 26 | streams_t create_stream(const on_event_fn_t); 27 | streams_t create_stream(const on_init_fn_t); 28 | streams_t create_stream(const on_init_simple_fn_t); 29 | 30 | 31 | void init_streams(streams_t & streams); 32 | 33 | next_events_t push_event(const streams_t &, const Event &); 34 | 35 | streams_t operator>>(const streams_t & left , const streams_t & right); 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/proto.proto: -------------------------------------------------------------------------------- 1 | option java_package = "com.aphyr.riemann"; 2 | option java_outer_classname = "Proto"; 3 | package riemann; 4 | 5 | message State { 6 | optional int64 time = 1; 7 | optional string state = 2; 8 | optional string service = 3; 9 | optional string host = 4; 10 | optional string description = 5; 11 | optional bool once = 6; 12 | repeated string tags = 7; 13 | optional float ttl = 8; 14 | } 15 | 16 | message Event { 17 | optional int64 time = 1; 18 | optional string state = 2; 19 | optional string service = 3; 20 | optional string host = 4; 21 | optional string description = 5; 22 | repeated string tags = 7; 23 | optional float ttl = 8; 24 | repeated Attribute attributes = 9; 25 | 26 | optional sint64 metric_sint64 = 13; 27 | optional double metric_d = 14; 28 | optional float metric_f = 15; 29 | } 30 | 31 | message Query { 32 | optional string string = 1; 33 | } 34 | 35 | message Msg { 36 | optional bool ok = 2; 37 | optional string error = 3; 38 | repeated State states = 4; 39 | optional Query query = 5; 40 | repeated Event events = 6; 41 | } 42 | 43 | message Attribute { 44 | required string key = 1; 45 | optional string value = 2; 46 | } 47 | -------------------------------------------------------------------------------- /ci/send-test/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import bernhard 4 | import socket 5 | import struct 6 | import time 7 | import sys 8 | 9 | def events(n): 10 | events = list() 11 | for i in range(0, n): 12 | events.append(bernhard.Event(params={'host': "host-%i" % i, 13 | 'service' : 'service-foo', 14 | 'tags': ['foo', 'bar', 'baz']})) 15 | return events 16 | 17 | 18 | def encoded_events(n): 19 | msg = bernhard.Message(events=events(n)) 20 | return msg.raw 21 | 22 | 23 | def send(n, msgs): 24 | raw = encoded_events(n) 25 | 26 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 27 | sock.connect(('localhost', 5555)) 28 | 29 | buff = struct.pack('!I', len(raw)) + raw 30 | 31 | for i in range(0, msgs): 32 | sock.sendall(buff) 33 | 34 | received = 0 35 | for i in range(0, msgs): 36 | rxlen = struct.unpack('!I', sock.recv(4, socket.MSG_WAITALL))[0] 37 | msg = bernhard.Message(raw=sock.recv(rxlen, socket.MSG_WAITALL)) 38 | assert(msg.ok) 39 | received += 1 40 | print "ok: %i" % received 41 | 42 | send(1, 170000) 43 | -------------------------------------------------------------------------------- /src/instrumentation/mem.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | void process_mem_usage(double& vm_usage, double& resident_set) 8 | { 9 | using std::ios_base; 10 | using std::ifstream; 11 | using std::string; 12 | 13 | vm_usage = 0.0; 14 | resident_set = 0.0; 15 | 16 | ifstream stat_stream("/proc/self/stat",ios_base::in); 17 | 18 | string pid, comm, state, ppid, pgrp, session, tty_nr; 19 | string tpgid, flags, minflt, cminflt, majflt, cmajflt; 20 | string utime, stime, cutime, cstime, priority, nice; 21 | string O, itrealvalue, starttime; 22 | 23 | 24 | unsigned long vsize; 25 | long rss; 26 | 27 | stat_stream >> pid >> comm >> state >> ppid >> pgrp >> session >> tty_nr 28 | >> tpgid >> flags >> minflt >> cminflt >> majflt >> cmajflt 29 | >> utime >> stime >> cutime >> cstime >> priority >> nice 30 | >> O >> itrealvalue >> starttime >> vsize >> rss; 31 | 32 | stat_stream.close(); 33 | 34 | long page_size_kb = sysconf(_SC_PAGE_SIZE) / 1024; 35 | 36 | vm_usage = vsize / 1024.0; 37 | resident_set = rss * page_size_kb; 38 | } 39 | -------------------------------------------------------------------------------- /include/rules/common.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_RULES_COMMON_H 2 | #define CAVALIERI_RULES_COMMON_H 3 | 4 | #include 5 | 6 | streams_t critical_above(double value); 7 | 8 | streams_t critical_under(double value); 9 | 10 | streams_t stable_metric(double dt, predicate_t trigger); 11 | 12 | streams_t stable_metric(double dt, predicate_t trigger, predicate_t cancel); 13 | 14 | streams_t agg_stable_metric(double dt, fold_fn_t fold_fn, predicate_t trigger, 15 | predicate_t cancel); 16 | 17 | streams_t max_critical_hosts(size_t n); 18 | 19 | streams_t ratio(const std::string a, const std::string b, 20 | const double default_zero); 21 | 22 | streams_t per_host_ratio(const std::string a, const std::string b, 23 | const double default_zero, double dt, 24 | predicate_t trigger, 25 | predicate_t cancel); 26 | 27 | streams_t stable_event_stream(size_t events); 28 | 29 | struct target_t { 30 | streams_t pagerduty; 31 | streams_t email; 32 | streams_t index; 33 | streams_t all; 34 | }; 35 | 36 | target_t create_targets(const std::string pd_key, const std::string email); 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /include/external/external.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_EXTERNAL_H 2 | #define CAVALIERI_EXTERNAL_EXTERNAL_H 3 | 4 | #include 5 | #include 6 | 7 | class external_interface { 8 | public: 9 | 10 | // Forward to other cavalieri/riemann server 11 | virtual void forward(const std::string server, const int port, 12 | const Event event) = 0; 13 | 14 | // Send to graphite using TCP carbon with new line separator 15 | virtual void graphite(const std::string server, const int port, 16 | const Event event) = 0; 17 | 18 | // Pager pager_duty 19 | virtual void pager_duty_trigger(const std::string pg_key, 20 | const Event event) = 0; 21 | virtual void pager_duty_resolve(const std::string pg_key, 22 | const Event event) = 0; 23 | virtual void pager_duty_acknowledge(const std::string pg_key, 24 | const Event event) = 0; 25 | 26 | // Send email 27 | virtual void email(const std::string server, const std::string from, 28 | const std::vector to, const Event event) = 0; 29 | 30 | virtual ~external_interface() {}; 31 | }; 32 | 33 | #endif 34 | 35 | -------------------------------------------------------------------------------- /include/scheduler/mock_scheduler.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_SCHEDULER_MOCK_SCHEDULER_H 2 | #define CAVALIERI_SCHEDULER_MOCK_SCHEDULER_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | typedef std::tuple queue_element_t; 9 | 10 | class mock_scheduler_queue_cmp 11 | { 12 | public: 13 | bool operator() (const queue_element_t & lhs, const queue_element_t & rhs) const 14 | { 15 | return (std::get<0>(lhs) > std::get<0>(rhs)); 16 | } 17 | }; 18 | 19 | class mock_scheduler : public scheduler_interface { 20 | public: 21 | mock_scheduler(); 22 | remove_task_future_t add_periodic_task(task_fn_t task, 23 | float interval) override; 24 | remove_task_future_t add_once_task(task_fn_t task, float dt) override; 25 | remove_ns_tasks_future_t remove_ns_tasks(const std::string nm) override; 26 | time_t unix_time() override; 27 | void set_time(const time_t t) override; 28 | void clear() override; 29 | 30 | private: 31 | void set_forward_time(time_t time); 32 | 33 | private: 34 | time_t unix_time_; 35 | std::priority_queue< 36 | queue_element_t, 37 | std::vector, 38 | mock_scheduler_queue_cmp 39 | > tasks_; 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /ci/graphite-test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cleanup() { 6 | pkill cavalieri || true 7 | pkill graphite-read-events.sh || true 8 | pkill tcpserver || true 9 | pkill send-events.py || true 10 | 11 | rm -rf *.out || true 12 | 13 | rm -rf rules-forward rules-graphite || true 14 | } 15 | 16 | git clone https://github.com/juruen/cavalieri-rules.git 17 | 18 | cp -a cavalieri-rules rules-forward 19 | cp -a cavalieri-rules rules-graphite 20 | 21 | cp rules-forward.cpp rules-forward/rules.cpp 22 | cd rules-forward && mkdir build && cd build && cmake .. && make 23 | 24 | cavalieri -v 0 -rules_directory . & 25 | 26 | cd ../.. 27 | 28 | cp rules-graphite.cpp rules-graphite/rules.cpp 29 | cd rules-graphite && mkdir build && cd build && cmake .. && make 30 | 31 | cavalieri -v 0 -rules_directory . -events_port 15555 -ws_port 15556 & 32 | 33 | cd ../.. 34 | 35 | ./graphite-read-events.sh & 36 | 37 | touch start.out 38 | 39 | sleep 5 40 | 41 | ./send-events.py & 42 | 43 | for i in $(seq 0 60); do 44 | 45 | LINES=$(wc -l *.out | tail -n 1) 46 | 47 | echo "LINES: ${LINES}" 48 | 49 | if [ "$LINES" == " 1000 total" ]; then 50 | cleanup 51 | exit 0 52 | fi 53 | 54 | sleep 1 55 | 56 | done 57 | 58 | echo "graphite-forward test failed" 59 | 60 | cleanup 61 | 62 | exit 1 63 | -------------------------------------------------------------------------------- /include/external/real_external.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_EXTERNAL_REAL_EXTERNAL_H 2 | #define CAVALIERI_EXTERNAL_REAL_EXTERNAL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class real_external : public external_interface { 13 | public: 14 | real_external(const config, instrumentation::instrumentation & instrumentation); 15 | void forward(const std::string server, const int port, const Event event); 16 | void graphite(const std::string server, const int port, const Event event); 17 | void pager_duty_trigger(const std::string pg_key, const Event event); 18 | void pager_duty_resolve(const std::string pg_key, const Event event); 19 | void pager_duty_acknowledge(const std::string pg_key, const Event event); 20 | void email(const std::string server, const std::string from, 21 | const std::vector to, const Event event); 22 | void stop(); 23 | 24 | private: 25 | riemann_tcp_client riemann_tcp_client_; 26 | class graphite graphite_; 27 | pagerduty_pool pagerduty_; 28 | mailer_pool email_; 29 | std::vector rates_; 30 | }; 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /ci/setup-drone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to set up building environment for drone.io 4 | # We need to add a few repos because drone's builders are precise (12.04) 5 | 6 | set -e 7 | 8 | # Use g++4.8 9 | echo 2 | sudo update-alternatives --config gcc 10 | 11 | # Preconfigure postfix 12 | echo "postfix postfix/mailname string foo.com" | sudo debconf-set-selections 13 | echo "postfix postfix/main_mailer_type string \'Local only\'" | sudo debconf-set-selections 14 | 15 | # Some deps 16 | sudo add-apt-repository -y ppa:fcitx-team/nightly 17 | sudo add-apt-repository -y ppa:svn 18 | sudo add-apt-repository -y ppa:mapnik/boost-backports-1-54 19 | 20 | # Some repos seem to fail quite ofen 21 | while ! timeout 45 sh -c "sudo apt-get update"; do 22 | true 23 | done 24 | 25 | 26 | 27 | # Instal deps 28 | sudo apt-get install debhelper cmake subversion protobuf-compiler \ 29 | libprotobuf-dev libev-dev libgoogle-glog-dev \ 30 | libcurl4-openssl-dev libssl-dev libtbb-dev libjsoncpp-dev lcov \ 31 | flex bison libboost-filesystem-dev libboost-system-dev python2.7-dev \ 32 | postfix 33 | 34 | # Drone.io builders are precise 35 | wget https://gflags.googlecode.com/files/libgflags0_2.0-1_amd64.deb 36 | wget https://gflags.googlecode.com/files/libgflags-dev_2.0-1_amd64.deb 37 | 38 | sudo dpkg -i libgflags0_2.0-1_amd64.deb libgflags-dev_2.0-1_amd64.deb 39 | -------------------------------------------------------------------------------- /include/index/real_index.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INDEX_REAL_INDEX_H 2 | #define CAVALIERI_INDEX_REAL_INDEX_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class real_index : public index_interface { 13 | public: 14 | real_index(pub_sub & pubsub, push_event_fn_t push_event, 15 | const int64_t expire_interval, 16 | scheduler_interface & sched, 17 | instrumentation::instrumentation & instr, 18 | spwan_thread_fn_t spwan_thread_fn); 19 | std::vector query_index(const match_fn_t, const size_t max_matches); 20 | void add_event(const Event& e); 21 | ~real_index(); 22 | 23 | private: 24 | void timer_cb(); 25 | void expire_events(); 26 | 27 | private: 28 | using real_index_t = std::unordered_map; 29 | 30 | private: 31 | pub_sub & pubsub_; 32 | instrumentation::update_gauge_t pre_gauge_; 33 | instrumentation::update_gauge_t post_gauge_; 34 | push_event_fn_t push_event_fn_; 35 | std::atomic expiring_; 36 | spwan_thread_fn_t spwan_thread_fn_; 37 | scheduler_interface & sched_; 38 | std::vector indexes_; 39 | std::vector mutexes_; 40 | 41 | }; 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/instrumentation/reservoir.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | namespace { 6 | 7 | const size_t k_default_size = 1000; 8 | 9 | std::random_device rd; 10 | std::mt19937_64 eng(rd()); 11 | 12 | } 13 | 14 | reservoir::reservoir() 15 | : 16 | reservoir_size_(k_default_size), 17 | samples_(k_default_size, 0), 18 | n_(0) 19 | { 20 | } 21 | 22 | reservoir::reservoir(const size_t size) 23 | : 24 | reservoir_size_(size), 25 | samples_(size, 0), 26 | n_(0) 27 | { 28 | } 29 | 30 | void reservoir::add_sample(const double sample) 31 | { 32 | 33 | VLOG(3) << "add_sample"; 34 | 35 | auto idx(n_.fetch_add(1)); 36 | if (idx < reservoir_size_) { 37 | 38 | std::lock_guard lock(mutex_); 39 | samples_[idx] = sample; 40 | 41 | } else { 42 | 43 | std::uniform_int_distribution<> distr(0, n_); 44 | 45 | auto index = distr(eng); 46 | if (static_cast(index) < reservoir_size_) { 47 | std::lock_guard lock(mutex_); 48 | samples_[index] = sample; 49 | } 50 | 51 | n_++; 52 | 53 | } 54 | 55 | } 56 | 57 | std::vector reservoir::snapshot() { 58 | 59 | std::lock_guard lock(mutex_); 60 | 61 | auto snap(samples_); 62 | samples_ = std::vector(k_default_size, 0); 63 | n_.store(0); 64 | 65 | return snap; 66 | } 67 | -------------------------------------------------------------------------------- /ci/email-test/run-memory-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cleanup() { 6 | pkill send-events.py || true 7 | 8 | rm -rf cavalieri-rules || true 9 | 10 | sudo sh -c "cat > /var/mail/ubuntu" 11 | } 12 | 13 | git clone https://github.com/juruen/cavalieri-rules.git 14 | 15 | cp rules-mail.cpp cavalieri-rules/rules.cpp 16 | cd cavalieri-rules && mkdir build && cd build && cmake .. && make 17 | 18 | export LD_LIBRARY_PATH=$(pwd) 19 | 20 | valgrind --error-exitcode=1 --show-possibly-lost=no \ 21 | --workaround-gcc296-bugs=yes --log-file=../../valgrind.out \ 22 | cavalieri -v 0 -rules_directory . -index_expire_interval 10000 & 23 | 24 | valgrind_pid=$! 25 | 26 | cd ../.. 27 | 28 | sleep 5 29 | 30 | ./send-events.py 100 & 31 | 32 | for i in $(seq 0 300); do 33 | 34 | EMAILS=$(sudo sh -c 'grep "^To:" /var/mail/ubuntu | wc -l') || true 35 | 36 | echo "EMAILS: ${EMAILS}" 37 | 38 | if [ "$EMAILS" == "100" ]; then 39 | cleanup 40 | 41 | sleep 10 42 | 43 | kill -INT $valgrind_pid 44 | 45 | if ! wait $valgrind_pid; then 46 | echo "valgrind reported an error" 47 | cat valgrind.out 48 | exit 1 49 | fi 50 | 51 | grep -q "definitely lost: 0 bytes" valgrind.out \ 52 | || (cat valgrind.out && false) 53 | 54 | exit 0 55 | fi 56 | 57 | sleep 1 58 | 59 | done 60 | 61 | echo "email test failed" 62 | 63 | cleanup 64 | 65 | exit 1 66 | -------------------------------------------------------------------------------- /src/external/graphite_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std::placeholders; 9 | 10 | namespace { 11 | 12 | const size_t k_batch_size = 100; 13 | 14 | std::string escape(const std::string str) { 15 | 16 | const auto s(boost::replace_all_copy(str, ".", "-")); 17 | return boost::replace_all_copy(s, " ", "-"); 18 | 19 | } 20 | 21 | } 22 | 23 | graphite_pool::graphite_pool(size_t thread_num, const std::string host, 24 | const int port) 25 | : 26 | tcp_client_pool_( 27 | thread_num, 28 | host, 29 | port, 30 | k_batch_size, 31 | std::bind(&graphite_pool::output_events, this, _1) 32 | ) 33 | { 34 | } 35 | 36 | void graphite_pool::push_event(const Event & event) { 37 | 38 | tcp_client_pool_.push_event(event); 39 | 40 | } 41 | 42 | std::vector graphite_pool::output_events(const std::vector events) 43 | { 44 | 45 | std::stringstream sstream; 46 | 47 | for (const auto & event : events) { 48 | 49 | sstream << event.host() << "." << escape(event.service()) << " " 50 | << event.metric_to_str() << " " << event.time() << "\n"; 51 | 52 | } 53 | 54 | std::string ev_str = sstream.str(); 55 | 56 | return std::vector(ev_str.begin(), ev_str.end()); 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/external/riemann_tcp_client_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace { 9 | 10 | size_t k_default_batch_size = 100; 11 | 12 | } 13 | 14 | using namespace std::placeholders; 15 | 16 | riemann_tcp_client_pool::riemann_tcp_client_pool(size_t thread_num, 17 | const std::string host, 18 | const int port) 19 | : 20 | tcp_client_pool_( 21 | thread_num, 22 | host, 23 | port, 24 | k_default_batch_size, 25 | std::bind(&riemann_tcp_client_pool::output_events, this, _1) 26 | ) 27 | { 28 | } 29 | 30 | void riemann_tcp_client_pool::push_event(const Event & event) { 31 | tcp_client_pool_.push_event(event); 32 | } 33 | 34 | std::vector riemann_tcp_client_pool::output_events( 35 | const std::vector events) 36 | { 37 | 38 | std::vector buffer; 39 | 40 | riemann::Msg msg; 41 | msg.set_ok(true); 42 | 43 | for (const auto & event : events) { 44 | *msg.add_events() = event.riemann_event(); 45 | } 46 | 47 | auto nsize = htonl(msg.ByteSize()); 48 | 49 | buffer.resize(sizeof(nsize) + msg.ByteSize()); 50 | 51 | memcpy(&buffer[0], static_cast(&nsize), sizeof(nsize)); 52 | msg.SerializeToArray(&buffer[0] + sizeof(nsize), msg.ByteSize()); 53 | 54 | return buffer; 55 | } 56 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | cavalieri (0.1.2) unstable; urgency=medium 2 | 3 | * Release 0.1.2 4 | 5 | -- Javier Uruen Val Sun, 09 Nov 2014 16:51:19 +0200 6 | 7 | cavalieri (0.1.1) unstable; urgency=medium 8 | 9 | * Release 0.1.1 10 | 11 | -- Javier Uruen Val Wed, 24 Sep 2014 19:05:34 +0200 12 | 13 | cavalieri (0.1.0) unstable; urgency=medium 14 | 15 | * Release 0.1.0 16 | 17 | -- Javier Uruen Val Tue, 23 Sep 2014 21:46:42 +0200 18 | 19 | cavalieri (0.0.7) unstable; urgency=medium 20 | 21 | * Release 0.0.7 22 | 23 | -- Javier Uruen Val Sun, 14 Sep 2014 15:58:18 +0200 24 | 25 | cavalieri (0.0.6) unstable; urgency=medium 26 | 27 | * Release 0.0.6 28 | 29 | -- Javier Uruen Val Tue, 05 Aug 2014 16:40:31 +0200 30 | 31 | cavalieri (0.0.5) unstable; urgency=low 32 | 33 | * Release 0.0.5 34 | 35 | -- Javier Uruen Val Wed, 21 May 2014 23:42:10 +0200 36 | 37 | cavalieri (0.0.3) unstable; urgency=low 38 | 39 | * Release 0.0.3 40 | 41 | -- Javier Uruen Val Sat, 19 Apr 2014 16:48:33 +0200 42 | 43 | cavalieri (0.0.2) unstable; urgency=low 44 | 45 | * Release 0.0.2 46 | 47 | -- Javier Uruen Val Thu, 10 Apr 2014 13:20:49 +0200 48 | 49 | cavalieri (0.0.1) unstable; urgency=low 50 | 51 | * Initial Release. 52 | 53 | -- Javier Uruen Val Sat, 22 Mar 2014 22:31:42 +0100 54 | -------------------------------------------------------------------------------- /src/transport/listen_tcp_socket.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace { 7 | const uint32_t k_listen_backlog = 100; 8 | } 9 | 10 | int create_tcp_listen_socket(int port) { 11 | 12 | struct sockaddr_in addr; 13 | 14 | int socket_fd = socket(PF_INET, SOCK_STREAM, 0); 15 | 16 | int reuse_addr = 1; 17 | 18 | if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, &reuse_addr, 19 | sizeof(reuse_addr)) != 0) 20 | { 21 | LOG(ERROR) << "failed to set SO_REUSEADDR in socket"; 22 | } 23 | 24 | /* 25 | int recv_buf = 21299200; 26 | 27 | if (setsockopt(socket_fd, SOL_SOCKET, SO_RCVBUF, &recv_buf, 28 | sizeof(recv_buf)) != 0) 29 | { 30 | LOG(ERROR) << "failed to set SO_REUSEADDR in socket"; 31 | } 32 | 33 | if (setsockopt(socket_fd, SOL_SOCKET, SO_SNDBUF, &recv_buf, 34 | sizeof(recv_buf)) != 0) 35 | { 36 | LOG(ERROR) << "failed to set SO_REUSEADDR in socket"; 37 | } 38 | */ 39 | 40 | 41 | 42 | 43 | 44 | addr.sin_family = AF_INET; 45 | addr.sin_port = htons(port); 46 | addr.sin_addr.s_addr = INADDR_ANY; 47 | 48 | if (bind(socket_fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) { 49 | LOG(FATAL) << "bind error: " << strerror(errno); 50 | exit(1); 51 | } 52 | 53 | fcntl(socket_fd, F_SETFL, fcntl(socket_fd, F_GETFL, 0) | O_NONBLOCK); 54 | listen(socket_fd, k_listen_backlog); 55 | 56 | return socket_fd; 57 | } 58 | -------------------------------------------------------------------------------- /src/streams/lib.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | thread_local std::string * thread_ns = nullptr; 4 | 5 | void free_thread_ns() { 6 | if (thread_ns != nullptr) { 7 | delete thread_ns; 8 | thread_ns = nullptr; 9 | } 10 | } 11 | 12 | void set_thread_ns(const std::string ns) { 13 | free_thread_ns(); 14 | thread_ns = new std::string(ns); 15 | } 16 | 17 | void set_thread_global_ns() { 18 | free_thread_ns(); 19 | } 20 | 21 | std::string get_thread_ns() { 22 | if (thread_ns) { 23 | return *thread_ns; 24 | } else { 25 | return k_global_ns; 26 | } 27 | } 28 | 29 | bool stream_lib::used() const { 30 | return ref_counter.load() & 1; 31 | } 32 | 33 | bool stream_lib::inc() { 34 | unsigned int current = ref_counter.load(); 35 | unsigned int new_val; 36 | 37 | do { 38 | 39 | if (!(current & 1)) { 40 | // Stream is not being used. Abort inc 41 | return false; 42 | } 43 | 44 | new_val = (((current >> 1) + 1) << 1) | 1; 45 | 46 | } while(!ref_counter.compare_exchange_strong(current, new_val)); 47 | 48 | return true; 49 | } 50 | 51 | void stream_lib::dec() { 52 | unsigned int current = ref_counter.load(); 53 | unsigned int new_val; 54 | 55 | do { 56 | 57 | new_val = (((current >> 1) - 1) << 1) | (current & 1); 58 | 59 | } while(!ref_counter.compare_exchange_strong(current, new_val)); 60 | } 61 | 62 | void stream_lib::set_used(const bool used) { 63 | if (used) { 64 | ref_counter |= 1; 65 | } else { 66 | ref_counter &= ~1; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/websocket/worker_threads.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace { 5 | 6 | const size_t k_stop_attempts = 50; 7 | const size_t k_stop_interval_check_ms = 100; 8 | 9 | } 10 | 11 | worker_threads::worker_threads(const size_t threads, task_t task) 12 | : 13 | task_(task), 14 | finished_threads_(threads, 0) 15 | { 16 | 17 | auto run_fn = [=](const int i) 18 | { 19 | run_tasks(i); 20 | }; 21 | 22 | 23 | for (size_t i = 0; i < threads; i++) { 24 | threads_.push_back(std::move(std::thread(run_fn, i))); 25 | } 26 | 27 | } 28 | 29 | void worker_threads::stop() { 30 | 31 | VLOG(3) << "stopping worker_threads"; 32 | 33 | for (size_t attempts = k_stop_attempts; attempts > 0; attempts--) { 34 | 35 | size_t stopped = 0 ; 36 | 37 | for (const auto & t: finished_threads_) { 38 | if (t) stopped++; 39 | } 40 | 41 | if (stopped == threads_.size()) { 42 | break; 43 | } 44 | 45 | VLOG(3) << "Waiting for " << threads_.size() - stopped << " threads"; 46 | 47 | std::this_thread::sleep_for( 48 | std::chrono::milliseconds(k_stop_interval_check_ms)); 49 | } 50 | 51 | for (size_t i = 0; i < threads_.size(); i++) { 52 | threads_[i].join(); 53 | } 54 | 55 | } 56 | 57 | void worker_threads::run_tasks(const int i) { 58 | 59 | VLOG(3) << "starting worker thread " << i; 60 | 61 | task_(); 62 | 63 | finished_threads_[i] = 1; 64 | 65 | VLOG(3) << "stopping worker thread " << i; 66 | 67 | } 68 | -------------------------------------------------------------------------------- /include/core/real_core.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_CORE_REAL_CORE_H 2 | #define CAVALIERI_CORE_REAL_CORE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | class real_core : public core_interface { 19 | public: 20 | 21 | real_core(const config &); 22 | void start(); 23 | void add_stream(std::shared_ptr stream); 24 | void reload_rules(); 25 | index_interface & idx(); 26 | scheduler_interface & sched(); 27 | external_interface & externals(); 28 | ~real_core(); 29 | 30 | private: 31 | config config_; 32 | 33 | instrumentation::instrumentation instrumentation_; 34 | 35 | executor_thread_pool executor_pool_; 36 | 37 | std::unique_ptr main_loop_; 38 | std::unique_ptr scheduler_; 39 | std::unique_ptr externals_; 40 | std::shared_ptr streams_; 41 | std::unique_ptr pubsub_; 42 | std::unique_ptr index_; 43 | std::unique_ptr tcp_server_; 44 | std::unique_ptr udp_server_; 45 | std::unique_ptr ws_server_; 46 | 47 | std::vector> sh_streams_; 48 | 49 | }; 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /tests/folds_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef FOLDS_TEST_CASE 2 | #define FOLDS_TEST_CASE 3 | 4 | #include 5 | 6 | TEST(sum_test_case, test) 7 | { 8 | std::vector v; 9 | 10 | std::vector events(5); 11 | for (auto & e: events) { 12 | e.set_metric_d(1); 13 | } 14 | 15 | ASSERT_EQ(5, sum(events).metric_d()); 16 | } 17 | 18 | TEST(product_test_case, test) 19 | { 20 | std::vector v; 21 | 22 | std::vector events(5); 23 | for (auto & e: events) { 24 | e.set_metric_d(2); 25 | } 26 | 27 | ASSERT_EQ(1<<5, product(events).metric_d()); 28 | } 29 | 30 | TEST(difference_test_case, test) 31 | { 32 | std::vector v; 33 | 34 | std::vector events(3); 35 | for (auto & e: events) { 36 | e.set_metric_d(1); 37 | } 38 | 39 | ASSERT_EQ(-1, difference(events).metric_d()); 40 | } 41 | 42 | TEST(mean_test_case, test) 43 | { 44 | std::vector v; 45 | 46 | std::vector events(3); 47 | for (size_t i = 1; i < 4; i++) { 48 | events[i - 1].set_metric_d(i); 49 | } 50 | 51 | ASSERT_EQ(2, mean(events).metric_d()); 52 | } 53 | 54 | TEST(maximum_test_case, test) 55 | { 56 | std::vector v; 57 | 58 | std::vector events(3); 59 | for (size_t i = 1; i < 4; i++) { 60 | events[i - 1].set_metric_d(i); 61 | } 62 | 63 | ASSERT_EQ(3, maximum(events).metric_d()); 64 | } 65 | 66 | TEST(minimum_test_case, test) 67 | { 68 | std::vector v; 69 | 70 | std::vector events(3); 71 | for (size_t i = 1; i < 4; i++) { 72 | events[i - 1].set_metric_d(i); 73 | } 74 | 75 | ASSERT_EQ(1, minimum(events).metric_d()); 76 | } 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /tests/index_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_TEST_CASE_H 2 | #define INDEX_TEST_CASE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | TEST(index_test_case, test) 11 | { 12 | g_core->sched().clear(); 13 | 14 | std::vector s; 15 | 16 | pub_sub pubsub; 17 | 18 | auto no_thread = [](std::function fn) { fn(); }; 19 | 20 | config conf; 21 | instrumentation::instrumentation instr(conf); 22 | 23 | real_index index(pubsub, [&](const Event & pe) { s.push_back(pe); }, 24 | 60, g_core->sched(), instr, no_thread); 25 | 26 | std::vector vec; 27 | pubsub.subscribe("index", [&](const Event&e) { vec.push_back(e); }); 28 | 29 | Event e; 30 | e.set_host("foo"); 31 | e.set_service("bar"); 32 | e.set_time(1); 33 | e.set_ttl(120); 34 | 35 | index.add_event(e); 36 | 37 | auto all_events = index.query_index([=](const Event&) { return true; }, 10); 38 | ASSERT_EQ(1u, all_events.size()); 39 | ASSERT_EQ("foo", all_events[0].host()); 40 | ASSERT_EQ("bar", all_events[0].service()); 41 | 42 | vec.clear(); 43 | 44 | 45 | e.set_host("baz"); 46 | e.set_time(100); 47 | e.set_ttl(120); 48 | index.add_event(e); 49 | 50 | ASSERT_EQ(1u, vec.size()); 51 | ASSERT_EQ("baz", vec[0].host()); 52 | ASSERT_EQ("bar", vec[0].service()); 53 | 54 | g_core->sched().set_time(180); 55 | ASSERT_EQ(1u, s.size()); 56 | ASSERT_EQ("foo", s[0].host()); 57 | ASSERT_EQ("bar", s[0].service()); 58 | ASSERT_EQ("expired", s[0].state()); 59 | s.clear(); 60 | 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/scheduler/mock_scheduler.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | mock_scheduler::mock_scheduler() 4 | : 5 | unix_time_(0), 6 | tasks_(mock_scheduler_queue_cmp()) 7 | { 8 | } 9 | 10 | remove_task_future_t mock_scheduler::add_periodic_task(task_fn_t task, 11 | float interval) 12 | { 13 | auto p = static_cast(interval); 14 | tasks_.push(std::make_tuple(unix_time_ + p, p, task)); 15 | 16 | return {}; 17 | } 18 | 19 | remove_task_future_t mock_scheduler::add_once_task(task_fn_t task, float dt) { 20 | auto p = static_cast(dt); 21 | tasks_.push(std::make_tuple(unix_time_ + p, 0, task)); 22 | 23 | return {}; 24 | } 25 | 26 | remove_ns_tasks_future_t mock_scheduler::remove_ns_tasks(const std::string) { 27 | return {}; 28 | } 29 | 30 | time_t mock_scheduler::unix_time() { 31 | return unix_time_; 32 | } 33 | 34 | void mock_scheduler::set_time(const time_t event_time) { 35 | while (!tasks_.empty()) { 36 | auto lowest = std::get<0>(tasks_.top()); 37 | if (lowest > event_time) { 38 | break; 39 | } 40 | set_forward_time(lowest); 41 | auto interval = std::get<1>(tasks_.top()); 42 | auto fn = std::get<2>(tasks_.top()); 43 | tasks_.pop(); 44 | fn(); 45 | if (interval) { 46 | tasks_.push(std::make_tuple(unix_time_ + interval, interval, fn)); 47 | } 48 | } 49 | set_forward_time(event_time); 50 | } 51 | 52 | void mock_scheduler::set_forward_time(time_t time) { 53 | if (time > unix_time_) { 54 | unix_time_ = time; 55 | } 56 | } 57 | 58 | void mock_scheduler::clear() { 59 | while (!tasks_.empty()) { 60 | tasks_.pop(); 61 | } 62 | unix_time_ = 0; 63 | } 64 | 65 | 66 | -------------------------------------------------------------------------------- /include/predicates/predicates.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_PREDICATES_PREDICATES_H 2 | #define CAVALIERI_PREDICATES_PREDICATES_H 3 | 4 | #define PRED(EXP) [=](e_t e) { return (EXP); } 5 | 6 | namespace predicates { 7 | 8 | predicate_t above_eq(const double value); 9 | 10 | predicate_t above(const double value); 11 | 12 | predicate_t under_eq(const double value); 13 | 14 | predicate_t under(const double value); 15 | 16 | predicate_t state(const std::string state); 17 | 18 | predicate_t service(const std::string state); 19 | 20 | predicate_t match(const std::string key, const std::string value); 21 | 22 | predicate_t match_any(const std::string key, 23 | const std::vector values); 24 | 25 | predicate_t match_re(const std::string key, const std::string value); 26 | 27 | predicate_t match_re_any(const std::string key, 28 | const std::vector values); 29 | 30 | predicate_t match_like(const std::string key, const std::string value); 31 | 32 | predicate_t match_like_any(const std::string key, 33 | const std::vector values); 34 | 35 | predicate_t default_true(); 36 | 37 | bool tagged_any(e_t e, const tags_t& tags); 38 | 39 | bool tagged_all(e_t e, const tags_t& tags); 40 | 41 | bool expired(e_t e); 42 | 43 | bool above_eq(e_t e, const double value); 44 | 45 | bool above(e_t e, const double value); 46 | 47 | bool under_eq(e_t e, const double value); 48 | 49 | bool under(e_t e, const double value); 50 | 51 | bool match(e_t e, const std::string key, const std::string value); 52 | 53 | bool match_re(e_t e, const std::string key, const std::string value); 54 | 55 | bool match_like(e_t e, const std::string key, const std::string value); 56 | 57 | } 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: miniriemann 3 | Source: 4 | 5 | Files: * 6 | Copyright: 7 | 8 | License: MIT 9 | 10 | Files: debian/* 11 | Copyright: 2014 Javier Uruen 12 | License: MIT 13 | 14 | License: MIT 15 | Permission is hereby granted, free of charge, to any person obtaining a 16 | copy of this software and associated documentation files (the "Software"), 17 | to deal in the Software without restriction, including without limitation 18 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 19 | and/or sell copies of the Software, and to permit persons to whom the 20 | Software is furnished to do so, subject to the following conditions: 21 | . 22 | The above copyright notice and this permission notice shall be included 23 | in all copies or substantial portions of the Software. 24 | . 25 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 26 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 28 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 29 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 30 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 31 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | 33 | # Please also look if there are files or directories which have a 34 | # different copyright/license attached and list them here. 35 | # Please avoid to pick license terms that are more restrictive than the 36 | # packaged work, as it may make Debian's contributions unacceptable upstream. 37 | -------------------------------------------------------------------------------- /src/external/mock_external.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace { 5 | 6 | void add_call(std::vector & calls, 7 | const std::string external, 8 | const std::string message, 9 | const std::string extra, 10 | const Event e) 11 | { 12 | calls.push_back({external, message, extra, g_core->sched().unix_time(), e}); 13 | } 14 | 15 | } 16 | 17 | void mock_external::forward(const std::string server, const int, const Event e) 18 | { 19 | add_call(calls_, "forward", "forward event to " + server, "", e); 20 | } 21 | 22 | void mock_external::graphite(const std::string server, const int, const Event e) 23 | { 24 | add_call(calls_, "graphite", "send event to " + server, "", e); 25 | } 26 | 27 | void mock_external::pager_duty_trigger(const std::string pg_key, 28 | const Event e) 29 | { 30 | add_call(calls_, "pagerduty", "trigger pagerduty for key " + pg_key, "", e); 31 | } 32 | 33 | void mock_external::pager_duty_resolve(const std::string pg_key, 34 | const Event e) 35 | { 36 | add_call(calls_, "pagerduty", "resolve pagerduty for key " + pg_key, "", e); 37 | } 38 | 39 | void mock_external::pager_duty_acknowledge(const std::string pg_key, 40 | const Event e) 41 | { 42 | add_call(calls_, "pagerduty", "acknowledge pagerduty for key " + pg_key, 43 | "", e); 44 | } 45 | 46 | void mock_external::email(const std::string, const std::string from, 47 | const std::vector, const Event e) 48 | { 49 | add_call(calls_, "email", "send email from " + from + " to ", "", e); 50 | } 51 | 52 | std::vector mock_external::calls() const { 53 | return calls_; 54 | } 55 | -------------------------------------------------------------------------------- /include/util/util.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_UTIL_H 2 | #define CAVALIERI_UTIL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #define UNUSED_VAR(x) (void)x 12 | 13 | bool match_regex(const std::string value, const std::string re); 14 | 15 | bool match_like(const std::string value, const std::string re); 16 | 17 | void set_event_value( 18 | Event & e, 19 | const std::string & key, 20 | const std::string & value, 21 | const bool & replace 22 | ); 23 | 24 | void set_event_value( 25 | Event & e, 26 | const std::string & key, 27 | const int & value, 28 | const bool& replace 29 | ); 30 | 31 | void set_event_value( 32 | Event & e, 33 | const std::string & key, 34 | const double & value, 35 | const bool & replace 36 | ); 37 | 38 | void set_event_value( 39 | Event & e, 40 | const std::string & key, 41 | const boost::variant & val, 42 | const bool & replace 43 | ); 44 | 45 | std::basic_string base64Encode(std::vector inputBuffer); 46 | 47 | std::string sha1(const std::string& str); 48 | 49 | bool parse_uri( 50 | const std::string& escaped_uri, 51 | std::string& index, 52 | std::map& params ); 53 | 54 | template 55 | std::vector conj(const std::vector &v, T t) { 56 | std::vector nv(v); 57 | nv.push_back(t); 58 | return std::move(nv); 59 | } 60 | 61 | template 62 | std::list conj(const std::list &v, T t) { 63 | std::list nv(v); 64 | nv.push_back(t); 65 | return std::move(nv); 66 | } 67 | 68 | char** copy_args(int argc, char **argv); 69 | 70 | void free_args(int argc, char **argv); 71 | 72 | void ld_environment(char **argv, const std::string dir); 73 | 74 | #endif 75 | 76 | -------------------------------------------------------------------------------- /include/websocket/websocket_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_WEBSOCKET_WEBSOCKET_POOL_H 2 | #define CAVALIERI_WEBSOCKET_WEBSOCKET_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | class websocket_pool { 13 | public: 14 | websocket_pool(size_t thread_num, pub_sub & pubsub, real_index & index); 15 | void add_client(const int fd); 16 | void stop(); 17 | 18 | private: 19 | using query_fn_t = std::function; 20 | using fd_queue_t = std::queue; 21 | using fd_ctx_t = struct 22 | { 23 | ws_connection ws_cnx; 24 | query_fn_t query_fn; 25 | fd_queue_t queue; 26 | size_t token; 27 | }; 28 | using event_t = struct 29 | { 30 | Event event; 31 | int fd; 32 | size_t token; 33 | }; 34 | 35 | using event_queue_t = tbb::concurrent_bounded_queue; 36 | 37 | private: 38 | void on_new_cnx(int fd, async_loop & loop, tcp_connection & cnx); 39 | void on_fd_ready(async_fd & async, tcp_connection & cnx); 40 | void on_async_signal(async_loop & loop); 41 | 42 | bool pending_ws_init(fd_ctx_t & fd_ctx); 43 | bool handle_ws_init(async_fd & async, fd_ctx_t & fd_ctx); 44 | 45 | bool flush_fd_queue(async_loop & loop, int fd, fd_ctx_t & fd_ctx); 46 | 47 | void update_filters(const size_t loop_id); 48 | 49 | private: 50 | real_index & index_; 51 | tcp_pool tcp_pool_; 52 | worker_pool worker_pool_; 53 | std::vector> fd_ctxes_; 54 | std::vector event_queues_; 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/rule_tester.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | DEFINE_string(input_events, "", "json string containing input events"); 14 | DEFINE_string(rules_directory, ".", "directory containing the rules to test"); 15 | 16 | int main(int argc, char **argv) { 17 | 18 | 19 | int orig_argc = argc; 20 | char **orig_argv = copy_args(argc, argv); 21 | 22 | FLAGS_logtostderr = true; 23 | FLAGS_v = 1; 24 | 25 | google::ParseCommandLineFlags(&argc, &argv, true); 26 | 27 | google::InitGoogleLogging(argv[0]); 28 | 29 | ld_environment(orig_argv, FLAGS_rules_directory); 30 | 31 | free_args(orig_argc, orig_argv); 32 | 33 | bool ok; 34 | std::vector events = json_to_events(FLAGS_input_events, ok); 35 | 36 | auto m_core = std::make_shared(); 37 | g_core = std::dynamic_pointer_cast(m_core); 38 | 39 | if (!ok) { 40 | LOG(ERROR) << "couldn't parse input events"; 41 | return -1; 42 | } 43 | 44 | start_core(argc, argv); 45 | 46 | std::vector streams(100); 47 | load_rules(FLAGS_rules_directory, streams); 48 | 49 | if (!streams[0].used()) { 50 | LOG(ERROR) << "failed to load rules"; 51 | return -1; 52 | } 53 | 54 | for (const auto & event: events) { 55 | g_core->sched().set_time(event.time()); 56 | for (const auto & stream : streams) { 57 | if (stream.used()) { 58 | push_event(*stream.stream, event); 59 | } 60 | } 61 | } 62 | 63 | auto & idx = m_core->mock_index_impl(); 64 | std::cout << results(idx.events(), m_core->mock_external_impl().calls()) 65 | << "\n"; 66 | 67 | 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /tests/tcp_connection_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef TCP_CONNECTION_TEST_CASE 2 | #define TCP_CONNECTION_TEST_CASE 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | extern mock_os_functions mock_os; 10 | 11 | /* 12 | TEST(tcp_connection_read_test_case, test) 13 | { 14 | mock_os.buffer.clear(); 15 | for (int i = 0; i < 1024; i++) { 16 | mock_os.buffer.push_back(i); 17 | } 18 | 19 | tcp_connection conn(0); 20 | 21 | conn.read(3); 22 | ASSERT_EQ(conn.close_connection, false); 23 | ASSERT_EQ(conn.r_buffer[0], 0); 24 | ASSERT_EQ(conn.r_buffer[1], 1); 25 | ASSERT_EQ(conn.r_buffer[2], 2); 26 | ASSERT_EQ(conn.bytes_read, 3); 27 | 28 | conn.read(0); 29 | ASSERT_EQ(conn.close_connection, true); 30 | } 31 | 32 | TEST(tcp_connection_copy_to_write_buffer_test_case, test) 33 | { 34 | mock_os.buffer.clear(); 35 | 36 | tcp_connection conn(0); 37 | 38 | std::string buffer("foobar"); 39 | 40 | auto ret = conn.copy_to_write_buffer(buffer.c_str(), buffer.size()); 41 | ASSERT_TRUE(ret); 42 | ASSERT_EQ(conn.bytes_to_write, buffer.size()); 43 | 44 | ret = conn.copy_to_write_buffer(buffer.c_str(), conn.buffer_size + 1); 45 | ASSERT_FALSE(ret); 46 | 47 | ASSERT_TRUE(conn.pending_write()); 48 | ASSERT_TRUE(conn.pending_read()); 49 | } 50 | 51 | TEST(tcp_connection_write_test_case, test) 52 | { 53 | mock_os.buffer.clear(); 54 | mock_os.buffer.reserve(100); 55 | tcp_connection conn(0); 56 | 57 | std::string buffer("foobar"); 58 | 59 | auto ret = conn.copy_to_write_buffer(buffer.c_str(), buffer.size()); 60 | ASSERT_TRUE(ret); 61 | ASSERT_EQ(conn.bytes_to_write, buffer.size()); 62 | 63 | ret = conn.write(); 64 | ASSERT_EQ(conn.bytes_written, buffer.size()); 65 | ASSERT_EQ(conn.bytes_to_write,0); 66 | ASSERT_TRUE((buffer 67 | == std::string{mock_os.buffer.begin(), mock_os.buffer.end()})); 68 | 69 | } 70 | 71 | */ 72 | #endif 73 | -------------------------------------------------------------------------------- /include/transport/tcp_client_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TCP_CLIENT_POOL_H 2 | #define CAVALIERI_TCP_CLIENT_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | /* This callback is used to translate events into whatever needs to be 13 | * sent in the wire. 14 | */ 15 | typedef std::function(const Event & event)> 16 | output_event_fn_t; 17 | 18 | typedef std::function(const std::vector)> 19 | output_events_fn_t; 20 | 21 | 22 | class tcp_client_pool { 23 | public: 24 | tcp_client_pool(size_t thread_num, const std::string host, const int port, 25 | output_event_fn_t output_event_fn ); 26 | tcp_client_pool(size_t thread_num, const std::string host, const int port, 27 | size_t batch_size, output_events_fn_t output_events_fn); 28 | void push_event(const Event & event); 29 | void stop(); 30 | 31 | private: 32 | void create_conn(int fd, async_loop & loop, tcp_connection & conn); 33 | void data_ready(async_fd & async, tcp_connection & conn); 34 | void async(async_loop & loop); 35 | void signal_batch_flush(const size_t loop_id); 36 | void connect_clients(const size_t loop_id); 37 | 38 | private: 39 | typedef tbb::concurrent_bounded_queue event_queue_t; 40 | typedef std::queue> fd_event_queue_t; 41 | typedef std::pair fd_conn_data_t; 42 | 43 | tcp_pool tcp_pool_; 44 | const std::string host_; 45 | const int port_; 46 | std::vector> thread_event_queues_; 47 | std::vector> fd_event_queues_; 48 | output_event_fn_t output_event_fn_; 49 | output_events_fn_t output_events_fn_; 50 | bool batched_; 51 | size_t batch_size_; 52 | std::vector flush_batch_; 53 | size_t next_thread_; 54 | 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /tests/mock_scheduler_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef MOCK_SCHEDULER_TEST_CASE 2 | #define MOCK_SCHEDULER_TEST_CASE 3 | 4 | #include 5 | 6 | TEST(mock_scheduler_unix_time_test_case, test) 7 | { 8 | mock_scheduler sched; 9 | 10 | ASSERT_EQ(0, sched.unix_time()); 11 | 12 | sched.set_time(1); 13 | ASSERT_EQ(1, sched.unix_time()); 14 | 15 | sched.set_time(5); 16 | ASSERT_EQ(5, sched.unix_time()); 17 | 18 | // Make sure we don't go back in time 19 | sched.set_time(4); 20 | ASSERT_EQ(5, sched.unix_time()); 21 | 22 | } 23 | 24 | TEST(mock_scheduler_add_periodic_task_test_case, test) 25 | { 26 | mock_scheduler sched; 27 | 28 | int calls1 = 0; 29 | auto t1 = [&]() { calls1++; }; 30 | sched.add_periodic_task(t1, 5); 31 | 32 | sched.set_time(1); 33 | sched.set_time(2); 34 | ASSERT_EQ(0, calls1); 35 | 36 | sched.set_time(4); 37 | sched.set_time(5); 38 | ASSERT_EQ(1, calls1); 39 | 40 | sched.set_time(6); 41 | ASSERT_EQ(1, calls1); 42 | 43 | sched.set_time(15); 44 | ASSERT_EQ(3, calls1); 45 | 46 | int calls2 = 0; 47 | auto t2 = [&]() { calls2++; }; 48 | sched.add_periodic_task(t2, 2); 49 | 50 | sched.set_time(15); 51 | ASSERT_EQ(3, calls1); 52 | ASSERT_EQ(0, calls2); 53 | 54 | sched.set_time(17); 55 | ASSERT_EQ(3, calls1); 56 | ASSERT_EQ(1, calls2); 57 | 58 | sched.set_time(20); 59 | ASSERT_EQ(4, calls1); 60 | ASSERT_EQ(2, calls2); 61 | 62 | sched.set_time(100); 63 | ASSERT_EQ(20, calls1); 64 | ASSERT_EQ(42, calls2); 65 | 66 | int calls3 = 0; 67 | auto t3 = [&]() { calls3++; }; 68 | sched.add_periodic_task(t3, 5); 69 | sched.clear(); 70 | ASSERT_EQ(0, calls3); 71 | ASSERT_EQ(0, sched.unix_time()); 72 | } 73 | 74 | TEST(mock_scheduler_add_once_task_test_case, test) 75 | { 76 | mock_scheduler sched; 77 | 78 | int calls1 = 0; 79 | auto t1 = [&]() { calls1++; }; 80 | sched.add_once_task(t1, 5); 81 | 82 | sched.set_time(1); 83 | sched.set_time(2); 84 | ASSERT_EQ(0, calls1); 85 | 86 | sched.set_time(4); 87 | sched.set_time(5); 88 | ASSERT_EQ(1, calls1); 89 | 90 | sched.set_time(10); 91 | sched.set_time(11); 92 | ASSERT_EQ(1, calls1); 93 | } 94 | 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/transport/udp_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std::placeholders; 9 | 10 | namespace { 11 | 12 | const size_t k_udp_buffer_size = 4096; 13 | 14 | int create_listen_udp_socket(uint32_t port) { 15 | int sd = socket(PF_INET, SOCK_DGRAM, 0); 16 | 17 | struct sockaddr_in addr; 18 | 19 | bzero(&addr, sizeof(addr)); 20 | addr.sin_family = AF_INET; 21 | addr.sin_port = htons(port); 22 | addr.sin_addr.s_addr = INADDR_ANY; 23 | 24 | if (bind(sd, (struct sockaddr*) &addr, sizeof(addr)) != 0) { 25 | LOG(FATAL) << "bind"; 26 | exit(1); 27 | } 28 | 29 | return sd; 30 | } 31 | } 32 | 33 | udp_pool::udp_pool( 34 | size_t thread_num, 35 | uint32_t port, 36 | udp_read_fn_t udp_read_fn) 37 | : 38 | async_thread_pool_(thread_num), 39 | port_(port), 40 | udp_read_fn_(udp_read_fn) 41 | { 42 | VLOG(3) << "udp_pool() size: " << thread_num; 43 | async_thread_pool_.set_run_hook(std::bind(&udp_pool::run_hook, this, _1)); 44 | } 45 | 46 | void udp_pool::run_hook(async_loop & loop) { 47 | size_t tid = loop.id(); 48 | VLOG(3) << "udp pool run_hook() tid: " << tid; 49 | 50 | auto socket_cb = std::bind(&udp_pool::socket_callback, this, _1); 51 | loop.add_fd(create_listen_udp_socket(port_), async_fd::read, 52 | socket_cb); 53 | } 54 | 55 | void udp_pool::socket_callback(async_fd & async) { 56 | 57 | auto tid = async.loop().id(); 58 | VLOG(3) << "socket_callback() tid: " << tid; 59 | 60 | if (async.error()) { 61 | VLOG(3) << "got invalid event: " << strerror(errno); 62 | return; 63 | } 64 | 65 | std::vector buffer(k_udp_buffer_size); 66 | 67 | size_t bytes = recvfrom(async.fd(), reinterpret_cast(&buffer[0]), 68 | buffer.size(), 0, NULL, NULL); 69 | 70 | buffer.resize(bytes); 71 | 72 | udp_read_fn_(buffer); 73 | 74 | async.set_mode(async_fd::read); 75 | 76 | VLOG(3) << "--socket_callback() tid: " << tid; 77 | } 78 | 79 | void udp_pool::start_threads() { 80 | async_thread_pool_.start_threads(); 81 | } 82 | 83 | void udp_pool::stop_threads() { 84 | async_thread_pool_.stop_threads(); 85 | } 86 | 87 | udp_pool::~udp_pool() { 88 | async_thread_pool_.stop_threads(); 89 | } 90 | -------------------------------------------------------------------------------- /include/scheduler/real_scheduler.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_SCHEDULER_REAL_SCHEDULER_H 2 | #define CAVALIERI_SCHEDULER_REAL_SCHEDULER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | class real_scheduler : public scheduler_interface { 12 | public: 13 | real_scheduler(); 14 | remove_task_future_t add_periodic_task(task_fn_t task, 15 | float interval) override; 16 | remove_task_future_t add_once_task(task_fn_t task, float dt) override; 17 | remove_ns_tasks_future_t remove_ns_tasks(const std::string) override; 18 | time_t unix_time() override; 19 | void set_time(const time_t t) override; 20 | void clear() override; 21 | void stop(); 22 | 23 | private: 24 | void async_callback(async_loop & loop); 25 | void add_tasks(async_loop & loop); 26 | void remove_ns_tasks(async_loop & loop); 27 | std::future add_task(task_fn_t task, 28 | float interval, bool once); 29 | 30 | private: 31 | using task_promise_shared_t = std::shared_ptr>; 32 | 33 | using task_promise_t = struct { 34 | task_promise_shared_t promise; 35 | task_cb_fn_t task; 36 | float interval; 37 | bool once; 38 | std::string nm; 39 | }; 40 | using promise_queue_t = 41 | tbb::concurrent_bounded_queue; 42 | 43 | using remove_task_queue_t = tbb::concurrent_bounded_queue; 44 | 45 | 46 | using ns_promise_shared_t = std::shared_ptr>; 47 | 48 | using ns_promise_t = struct { 49 | ns_promise_shared_t promise; 50 | std::shared_ptr> loops; 51 | std::string nm; 52 | }; 53 | 54 | using ns_queue_t = tbb::concurrent_bounded_queue; 55 | 56 | private: 57 | async_thread_pool threads_; 58 | std::vector task_promises_; 59 | std::vector remove_tasks_; 60 | std::vector ns_promises_; 61 | 62 | }; 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /src/riemann_tcp_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | using namespace std::placeholders; 6 | 7 | namespace { 8 | 9 | async_fd::mode conn_to_mode(const tcp_connection & conn) { 10 | 11 | VLOG(3) << "conn_to_mode"; 12 | 13 | if (conn.pending_read() && conn.pending_write()) { 14 | VLOG(3) << "conn_to_mode rw"; 15 | return async_fd::readwrite; 16 | VLOG(3) << "conn_to_mode r"; 17 | } else if (conn.pending_read()) { 18 | return async_fd::read; 19 | VLOG(3) << "conn_to_mode w"; 20 | } else if (conn.pending_write()) { 21 | return async_fd::write; 22 | } else { 23 | return async_fd::none; 24 | } 25 | } 26 | 27 | const std::string k_tcp_service = "tcp connections"; 28 | const std::string k_tcp_desc = "number of tcp connections"; 29 | 30 | } 31 | 32 | riemann_tcp_pool::riemann_tcp_pool(size_t thread_num, raw_msg_fn_t raw_msg_fn, 33 | instrumentation::instrumentation & instr) 34 | : 35 | tcp_pool_(thread_num, 36 | {}, 37 | std::bind(&riemann_tcp_pool::create_conn, this, _1, _2, _3), 38 | std::bind(&riemann_tcp_pool::data_ready, this, _1, _2)), 39 | raw_msg_fn_(raw_msg_fn), 40 | connection_gauge_(instr.add_gauge(k_tcp_service, k_tcp_desc)), 41 | connections_(thread_num) 42 | { 43 | tcp_pool_.start_threads(); 44 | } 45 | 46 | void riemann_tcp_pool::add_client(int fd) { 47 | tcp_pool_.add_client(fd); 48 | } 49 | 50 | void riemann_tcp_pool::stop() { 51 | VLOG(3) << "stop()"; 52 | 53 | tcp_pool_.stop_threads(); 54 | } 55 | 56 | void riemann_tcp_pool::create_conn(int fd, async_loop & loop, 57 | tcp_connection & conn) 58 | { 59 | 60 | auto & fd_conn = connections_[loop.id()]; 61 | 62 | fd_conn.insert({fd, riemann_tcp_connection(conn, raw_msg_fn_)}); 63 | 64 | connection_gauge_.incr_fn(1); 65 | } 66 | 67 | void riemann_tcp_pool::data_ready(async_fd & async, tcp_connection & tcp_conn) { 68 | 69 | auto & fd_conn = connections_[async.loop().id()]; 70 | 71 | auto it = fd_conn.find(async.fd()); 72 | CHECK(it != fd_conn.end()) << "fd not found"; 73 | 74 | auto & riemann_conn = it->second; 75 | 76 | riemann_conn.callback(async); 77 | 78 | async.set_mode(conn_to_mode(tcp_conn)); 79 | 80 | if (tcp_conn.close_connection) { 81 | fd_conn.erase(it); 82 | connection_gauge_.decr_fn(1); 83 | } 84 | 85 | } 86 | 87 | riemann_tcp_pool::~riemann_tcp_pool() { 88 | tcp_pool_.stop_threads(); 89 | } 90 | -------------------------------------------------------------------------------- /src/streams/stream_infra.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace { 8 | 9 | streams_t join(const streams_t & left, const streams_t & right) { 10 | 11 | streams_t stream; 12 | 13 | stream.insert(stream.end(), left.begin(), left.end()); 14 | stream.insert(stream.end(), right.begin(), right.end()); 15 | 16 | return std::move(stream); 17 | } 18 | 19 | fwd_new_stream_fn_t forward(streams_t::iterator begin, streams_t::iterator end) 20 | { 21 | 22 | streams_t s; 23 | std::copy(begin, end, back_inserter(s)); 24 | 25 | return [=]() mutable -> forward_fn_t 26 | { 27 | 28 | init_streams(s); 29 | 30 | return [=](next_events_t events) 31 | { 32 | for (const auto & event : events) { 33 | push_event(s, event); 34 | } 35 | }; 36 | 37 | }; 38 | 39 | } 40 | 41 | } 42 | 43 | streams_t create_stream(const on_event_fn_t fn) { 44 | 45 | return {{[=](fwd_new_stream_fn_t){ return fn; }, fn}}; 46 | 47 | } 48 | 49 | streams_t create_stream(const on_init_simple_fn_t fn) { 50 | 51 | return {{[=](fwd_new_stream_fn_t){ return fn(); }, {}}}; 52 | 53 | } 54 | 55 | streams_t create_stream(const on_init_fn_t fn) { 56 | 57 | return {{fn, {}}}; 58 | 59 | } 60 | 61 | 62 | 63 | void init_streams(streams_t & streams) { 64 | 65 | for (size_t j = 0, i = streams.size() - 1; j < streams.size() ; i--, j++) { 66 | 67 | auto & node = streams[i]; 68 | 69 | node.on_event = node.on_init(forward(streams.end() - j , streams.end())); 70 | 71 | } 72 | 73 | } 74 | 75 | streams_t operator>>(const streams_t & left, const streams_t & right) { 76 | return std::move(join(left, right)); 77 | } 78 | 79 | next_events_t push_event(const streams_t & stream, const Event & event) { 80 | 81 | next_events_t next_events; 82 | 83 | for (const auto & node: stream) { 84 | 85 | if (next_events.empty()) { // First iteration 86 | 87 | next_events = node.on_event(event); 88 | 89 | } else { 90 | 91 | const auto aux_next_events(next_events); 92 | next_events.clear(); 93 | 94 | for (const auto & e : aux_next_events) { 95 | 96 | const auto res = node.on_event(e); 97 | std::copy(begin(res), end(res), back_inserter(next_events)); 98 | 99 | } 100 | 101 | } 102 | 103 | if (next_events.empty()) { 104 | return {}; 105 | } 106 | 107 | } 108 | 109 | return next_events; 110 | } 111 | -------------------------------------------------------------------------------- /src/external/pagerduty_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | using namespace std::placeholders; 7 | 8 | namespace { 9 | 10 | const std::string k_pd_url = "https://events.pagerduty.com/generic/" 11 | "2010-04-15/create_event.json"; 12 | 13 | struct pd_extra { 14 | const std::string pg_key; 15 | pagerduty_pool::pd_action action; 16 | }; 17 | 18 | 19 | std::string pd_action_to_string(const pagerduty_pool::pd_action action) { 20 | 21 | if (action == pagerduty_pool::pd_action::trigger) { 22 | return "trigger"; 23 | } else if (action == pagerduty_pool::pd_action::acknowledge) { 24 | return "acknowledge"; 25 | } else { 26 | return "resolve"; 27 | } 28 | 29 | } 30 | 31 | }; 32 | 33 | pagerduty_pool::pagerduty_pool(const size_t thread_num, const bool enable_debug) 34 | : 35 | curl_pool_( 36 | thread_num, 37 | std::bind(&pagerduty_pool::curl_event, this, _1, _2, _3) 38 | ), 39 | enable_debug_(enable_debug) 40 | { 41 | } 42 | 43 | 44 | void pagerduty_pool::push_event(const pd_action action, 45 | const std::string pd_key, const Event & event) 46 | { 47 | VLOG(3) << "push_event() key: " << pd_key; 48 | 49 | pd_extra extra = {pd_key, action}; 50 | curl_pool_.push_event(event, extra); 51 | 52 | } 53 | 54 | void pagerduty_pool::curl_event(const queued_event_t queued_event, 55 | const std::shared_ptr easy, 56 | std::function&) 57 | { 58 | 59 | const auto extra = boost::any_cast(queued_event.extra); 60 | const auto & e = queued_event.event; 61 | 62 | Json::Value event; 63 | 64 | std::string desc = e.host() + " " + e.service() + " is " + e.state() 65 | + "(" + e.metric_to_str() + ")"; 66 | 67 | event ["incident_key"] = Json::Value(e.host() + "/" + e.service()); 68 | event["service_key"] = Json::Value(extra.pg_key); 69 | event["event_type"] = Json::Value(pd_action_to_string(extra.action)); 70 | event["description"] = desc; 71 | event["details"] = e.json_str(); 72 | 73 | const std::string json_str(event.toStyledString()); 74 | 75 | VLOG(3) << "sending: " << json_str; 76 | 77 | curl_easy_setopt(easy.get(), CURLOPT_URL, k_pd_url.c_str()); 78 | curl_easy_setopt(easy.get(), CURLOPT_COPYPOSTFIELDS, json_str.c_str()); 79 | curl_easy_setopt(easy.get(), CURLOPT_VERBOSE, enable_debug_); 80 | 81 | } 82 | 83 | void pagerduty_pool::stop() { 84 | VLOG(3) << "stop()"; 85 | 86 | curl_pool_.stop(); 87 | } 88 | -------------------------------------------------------------------------------- /src/pool/executor_thread_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace { 5 | 6 | const std::string k_exec_pool_service = "executor pool queue size"; 7 | const std::string k_exec_pool_desc = "number of pending tasks"; 8 | const size_t k_stop_attempts = 50; 9 | const size_t k_stop_interval_check_ms = 100; 10 | const size_t k_max_queue_size = 1e+7; 11 | 12 | } 13 | 14 | executor_thread_pool::executor_thread_pool( 15 | instrumentation::instrumentation & instr, 16 | const config & conf) 17 | : 18 | task_guague_(instr.add_gauge(k_exec_pool_service, k_exec_pool_desc)), 19 | finished_threads_(conf.executor_pool_size, 0), 20 | next_thread_(0), 21 | tasks_(conf.executor_pool_size) 22 | { 23 | 24 | auto run_fn = [=](const int i) 25 | { 26 | run_tasks(i); 27 | }; 28 | 29 | 30 | for (size_t i = 0; i < conf.executor_pool_size; i++) { 31 | tasks_[i].set_capacity(k_max_queue_size); 32 | threads_.push_back(std::move(std::thread(run_fn, i))); 33 | } 34 | 35 | } 36 | 37 | void executor_thread_pool::add_task(const task_fn_t & task) { 38 | 39 | task_guague_.incr_fn(1); 40 | tasks_[next_thread_].push({task, false}); 41 | 42 | next_thread_ = (next_thread_ + 1) % tasks_.size(); 43 | 44 | } 45 | 46 | void executor_thread_pool::stop() { 47 | 48 | VLOG(3) << "stopping executor_thread_pool"; 49 | 50 | for (size_t i = 0; i < threads_.size(); i++) { 51 | tasks_[i].clear(); 52 | tasks_[i].push({{}, true}); 53 | } 54 | 55 | for (size_t attempts = k_stop_attempts; attempts > 0; attempts--) { 56 | 57 | size_t stopped = 0 ; 58 | 59 | for (const auto & t: finished_threads_) { 60 | if (t) stopped++; 61 | } 62 | 63 | if (stopped == threads_.size()) { 64 | break; 65 | } 66 | 67 | VLOG(3) << "Waiting for " << threads_.size() - stopped << " threads"; 68 | 69 | std::this_thread::sleep_for( 70 | std::chrono::milliseconds(k_stop_interval_check_ms)); 71 | } 72 | 73 | for (size_t i = 0; i < threads_.size(); i++) { 74 | threads_[i].join(); 75 | } 76 | 77 | } 78 | 79 | void executor_thread_pool::run_tasks(const int i) { 80 | 81 | VLOG(3) << "starting executor thread"; 82 | 83 | task_t task; 84 | 85 | while (true) { 86 | 87 | tasks_[i].pop(task); 88 | 89 | if (task.stop) { 90 | break; 91 | } 92 | 93 | task.fn(); 94 | 95 | task_guague_.decr_fn(1); 96 | 97 | } 98 | 99 | finished_threads_[i] = 1; 100 | 101 | VLOG(3) << "run_tasks()--"; 102 | 103 | } 104 | -------------------------------------------------------------------------------- /include/transport/tcp_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_TCP_POOL_H 2 | #define CAVALIERI_TRANSPORT_TCP_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | class tcp_connection; 10 | 11 | /* This callback is run when a thread adds the file descriptor that 12 | * is passed from add_client() to its event loop. 13 | * 14 | * This is executed in the context of the thread that owns the file descriptor. 15 | * 16 | * You can use the passed tcp_connection to write and read from the socket 17 | * using the provided buffers in it. 18 | */ 19 | typedef std::function tcp_create_conn_fn_t; 21 | 22 | 23 | /* This callback is run when a file descriptor is ready to read or write. 24 | * 25 | * This is executed in the context of the thread that owns the file descriptor. 26 | * 27 | */ 28 | typedef std::function tcp_ready_fn_t; 29 | 30 | /* This callback is run when the thread's loop is signaled. 31 | * 32 | * This is executed in the context of the thread that owns the loop. 33 | * 34 | */ 35 | typedef std::function async_fn_t; 36 | 37 | class tcp_pool { 38 | public: 39 | tcp_pool( 40 | size_t thread_num, 41 | hook_fn_t run_fn, 42 | tcp_create_conn_fn_t create_conn_fn, 43 | tcp_ready_fn_t tcp_ready_fn 44 | ); 45 | tcp_pool( 46 | size_t thread_num, 47 | hook_fn_t run_fn, 48 | tcp_create_conn_fn_t create_conn_fn, 49 | tcp_ready_fn_t tcp_ready_fn_t, 50 | async_fn_t async_fn 51 | ); 52 | 53 | /* Async. Can be used from any thread */ 54 | void add_client(const int fd); 55 | void add_client(const size_t loop_id, int fd); 56 | 57 | /* Sync. Must be used for the thread that owns loop_id */ 58 | void add_client_sync(const size_t loop_id, int fd); 59 | void remove_client_sync(const size_t loop_id, int fd); 60 | 61 | void signal_thread(const size_t loop_id); 62 | void signal_threads(); 63 | void start_threads(); 64 | void stop_threads(); 65 | async_loop & loop(const size_t loop_id); 66 | virtual ~tcp_pool(); 67 | 68 | private: 69 | void async_hook(async_loop & loop); 70 | void add_fds(async_loop & loop); 71 | void socket_callback(async_fd & async); 72 | 73 | private: 74 | typedef std::map conn_map_t; 75 | 76 | async_thread_pool async_thread_pool_; 77 | std::vector mutexes_; 78 | std::vector> new_fds_; 79 | std::vector conn_maps_; 80 | tcp_create_conn_fn_t tcp_create_conn_fn_; 81 | tcp_ready_fn_t tcp_ready_fn_; 82 | async_fn_t async_fn_; 83 | }; 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /src/websocket/worker_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | namespace { 6 | 7 | const size_t k_worker_threads = 4; 8 | const long k_check_filter_ms = 2000; 9 | 10 | void process_events(worker_pool::filters_map_t & filters, 11 | std::mutex & filter_mutex, 12 | worker_pool::event_queue_t & queue) 13 | { 14 | 15 | worker_pool::event_t e; 16 | auto filter_fns(filters); 17 | 18 | auto t1 = std::chrono::system_clock::now(); 19 | 20 | while (true) { 21 | 22 | queue.pop(e); 23 | 24 | if (e.stop) { 25 | VLOG(3) << "finishing process_events"; 26 | return; 27 | } 28 | 29 | for (const auto & kv : filter_fns) { 30 | for (const auto & fn : kv.second) { 31 | fn (e.event); 32 | } 33 | } 34 | 35 | auto t2 = std::chrono::system_clock::now(); 36 | auto dt = std::chrono::duration_cast 37 | (t2 - t1).count(); 38 | 39 | if (dt > k_check_filter_ms) { 40 | 41 | t1 = t2; 42 | 43 | std::lock_guard lock(filter_mutex); 44 | 45 | filter_fns = filters; 46 | 47 | } 48 | 49 | } 50 | 51 | } 52 | 53 | } 54 | 55 | 56 | worker_pool::worker_pool() : 57 | stop_(false), has_clients_(false), filters_(), event_queue_(), mutex_(), 58 | worker_threads_(k_worker_threads, 59 | [&]() { process_events(filters_, mutex_, event_queue_); }) 60 | { 61 | 62 | event_queue_.set_capacity(k_max_ws_queue_size); 63 | 64 | } 65 | 66 | void worker_pool::add_event(const Event & event) { 67 | 68 | 69 | if (stop_ || !has_clients_) { 70 | return; 71 | } 72 | 73 | if (!event_queue_.try_push({event, false})) { 74 | LOG(WARNING) << "ws worker_pool queue is full"; 75 | } 76 | 77 | } 78 | 79 | void worker_pool::update_filters(const size_t id, 80 | const event_filters_t filters) { 81 | 82 | VLOG(3) << "updating event filters"; 83 | 84 | { 85 | std::lock_guard lock(mutex_); 86 | filters_[id] = filters; 87 | 88 | bool any_client = false; 89 | for (const auto & kv : filters_) { 90 | if (!kv.second.empty()) { 91 | any_client = true; 92 | break; 93 | } 94 | } 95 | 96 | has_clients_ = any_client; 97 | } 98 | 99 | VLOG(3) << "has_clients_ " << has_clients_; 100 | 101 | } 102 | 103 | void worker_pool::stop() { 104 | 105 | VLOG(3) << "stopping "; 106 | 107 | stop_ = true; 108 | event_queue_.clear(); 109 | 110 | for (size_t i = 0; i < k_worker_threads; i++) { 111 | event_queue_.try_push({{}, true}); 112 | } 113 | 114 | worker_threads_.stop(); 115 | 116 | } 117 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.1.3 TBD 2 | ================ 3 | * Add attrs to event class 4 | * Support 'tagged "foo"' without '=' in grammar 5 | 6 | 0.1.2 2014-09-11 7 | ================ 8 | 9 | * Use just one timer to schedule tasks 10 | * Add ability to reload individual rule libraries 11 | * Add try/catch to streams 12 | 13 | 0.1.1 2014-09-2014 14 | ================== 15 | 16 | * Update README.md with cavalieri-rules function's name 17 | * Enable Pager Duty 18 | 19 | 0.1.0 2014-09-23 20 | ================ 21 | 22 | * Add Event members to README.md 23 | * Fix concurrency issue with graphite and riemann client pool 24 | * Remove libcds depdenency and lock-free streams 25 | 26 | 0.0.7 2014-09-14 27 | ================ 28 | 29 | * Change stream data structures and functions 30 | * BY() macro is not necessary any more 31 | * Add support to replicate streams after by() 32 | * rate() works again 33 | * Make websockets more robust 34 | * Scheduler has its own thread pool for timers 35 | * Timers revamp 36 | * Add percentiles() stream function 37 | * Add has_attribute() stream function 38 | * Make prn() return the same event 39 | * Add null() stream function 40 | * Move predicates to their own file and namespace 41 | * Add changed_state() assuming "ok" as initial state 42 | * Add not_expired() stream function 43 | * Add count() fold function 44 | * Add set_host() function 45 | * Add email() signature for multiple receipients 46 | * Catch also SIGTERM 47 | * Add wrapper class to profobuf event class 48 | 49 | 50 | 0.0.6 2014-08-05 51 | ================ 52 | 53 | * Fix issue with queries mixing int and doubles 54 | * Support "true" as a query and not only "(true)" 55 | * Add internal metrics 56 | * Graphite fixes 57 | * Limit concurrent curl connections 58 | * Use boost::circular to read/write from sockets 59 | * Simplify stream infrastructure 60 | * Use lock-based index 61 | * Add set_description() 62 | * Add also lock-based stream functions 63 | * Use JsonCpp in event_to_json() 64 | 65 | 66 | 0.0.5 2014-05-21 67 | ================ 68 | 69 | * Add multi-threaded libcurl pool to implement HTTP(S) and emails clients. 70 | * Get rid of python interpreter 71 | * Stability fixes 72 | * Set SO_REUSEADDR on listening sockets 73 | 74 | 0.0.4 2014-04-19 75 | ================ 76 | 77 | * Add some convenient stream functions 78 | 79 | 0.0.3 2014-04-19 80 | ================ 81 | 82 | * Add TCP client pool infra 83 | * Add graphite support (new line TCP carbon protocol) 84 | * Add forward support to riemann/cavalieri (TCP protocol) 85 | * Add g_scheduler to g_core 86 | 87 | 0.0.2 2014-04-10 88 | ================ 89 | 90 | * Fix websocket and pubsub 91 | * Add integration test for websocket 92 | * Improve documentation 93 | 94 | 0.0.1 2014-04-06 95 | ================ 96 | 97 | * Initial release 98 | -------------------------------------------------------------------------------- /include/common/event.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_COMMON_EVENT_H 2 | #define CAVALIERI_COMMON_EVENT_H 3 | 4 | #include 5 | 6 | class Event { 7 | public: 8 | Event(); 9 | 10 | Event(const riemann::Event & event); 11 | 12 | Event(const Event &) = default; 13 | 14 | Event copy() const; 15 | 16 | riemann::Event riemann_event() const; 17 | 18 | std::string host() const; 19 | 20 | Event & set_host(const std::string host); 21 | 22 | bool has_host() const; 23 | 24 | Event & clear_host(); 25 | 26 | std::string service() const; 27 | 28 | Event & set_service(const std::string service); 29 | 30 | bool has_service() const; 31 | 32 | Event & clear_service(); 33 | 34 | std::string state() const; 35 | 36 | Event & set_state(const std::string state); 37 | 38 | bool has_state() const; 39 | 40 | Event & clear_state(); 41 | 42 | std::string description() const; 43 | 44 | Event & set_description(const std::string description); 45 | 46 | bool has_description() const; 47 | 48 | Event & clear_description(); 49 | 50 | int64_t time() const; 51 | 52 | Event & set_time(const int64_t time); 53 | 54 | bool has_time() const; 55 | 56 | Event & clear_time(); 57 | 58 | float ttl() const; 59 | 60 | Event & set_ttl(const float ttl); 61 | 62 | bool has_ttl() const; 63 | 64 | Event & clear_ttl(); 65 | 66 | double metric() const; 67 | 68 | Event & set_metric(const double metric); 69 | 70 | bool has_metric() const; 71 | 72 | Event & clear_metric(); 73 | 74 | std::string metric_to_str() const; 75 | 76 | float metric_f() const; 77 | 78 | Event & set_metric_f(const float metric); 79 | 80 | bool has_metric_f() const; 81 | 82 | Event & clear_metric_f(); 83 | 84 | float metric_d() const; 85 | 86 | Event & set_metric_d(const double metric); 87 | 88 | bool has_metric_d() const; 89 | 90 | Event & clear_metric_d(); 91 | 92 | int64_t metric_sint64() const; 93 | 94 | Event & set_metric_sint64(const int64_t metric); 95 | 96 | bool has_metric_sint64() const; 97 | 98 | Event & clear_metric_sint64(); 99 | 100 | std::string value_to_str(const std::string field) const; 101 | 102 | bool has_field_set(const std::string field) const; 103 | 104 | std::string json_str() const; 105 | 106 | bool has_tag(const std::string tag) const; 107 | 108 | Event & add_tag(const std::string tag); 109 | 110 | Event & clear_tags(); 111 | 112 | bool has_attr(const std::string attribute) const; 113 | 114 | std::string attr(const std::string attribute) const; 115 | 116 | Event & set_attr(const std::string attribute, const std::string value); 117 | 118 | std::vector> attrs() const; 119 | 120 | Event & clear_attrs(); 121 | 122 | private: 123 | riemann::Event event_; 124 | 125 | }; 126 | 127 | #endif 128 | -------------------------------------------------------------------------------- /include/transport/curl_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_TRANSPORT_CURL_POOL_H 2 | #define CAVALIERI_TRANSPORT_CURL_POOL_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | typedef struct { 17 | Event event; 18 | boost::any extra; 19 | } queued_event_t; 20 | 21 | typedef std::function, 23 | std::function &)> curl_event_fn_t; 24 | 25 | class curl_pool { 26 | public: 27 | curl_pool(const size_t thread_num, const curl_event_fn_t curl_event_fn); 28 | void push_event(const Event & event, const boost::any extra); 29 | void stop(); 30 | 31 | public: 32 | typedef std::function create_socket_cb_t; 33 | typedef std::function close_socket_cb_t; 34 | typedef std::function set_timer_cb_t; 35 | typedef std::function multi_socket_cb_t; 37 | 38 | private: 39 | void on_ready(async_fd & async, tcp_connection & conn); 40 | void async(async_loop & loop); 41 | void timer(const size_t id); 42 | 43 | void set_fd(const size_t loop_id, const int fd, async_fd::mode mode); 44 | 45 | void check_multi_info(const size_t loop_id); 46 | void multi_timer(const size_t loop_id, const long ms); 47 | void multi_socket_action(const size_t loop_id); 48 | void multi_socket(const size_t loop_id, const int fd, 49 | const int mode, const bool initialized); 50 | 51 | void add_fd(size_t loop_id, CURL* curl_conn, int fd); 52 | void remove_fds(size_t loop_id, CURL* curl_conn); 53 | 54 | void cleanup_conns(const size_t loop_id); 55 | 56 | private: 57 | typedef tbb::concurrent_bounded_queue event_queue_t; 58 | 59 | typedef struct { 60 | std::shared_ptr create_socket_fn; 61 | std::shared_ptr close_socket_fn; 62 | std::function cleanup_fn; 63 | std::unordered_set fds; 64 | } curl_conn_data_t; 65 | 66 | tcp_pool tcp_pool_; 67 | curl_event_fn_t curl_event_fn_; 68 | 69 | std::vector> thread_event_queues_; 70 | 71 | std::vector> curl_conns_; 72 | std::vector> finished_conns_; 73 | 74 | size_t next_thread_; 75 | 76 | std::vector> curl_multis_; 77 | 78 | std::vector> set_timer_cbs_; 79 | std::vector> multi_socket_cbs_; 80 | 81 | std::vector loop_timers_; 82 | }; 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /include/instrumentation/instrumentation.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_INSTRUMENTATION_INSTRUMENTATION_H 2 | #define CAVALIERI_INSTRUMENTATION_INSTRUMENTATION_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace instrumentation { 14 | 15 | using update_rate_fn_t = std::function; 16 | using update_latency_fn_t = std::function; 17 | 18 | typedef struct { 19 | const std::function update_fn; 20 | const std::function incr_fn; 21 | const std::function decr_fn; 22 | } update_gauge_t; 23 | 24 | class instrumentation { 25 | public: 26 | 27 | instrumentation(const config conf); 28 | 29 | update_rate_fn_t add_rate(const std::string service, 30 | const std::string description); 31 | 32 | update_latency_fn_t add_latency(const std::string service, 33 | const std::string description, 34 | std::vector percentiles); 35 | 36 | static std::vector reservoir_to_events(reservoir & reservoir, 37 | std::vector percentiles, 38 | const Event event_base); 39 | 40 | update_gauge_t add_gauge(const std::string service, 41 | const std::string description); 42 | std::vector snapshot(); 43 | 44 | private: 45 | using id_t = unsigned int; 46 | 47 | void update_rate(const id_t id,const unsigned int ticks); 48 | void update_latency(const unsigned int id, const double value); 49 | void update_gauge(const id_t id, unsigned int value); 50 | void incr_gauge(const id_t id, unsigned int value); 51 | void decr_gauge(const id_t id, unsigned int value); 52 | void set_rates(std::vector & events, Event event); 53 | void set_latencies(std::vector & events, Event event); 54 | void set_gauges(std::vector & events, Event event); 55 | 56 | private: 57 | 58 | typedef struct { 59 | std::string service; 60 | std::string description; 61 | class rate rate; 62 | } rate_conf_t; 63 | 64 | typedef struct { 65 | std::string service; 66 | std::string description; 67 | std::vector percentiles; 68 | class reservoir reservoir; 69 | } percentiles_conf_t; 70 | 71 | typedef struct { 72 | std::string service; 73 | std::string description; 74 | class gauge gauge; 75 | } gauge_conf_t; 76 | 77 | 78 | std::atomic rates_id_; 79 | std::atomic latencies_id_; 80 | std::atomic gauges_id_; 81 | 82 | std::vector rates_; 83 | std::vector latencies_; 84 | std::vector gauges_; 85 | 86 | }; 87 | 88 | } 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /include/async/async_loop.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_ASYNC_ASYNC_LOOP_H 2 | #define CAVALIERI_ASYNC_ASYNC_LOOP_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | class async_loop; 9 | 10 | class async_fd { 11 | public: 12 | enum mode { 13 | none, 14 | read, 15 | write, 16 | readwrite 17 | }; 18 | virtual int fd() const = 0; 19 | virtual bool error() const = 0; 20 | virtual void stop() = 0; 21 | virtual bool ready_read() const = 0; 22 | virtual bool ready_write() const = 0; 23 | virtual void set_mode(const mode&) = 0; 24 | virtual async_loop & loop() = 0; 25 | }; 26 | 27 | typedef std::function fd_cb_fn_t; 28 | typedef std::function timer_cb_fn_t; 29 | typedef std::function task_cb_fn_t; 30 | 31 | typedef struct { 32 | size_t loop_id; 33 | uint64_t timer_id; 34 | } timer_id_t; 35 | 36 | class async_loop { 37 | public: 38 | virtual size_t id() const = 0; 39 | virtual void start() = 0; 40 | virtual void stop() = 0; 41 | virtual void signal() = 0; 42 | virtual void add_fd(const int fd, const async_fd::mode mode, 43 | fd_cb_fn_t fd_cb_fn) = 0; 44 | virtual void remove_fd(const int fd) = 0; 45 | virtual void set_fd_mode(const int fd, const async_fd::mode mode) = 0; 46 | 47 | virtual timer_id_t add_once_task( 48 | const std::string lib_namespace, const timer_cb_fn_t, const float t) = 0; 49 | virtual timer_id_t add_periodic_task( 50 | const std::string lib_namespace, const timer_cb_fn_t, const float t) = 0; 51 | virtual void set_task_interval(const timer_id_t, const float t) = 0; 52 | virtual bool remove_task(const timer_id_t) = 0; 53 | virtual void remove_task_lib_namespace(const std::string lib_namespace) = 0; 54 | }; 55 | 56 | typedef std::function async_cb_fn_t; 57 | 58 | class async_events_interface { 59 | public: 60 | virtual void start_loop(size_t loop_id) = 0; 61 | virtual void signal_loop(size_t loop_id) = 0; 62 | virtual void stop_all_loops() = 0; 63 | virtual async_loop & loop(size_t loop_id) = 0; 64 | virtual ~async_events_interface() {}; 65 | }; 66 | 67 | std::unique_ptr make_async_events(size_t, 68 | async_cb_fn_t); 69 | 70 | std::unique_ptr make_async_events(size_t, 71 | async_cb_fn_t, 72 | const float, 73 | timer_cb_fn_t); 74 | 75 | typedef std::function on_new_client_fn_t; 76 | typedef std::function on_signal_fn_t; 77 | 78 | class main_async_loop_interface { 79 | public: 80 | virtual void start() = 0; 81 | virtual void add_tcp_listen_fd(const int fd, on_new_client_fn_t fn) = 0; 82 | virtual void add_periodic_task(task_cb_fn_t task, float interval) = 0; 83 | virtual ~main_async_loop_interface() {}; 84 | }; 85 | 86 | std::unique_ptr make_main_async_loop(); 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /tests/riemann_tcp_connection_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef RIEMANN_TCP_CONNECTION_TEST_CASE 2 | #define RIEMANN_TCP_CONNECTION_TEST_CASE 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mock_async_fd.h" 9 | #include 10 | #include 11 | #include 12 | 13 | extern mock_os_functions mock_os; 14 | 15 | /* 16 | * XXX Temporarily disabled 17 | TEST(riemann_tcp_connection_ok_message_test_case, test) 18 | { 19 | tcp_connection conn(0); 20 | 21 | std::vector sink; 22 | 23 | riemann_tcp_connection rconn (conn, 24 | [&](std::vector m) { sink = m;}); 25 | 26 | mock_async_fd async_fd; 27 | using ::testing::Return; 28 | 29 | EXPECT_CALL(async_fd, ready_read()).WillRepeatedly(Return(true)); 30 | EXPECT_CALL(async_fd, ready_write()).WillRepeatedly(Return(true)); 31 | 32 | Msg msg; 33 | msg.set_ok(true); 34 | msg.add_events()->set_host("foo"); 35 | 36 | mock_os.buffer.clear(); 37 | 38 | auto nsize = htonl(msg.ByteSize()); 39 | 40 | mock_os.buffer.resize(sizeof(nsize) + msg.ByteSize()); 41 | 42 | memcpy(&mock_os.buffer[0], static_cast(&nsize), sizeof(nsize)); 43 | msg.SerializeToArray(&mock_os.buffer[0] + sizeof(nsize), msg.ByteSize()); 44 | 45 | rconn.callback(async_fd); 46 | 47 | // Check the raw msg is forwarded correctly 48 | ASSERT_EQ(msg.ByteSize(), sink.size()); 49 | 50 | Msg msg_from_rconn; 51 | msg_from_rconn.ParseFromArray(&sink[0], sink.size()); 52 | ASSERT_TRUE(msg_from_rconn.ok()); 53 | ASSERT_EQ(1, msg_from_rconn.events_size()); 54 | ASSERT_EQ("foo", msg_from_rconn.events(0).host()); 55 | 56 | // Check riemann_tcp_connection acks the msg 57 | uint32_t response_size; 58 | memcpy(static_cast(&response_size), &mock_os.buffer[0], 4); 59 | response_size = ntohl(response_size); 60 | 61 | Msg msg_ack; 62 | msg_ack.ParseFromArray(&mock_os.buffer[4], response_size); 63 | ASSERT_TRUE(msg_ack.ok()); 64 | } 65 | */ 66 | 67 | TEST(riemann_tcp_connection_broken_size_test_case, test) 68 | { 69 | tcp_connection conn(0); 70 | 71 | std::vector sink; 72 | 73 | riemann_tcp_connection rconn (conn, 74 | [&](std::vector m) { sink = m;}); 75 | 76 | mock_async_fd async_fd; 77 | using ::testing::Return; 78 | 79 | EXPECT_CALL(async_fd, ready_read()).WillRepeatedly(Return(true)); 80 | EXPECT_CALL(async_fd, ready_write()).WillRepeatedly(Return(true)); 81 | 82 | riemann::Msg msg; 83 | msg.set_ok(true); 84 | 85 | mock_os.buffer.clear(); 86 | 87 | auto nsize = htonl(conn.buff_size + 1); 88 | 89 | mock_os.buffer.resize(sizeof(nsize) + msg.ByteSize()); 90 | 91 | memcpy(&mock_os.buffer[0], static_cast(&nsize), sizeof(nsize)); 92 | 93 | rconn.callback(async_fd); 94 | 95 | // Check the connection is closed when header reports a huge msg size 96 | ASSERT_TRUE(conn.close_connection); 97 | } 98 | 99 | #endif 100 | -------------------------------------------------------------------------------- /include/query/expression.h: -------------------------------------------------------------------------------- 1 | #ifndef CAVALIERI_QUERY_EXPRESSION_H 2 | #define CAVALIERI_QUERY_EXPRESSION_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | typedef std::function query_fn_t; 13 | 14 | class query_node 15 | { 16 | public: 17 | virtual query_fn_t evaluate() const = 0; 18 | virtual ~query_node(); 19 | }; 20 | 21 | class query_true : public query_node 22 | { 23 | public: 24 | virtual query_fn_t evaluate() const; 25 | }; 26 | 27 | class query_tagged : public query_node 28 | { 29 | public: 30 | query_tagged(std::string * string); 31 | virtual query_fn_t evaluate() const; 32 | 33 | private: 34 | std::unique_ptr string_; 35 | }; 36 | 37 | class query_field : public query_node 38 | { 39 | public: 40 | query_field(std::string * field, std::string * value, std::string * op); 41 | query_field(std::string * field, const int value, std::string * op); 42 | query_field(std::string * field, const double value, std::string * op); 43 | query_field(std::string * field); 44 | query_fn_t evaluate() const; 45 | 46 | private: 47 | 48 | query_fn_t evaluate(const std::string & value) const; 49 | query_fn_t evaluate(const int & value) const; 50 | query_fn_t evaluate(const double & value) const; 51 | query_fn_t evaluate_nil() const; 52 | 53 | std::unique_ptr op_; 54 | std::unique_ptr field_; 55 | boost::variant, int, double> value_; 56 | }; 57 | 58 | class query_and : public query_node 59 | { 60 | public: 61 | query_and(query_node * left, query_node * right); 62 | query_fn_t evaluate() const; 63 | 64 | private: 65 | std::unique_ptr left_; 66 | std::unique_ptr right_; 67 | }; 68 | 69 | class query_or : public query_node 70 | { 71 | public: 72 | query_or(query_node * left, query_node * right); 73 | query_fn_t evaluate() const; 74 | 75 | private: 76 | std::unique_ptr left_; 77 | std::unique_ptr right_; 78 | }; 79 | 80 | class query_not : public query_node 81 | { 82 | public: 83 | query_not(query_node* right); 84 | query_fn_t evaluate() const; 85 | 86 | private: 87 | std::unique_ptr right_; 88 | }; 89 | 90 | 91 | class query_context 92 | { 93 | public: 94 | void set_expression(query_node * expression); 95 | query_fn_t evaluate(); 96 | 97 | private: 98 | std::unique_ptr expression_; 99 | 100 | }; 101 | 102 | template 103 | bool compare(const T & left, const T & right, const std::string & op) { 104 | 105 | if (op == "=") { 106 | return left == right; 107 | } 108 | 109 | if (op == ">") { 110 | return left > right; 111 | } 112 | 113 | if (op == ">=") { 114 | return left >= right; 115 | } 116 | 117 | if (op == "<") { 118 | return left < right; 119 | } 120 | 121 | if (op == "<=") { 122 | return left <= right; 123 | } 124 | 125 | return false; 126 | } 127 | 128 | #endif 129 | -------------------------------------------------------------------------------- /src/external/real_external.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace { 5 | 6 | const std::string k_forward_service = "forward"; 7 | const std::string k_forward_description = "forward rate"; 8 | 9 | const std::string k_graphite_service = "graphite"; 10 | const std::string k_graphite_description = "graphite rate"; 11 | 12 | const std::string k_pagerduty_service = "pagerduty"; 13 | const std::string k_pagerduty_description = "pagerduty rate"; 14 | 15 | const std::string k_email_service = "email"; 16 | const std::string k_email_description = "email rate"; 17 | 18 | const size_t k_forward_id = 0; 19 | const size_t k_graphite_id = 1; 20 | const size_t k_pagerduty_id = 2; 21 | const size_t k_email_id = 3; 22 | 23 | } 24 | 25 | real_external::real_external(const config conf, 26 | instrumentation::instrumentation & instr) 27 | : external_interface(), 28 | riemann_tcp_client_(conf), 29 | graphite_(conf), 30 | pagerduty_(conf.pagerduty_pool_size, conf.enable_pagerduty_debug), 31 | email_(conf.mail_pool_size, conf.enable_mail_debug) 32 | { 33 | rates_.push_back(instr.add_rate(k_forward_service, 34 | k_forward_description)); 35 | 36 | rates_.push_back(instr.add_rate(k_graphite_service, 37 | k_graphite_description)); 38 | 39 | rates_.push_back(instr.add_rate(k_pagerduty_service, 40 | k_pagerduty_description)); 41 | 42 | rates_.push_back(instr.add_rate(k_email_service, 43 | k_email_description)); 44 | } 45 | 46 | void real_external::forward(const std::string server, const int port, 47 | const Event event) 48 | { 49 | riemann_tcp_client_.push_event(server, port, event); 50 | rates_[k_forward_id](1); 51 | } 52 | 53 | void real_external::graphite(const std::string server, const int port, 54 | const Event event) 55 | { 56 | graphite_.push_event(server, port, event); 57 | rates_[k_graphite_id](1); 58 | } 59 | 60 | void real_external::pager_duty_trigger(const std::string pg_key, 61 | const Event event) 62 | { 63 | pagerduty_.push_event(pagerduty_pool::pd_action::trigger, pg_key, event); 64 | rates_[k_pagerduty_id](1); 65 | } 66 | 67 | void real_external::pager_duty_resolve(const std::string pg_key, 68 | const Event event) 69 | { 70 | pagerduty_.push_event(pagerduty_pool::pd_action::resolve, pg_key, event); 71 | rates_[k_pagerduty_id](1); 72 | } 73 | 74 | void real_external::pager_duty_acknowledge(const std::string pg_key, 75 | const Event event) 76 | { 77 | pagerduty_.push_event(pagerduty_pool::pd_action::acknowledge, pg_key, event); 78 | rates_[k_pagerduty_id](1); 79 | } 80 | 81 | void real_external::email(const std::string server, const std::string from, 82 | const std::vector to, const Event event) 83 | { 84 | email_.push_event(server, from, to, event); 85 | rates_[k_email_id](1); 86 | } 87 | 88 | void real_external::stop() { 89 | 90 | VLOG(3) << "stop()"; 91 | 92 | pagerduty_.stop(); 93 | email_.stop(); 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/folds/folds.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace { 8 | 9 | typedef std::function reduce_fn_t; 10 | typedef std::vector events_t; 11 | typedef std::function fold_fn_t; 12 | 13 | double reduce(const reduce_fn_t & f, const events_t & events) { 14 | 15 | double t(events[0].metric()); 16 | 17 | for (size_t i = 1; i < events.size(); i++) { 18 | 19 | t = f(t, events[i].metric()); 20 | 21 | } 22 | 23 | return t; 24 | } 25 | 26 | double sum_fn(const double & x, const double & y) { 27 | return x + y; 28 | } 29 | 30 | double product_fn(const double & x, const double & y) { 31 | return x * y; 32 | } 33 | 34 | double difference_fn(const double & x, const double & y) { 35 | return x - y; 36 | } 37 | 38 | double max_time(const events_t & events) { 39 | 40 | double max_time = 0; 41 | 42 | for (const auto & e : events) { 43 | if (e.time() > max_time) { 44 | max_time = e.time(); 45 | } 46 | } 47 | 48 | return max_time; 49 | } 50 | 51 | Event fold(const reduce_fn_t & f, const events_t & events) { 52 | 53 | //TODO: Filter nil metric events 54 | if (events.empty()) { 55 | return {}; 56 | } 57 | 58 | return events.front().copy().set_metric(reduce(f, events)) 59 | .set_time(max_time(events)); 60 | 61 | } 62 | 63 | } 64 | 65 | 66 | Event sum(const events_t & events) { 67 | return fold(sum_fn, events); 68 | } 69 | 70 | Event product(const events_t & events) { 71 | return fold(product_fn, events); 72 | } 73 | 74 | Event difference(const events_t & events) { 75 | return fold(difference_fn, events); 76 | } 77 | 78 | Event mean(const events_t & events) { 79 | 80 | if (events.empty()) { 81 | return {}; 82 | } 83 | 84 | Event e(events.front()); 85 | e.set_metric_d(reduce(sum_fn, events) / events.size()); 86 | e.set_time(max_time(events)); 87 | 88 | return e; 89 | } 90 | 91 | Event minimum(const events_t & events) { 92 | 93 | if (events.empty()) { 94 | return {}; 95 | } 96 | 97 | double min = events[0].metric(); 98 | 99 | for (const auto & e: events) { 100 | 101 | auto tmp = e.metric(); 102 | if (tmp < min) { 103 | min = tmp; 104 | } 105 | 106 | } 107 | 108 | Event e(events.front()); 109 | e.set_metric_d(min); 110 | e.set_time(max_time(events)); 111 | 112 | return e; 113 | } 114 | 115 | Event maximum(const events_t & events) { 116 | 117 | if (events.empty()) { 118 | return {}; 119 | } 120 | 121 | double max = events[0].metric(); 122 | 123 | for (const auto & e: events) { 124 | 125 | auto tmp = e.metric(); 126 | if (tmp > max) { 127 | max = tmp; 128 | } 129 | } 130 | 131 | Event e(events.front()); 132 | e.set_metric_d(max); 133 | e.set_time(max_time(events)); 134 | 135 | return e; 136 | } 137 | 138 | Event count(const events_t & events) { 139 | 140 | if (events.empty()) { 141 | return {}; 142 | } 143 | 144 | return events.front().copy().set_metric(events.size()) 145 | .set_time(max_time(events)); 146 | } 147 | 148 | 149 | -------------------------------------------------------------------------------- /src/transport/tcp_connection.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace { 11 | 12 | const size_t k_default_buffer_size = 1024 * (1024 + 512); 13 | 14 | bool read_from_fd(boost::circular_buffer & buffer, 15 | const int & fd) 16 | { 17 | 18 | VLOG(3) << "read_from_fd() free: " << buffer.reserve(); 19 | 20 | unsigned char b[buffer.reserve()]; 21 | 22 | ssize_t nread = g_os_functions.recv(fd, &b[0], buffer.reserve(), 0); 23 | 24 | VLOG(3) << "read bytes: " << nread; 25 | 26 | if (nread < 0) { 27 | 28 | LOG(ERROR) << "read error: " << strerror(errno); 29 | 30 | return false; 31 | 32 | } 33 | 34 | if (nread == 0) { 35 | 36 | VLOG(3) << "peer has disconnected gracefully"; 37 | 38 | return false; 39 | 40 | } 41 | 42 | buffer.insert(buffer.end(), &b[0], &b[0] + nread); 43 | 44 | return true; 45 | 46 | } 47 | 48 | bool write_to_fd(boost::circular_buffer & buffer, 49 | const int & fd) 50 | { 51 | VLOG(3) << "write() up to " << buffer.size() << " bytes"; 52 | 53 | auto p = buffer.linearize(); 54 | auto n = g_os_functions.write(fd, p, buffer.size()); 55 | 56 | if (n < 0) { 57 | LOG(ERROR) << "write error: " << strerror(errno); 58 | return false; 59 | } 60 | 61 | buffer.erase_begin(n); 62 | 63 | return true; 64 | 65 | } 66 | 67 | 68 | 69 | } 70 | 71 | tcp_connection::tcp_connection(int sfd) : 72 | buff_size(k_default_buffer_size), 73 | sfd(sfd), 74 | close_connection(false), 75 | r_buffer(k_default_buffer_size), 76 | w_buffer(k_default_buffer_size) 77 | { 78 | } 79 | 80 | tcp_connection::tcp_connection(int sfd, size_t buff_size): 81 | buff_size(k_default_buffer_size), 82 | sfd(sfd), 83 | close_connection(false), 84 | r_buffer(buff_size), 85 | w_buffer(buff_size) 86 | { 87 | } 88 | 89 | 90 | 91 | bool tcp_connection::read() { 92 | 93 | if ((r_buffer.reserve()) <= 0) { 94 | LOG(ERROR) << "buffer is complete"; 95 | return false; 96 | } 97 | 98 | if (!read_from_fd(r_buffer, sfd)) { 99 | close_connection = true; 100 | return false; 101 | } 102 | 103 | return true; 104 | } 105 | 106 | bool tcp_connection::queue_write(const char *src, const size_t length) { 107 | 108 | VLOG(3) << "queue_write with size: " << length; 109 | 110 | if (w_buffer.reserve() < length) { 111 | LOG(ERROR) << "error write buffer is full"; 112 | return false; 113 | } 114 | 115 | w_buffer.insert(w_buffer.end(), src, src + length); 116 | 117 | return true; 118 | } 119 | 120 | bool tcp_connection::write() { 121 | 122 | if (!write_to_fd(w_buffer, sfd)) { 123 | 124 | close_connection = true; 125 | return false; 126 | 127 | } else { 128 | 129 | return true; 130 | } 131 | 132 | } 133 | 134 | bool tcp_connection::pending_read() const { 135 | return true; 136 | } 137 | 138 | bool tcp_connection::pending_write() const { 139 | return w_buffer.size() > 0; 140 | } 141 | 142 | size_t tcp_connection::read_bytes() const { 143 | return r_buffer.size(); 144 | } 145 | -------------------------------------------------------------------------------- /src/config/config.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | //FIXME conf validation 7 | 8 | const auto k_cores = std::thread::hardware_concurrency(); 9 | 10 | DEFINE_int32(events_port, 5555, "listening port for incoming riemann events"); 11 | 12 | DEFINE_int32(riemann_tcp_pool_size, 4, "number of threads for tcp pool"); 13 | 14 | DEFINE_int32(ws_port, 5556, "websocket listening port to query index"); 15 | 16 | DEFINE_int32(ws_pool_size, 4, "number of threads for websocket pool"); 17 | 18 | DEFINE_int32(executor_pool_size, k_cores, "number of threads for executor pool"); 19 | 20 | DEFINE_int32(index_expire_interval, 60, 21 | "interval in seconds to expire events from index"); 22 | 23 | DEFINE_string(rules_directory, ".", "directory to load rules"); 24 | 25 | DEFINE_int32(pagerduty_pool_size, 1, "number of threads for pagerduty pool"); 26 | 27 | DEFINE_int32(mail_pool_size, 1, "number of threads for mail pool"); 28 | 29 | DEFINE_int32(graphite_pool_size, 4, "number of threads for graphite pool"); 30 | 31 | DEFINE_int32(forward_pool_size, 4, "number of threads for forward pool"); 32 | 33 | DEFINE_bool(enable_mail_debug, false, "enable smtp libcurl debug"); 34 | 35 | DEFINE_bool(enable_pagerduty_debug, false, "enable pagerduty libcurl debug"); 36 | 37 | DEFINE_bool(enable_internal_metrics, false, "enable internal metrics"); 38 | 39 | 40 | config create_config() { 41 | 42 | struct config conf; 43 | 44 | conf.events_port = FLAGS_events_port; 45 | conf.riemann_tcp_pool_size = FLAGS_riemann_tcp_pool_size; 46 | conf.ws_port = FLAGS_ws_port; 47 | conf.ws_pool_size = FLAGS_ws_pool_size; 48 | conf.executor_pool_size = FLAGS_executor_pool_size; 49 | conf.index_expire_interval = FLAGS_index_expire_interval; 50 | conf.rules_directory = FLAGS_rules_directory; 51 | conf.pagerduty_pool_size = FLAGS_pagerduty_pool_size; 52 | conf.mail_pool_size = FLAGS_mail_pool_size; 53 | conf.graphite_pool_size = FLAGS_graphite_pool_size; 54 | conf.forward_pool_size = FLAGS_forward_pool_size; 55 | conf.enable_mail_debug = FLAGS_enable_mail_debug; 56 | conf.enable_pagerduty_debug = FLAGS_enable_pagerduty_debug; 57 | conf.enable_internal_metrics = FLAGS_enable_internal_metrics;; 58 | 59 | return conf; 60 | } 61 | 62 | void log_config(config conf) { 63 | 64 | VLOG(1) << "config:"; 65 | VLOG(1) << "\tevents_port: " << conf.events_port; 66 | VLOG(1) << "\trimeann_tcp_pool_size:: " << conf.riemann_tcp_pool_size; 67 | VLOG(1) << "\tws_port: " << conf.ws_port; 68 | VLOG(1) << "\tws_pool_size: " << conf.ws_pool_size; 69 | VLOG(1) << "\texecutor_pool_size: " << conf.executor_pool_size; 70 | VLOG(1) << "\tindex_expire_interval: " << conf.index_expire_interval; 71 | VLOG(1) << "\trules_directory: " << conf.rules_directory; 72 | VLOG(1) << "\tpagerduty_pool_size: " << conf.pagerduty_pool_size; 73 | VLOG(1) << "\tmail_pool_size: " << conf.mail_pool_size; 74 | VLOG(1) << "\tgraphite_pool_size: " << conf.graphite_pool_size; 75 | VLOG(1) << "\tforward_pool_size: " << conf.forward_pool_size; 76 | VLOG(1) << "\tenable_mail_debug: " << conf.enable_mail_debug; 77 | VLOG(1) << "\tenable_pagerduty_debug: " << conf.enable_pagerduty_debug; 78 | VLOG(1) << "\tenable_internal_metrics: " << conf.enable_internal_metrics; 79 | VLOG(1) << "--"; 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/core/real_core.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | real_core::real_core(const config & conf) 14 | : 15 | config_(conf), 16 | 17 | instrumentation_(conf), 18 | 19 | executor_pool_(instrumentation_, conf), 20 | 21 | main_loop_(make_main_async_loop()), 22 | 23 | scheduler_(new real_scheduler()), 24 | 25 | externals_(new real_external(conf, instrumentation_)), 26 | 27 | streams_(new streams(conf, instrumentation_)), 28 | 29 | pubsub_(new pub_sub()), 30 | 31 | index_(new real_index(*pubsub_, [=](e_t e) { streams_->push_event(e); }, 32 | conf.index_expire_interval, *scheduler_, 33 | instrumentation_, detach_thread)), 34 | 35 | tcp_server_(init_tcp_server(conf, *main_loop_, *streams_, 36 | executor_pool_, instrumentation_)), 37 | 38 | udp_server_(init_udp_server(conf, streams_)), 39 | 40 | ws_server_(init_ws_server(conf, *main_loop_, *pubsub_, *index_)) 41 | { 42 | 43 | if (conf.enable_internal_metrics) { 44 | start_instrumentation(*scheduler_, instrumentation_, *streams_); 45 | } 46 | 47 | } 48 | 49 | void real_core::start() { 50 | 51 | LOG(INFO) << "Brace for impact, starting nuclear core."; 52 | 53 | signal(SIGPIPE, SIG_IGN); 54 | 55 | streams_->reload_rules(); 56 | 57 | main_loop_->start(); 58 | 59 | VLOG(3) << "Stopping services"; 60 | 61 | scheduler_->stop(); 62 | executor_pool_.stop(); 63 | streams_->stop(); 64 | tcp_server_->stop(); 65 | udp_server_->stop(); 66 | ws_server_->stop(); 67 | externals_->stop(); 68 | 69 | LOG(INFO) << "Screw you guys, I'm going home."; 70 | } 71 | 72 | void real_core::add_stream(std::shared_ptr stream) { 73 | 74 | if (stream) { 75 | 76 | sh_streams_.push_back(stream); 77 | 78 | streams_->add_stream(*stream); 79 | 80 | } 81 | 82 | } 83 | 84 | real_core::~real_core() { 85 | VLOG(3) << "~real_core"; 86 | } 87 | 88 | void real_core::reload_rules() { 89 | VLOG(3) << "reload_rules()"; 90 | streams_->reload_rules(); 91 | } 92 | 93 | index_interface & real_core::idx() { 94 | return *index_; 95 | } 96 | 97 | scheduler_interface & real_core::sched() { 98 | return *scheduler_; 99 | } 100 | 101 | external_interface & real_core::externals() { 102 | return *externals_; 103 | } 104 | 105 | void start_core(int argc, char **argv) { 106 | 107 | int orig_argc = argc; 108 | char **orig_argv = copy_args(argc, argv); 109 | 110 | FLAGS_stderrthreshold = 0; 111 | FLAGS_logbuflevel = -1; 112 | 113 | google::ParseCommandLineFlags(&argc, &argv, true); 114 | 115 | google::InitGoogleLogging(argv[0]); 116 | 117 | auto conf = create_config(); 118 | 119 | ld_environment(orig_argv, conf.rules_directory); 120 | 121 | free_args(orig_argc, orig_argv); 122 | 123 | log_config(conf); 124 | 125 | g_core = make_real_core(conf); 126 | 127 | g_core->start(); 128 | 129 | g_core.reset(); 130 | 131 | free_thread_ns(); 132 | } 133 | -------------------------------------------------------------------------------- /src/core/real_core_helper.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace { 7 | 8 | const float k_snapshot_interval = 5; 9 | 10 | } 11 | 12 | void detach_thread(std::function fn) { 13 | std::thread t(fn); 14 | t.detach(); 15 | } 16 | 17 | std::shared_ptr make_real_core(const config conf) { 18 | 19 | return std::dynamic_pointer_cast( 20 | std::make_shared(conf)); 21 | 22 | } 23 | 24 | inline void incoming_event(const std::vector & raw_msg, 25 | streams & streams) 26 | { 27 | riemann::Msg msg; 28 | 29 | if (!msg.ParseFromArray(&raw_msg[0], raw_msg.size())) { 30 | VLOG(2) << "error parsing protobuf payload"; 31 | return; 32 | } 33 | 34 | if (msg.has_query()) { 35 | VLOG(1) << "query messages are not supported yet"; 36 | return; 37 | 38 | } 39 | 40 | streams.process_message(msg); 41 | } 42 | 43 | std::unique_ptr init_tcp_server( 44 | const config & conf, 45 | main_async_loop_interface & loop, 46 | streams & streams, 47 | executor_thread_pool & executor_pool, 48 | instrumentation::instrumentation & instr 49 | ) 50 | { 51 | 52 | auto income_tcp_event = [&](const std::vector & raw_msg) 53 | { 54 | executor_pool.add_task( 55 | [=, &streams]() { incoming_event(raw_msg, streams); }); 56 | }; 57 | 58 | std::unique_ptr tcp_server(new riemann_tcp_pool( 59 | conf.riemann_tcp_pool_size, 60 | income_tcp_event, 61 | instr 62 | )); 63 | 64 | auto ptr_server = tcp_server.get(); 65 | 66 | loop.add_tcp_listen_fd(create_tcp_listen_socket(conf.events_port), 67 | [=](int fd) {ptr_server->add_client(fd); }); 68 | 69 | return tcp_server; 70 | } 71 | 72 | std::unique_ptr init_udp_server( 73 | const config & conf, 74 | std::shared_ptr streams) 75 | { 76 | 77 | auto income_udp_event = [=](const std::vector raw_msg) 78 | { 79 | incoming_event(raw_msg, *streams); 80 | }; 81 | 82 | return std::unique_ptr( 83 | new riemann_udp_pool(conf.events_port, income_udp_event)); 84 | } 85 | 86 | std::unique_ptr init_ws_server( 87 | const config & conf, 88 | main_async_loop_interface & loop, 89 | pub_sub & pubsub, 90 | real_index & index) 91 | { 92 | 93 | std::unique_ptr ws_server(new websocket_pool( 94 | conf.ws_pool_size, pubsub, index)); 95 | 96 | auto ptr_server = ws_server.get(); 97 | 98 | loop.add_tcp_listen_fd(create_tcp_listen_socket(conf.ws_port), 99 | [=](int fd) {ptr_server->add_client(fd); }); 100 | 101 | return ws_server; 102 | } 103 | 104 | void snapshot(instrumentation::instrumentation & inst, streams & streams) { 105 | 106 | for (const auto & event : inst.snapshot()) { 107 | streams.push_event(event); 108 | } 109 | 110 | } 111 | 112 | 113 | void start_instrumentation(scheduler_interface & sched, 114 | instrumentation::instrumentation & instrumentation, 115 | streams & streams) 116 | { 117 | sched.add_periodic_task( 118 | [&]() { snapshot(instrumentation, streams); }, 119 | k_snapshot_interval 120 | ); 121 | } 122 | 123 | 124 | -------------------------------------------------------------------------------- /src/pool/async_thread_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | namespace { 7 | const size_t stop_attempts = 20; 8 | const size_t stop_interval_check_ms = 50; 9 | } 10 | 11 | using namespace std::placeholders; 12 | 13 | async_thread_pool::async_thread_pool(size_t thread_num) : 14 | stop_(false), 15 | thread_num_(thread_num), 16 | next_thread_(0), 17 | finished_threads_(thread_num, false), 18 | async_events_(make_async_events(thread_num, 19 | std::bind(&async_thread_pool::async_callback, 20 | this, _1))) 21 | { 22 | if (thread_num < 1) { 23 | LOG(FATAL) << "Thread number must be greater than 0"; 24 | } 25 | } 26 | 27 | void async_thread_pool::set_run_hook(hook_fn_t hook) { 28 | run_hook_fn_ = hook; 29 | } 30 | 31 | void async_thread_pool::set_async_hook(hook_fn_t hook) { 32 | async_hook_fn_ = hook; 33 | } 34 | 35 | void async_thread_pool::start_threads() { 36 | auto run_fn = [=](size_t i) 37 | { 38 | VLOG(3) << "run_fn: " << i; 39 | this->run(i); 40 | }; 41 | 42 | for (size_t i = 0; i < thread_num_; i++) { 43 | threads_.push_back(std::move(std::thread(run_fn, i))); 44 | } 45 | } 46 | 47 | void async_thread_pool::async_callback(async_loop & loop) { 48 | size_t loop_id = loop.id(); 49 | VLOG(3) << "async_callback loop_id: " << loop_id; 50 | if (!stop_) { 51 | async_hook_fn_(loop); 52 | } else { 53 | loop.stop(); 54 | } 55 | } 56 | 57 | void async_thread_pool::run(const size_t loop_id) { 58 | VLOG(3) << "run() thread id: " << loop_id; 59 | 60 | if (run_hook_fn_) { 61 | run_hook_fn_(async_events_->loop(loop_id)); 62 | } 63 | 64 | async_events_->start_loop(loop_id); 65 | 66 | mutex_.lock(); 67 | finished_threads_[loop_id] = true; 68 | mutex_.unlock(); 69 | VLOG(3) << "run() thread id: " << loop_id << " finished"; 70 | } 71 | 72 | 73 | void async_thread_pool::stop_threads() { 74 | if (stop_) { 75 | VLOG(3) << "stop_threads(): already stopped"; 76 | return; 77 | } 78 | 79 | stop_ = true; 80 | async_events_->stop_all_loops(); 81 | 82 | for (size_t attempts = stop_attempts; attempts > 0; attempts--) { 83 | 84 | size_t stopped = 0 ; 85 | 86 | mutex_.lock(); 87 | for (const auto & t: finished_threads_) { 88 | if (t) stopped++; 89 | } 90 | mutex_.unlock(); 91 | 92 | if (stopped == thread_num_) { 93 | break; 94 | } 95 | 96 | VLOG(3) << "Waiting for " << thread_num_- stopped << " threads"; 97 | 98 | std::this_thread::sleep_for( 99 | std::chrono::milliseconds(stop_interval_check_ms)); 100 | } 101 | 102 | for (size_t i = 0; i < thread_num_; i++) { 103 | threads_[i].join(); 104 | } 105 | 106 | } 107 | 108 | void async_thread_pool::signal_thread(size_t loop_id) { 109 | VLOG(3) << "signal_thread() loop_id: " << loop_id; 110 | 111 | if (loop_id >= thread_num_) { 112 | LOG(FATAL) << "Invalid loop_id"; 113 | return; 114 | } 115 | 116 | async_events_->signal_loop(loop_id); 117 | } 118 | 119 | size_t async_thread_pool::next_thread() { 120 | next_thread_ = (next_thread_ + 1) % thread_num_; 121 | return next_thread_; 122 | } 123 | 124 | async_loop & async_thread_pool::loop(const size_t id) { 125 | return async_events_->loop(id); 126 | } 127 | 128 | async_thread_pool::~async_thread_pool() { 129 | VLOG(3) << "~async_thread_pool()"; 130 | stop_threads(); 131 | } 132 | -------------------------------------------------------------------------------- /tests/ws_connection_test_case.h: -------------------------------------------------------------------------------- 1 | #ifndef WS_CONNECTION_TEST_CASE 2 | #define WS_CONNECTION_TEST_CASE 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mock_async_fd.h" 9 | #include 10 | #include 11 | #include 12 | 13 | extern mock_os_functions mock_os; 14 | 15 | const std::string k_ws_http_header( 16 | "GET /index?subscribed=true&query=(true) HTTP/1.1\r\n" 17 | "Upgrade: websocket\r\n" 18 | "Connection: Upgrade\r\n" 19 | "Host: localhost:5556\r\n" 20 | "Origin: http://localhost:5556\r\n" 21 | "Sec-WebSocket-Key: MPowoRLVQ9qicfX4apcBHg==\r\n" 22 | "Sec-WebSocket-Version: 13\r\n\r\n"); 23 | 24 | const std::string k_expected_response( 25 | "HTTP/1.1 101 Switching Protocols\r\n" 26 | "Upgrade: websocket\r\n" 27 | "Connection: Upgrade\r\n" 28 | "Sec-WebSocket-Accept: x53iN644piHAMydyR2BZ3wqY44U=\r\n\r\n"); 29 | 30 | std::string make_word(const size_t len) { 31 | size_t i = len; 32 | std::string word; 33 | while (i-- > 0) { 34 | word += 'a'; 35 | } 36 | return word; 37 | } 38 | 39 | TEST(ws_connection_ok_message_test_case, test) 40 | { 41 | tcp_connection conn(0); 42 | 43 | ws_connection ws_conn (conn); 44 | 45 | mock_async_fd async_fd; 46 | using ::testing::Return; 47 | 48 | EXPECT_CALL(async_fd, ready_read()).WillRepeatedly(Return(true)); 49 | EXPECT_CALL(async_fd, ready_write()).WillRepeatedly(Return(true)); 50 | 51 | mock_os.buffer.clear(); 52 | mock_os.buffer.reserve(200000); 53 | 54 | std::copy(k_ws_http_header.begin(), k_ws_http_header.end(), 55 | back_inserter(mock_os.buffer)); 56 | 57 | ASSERT_EQ(k_ws_http_header.size(), mock_os.buffer.size()); 58 | 59 | ws_conn.callback(async_fd); 60 | 61 | // Check we upgrade to websocket 62 | ASSERT_EQ( 63 | k_expected_response, 64 | std::string(mock_os.buffer.begin() + k_ws_http_header.size(), 65 | mock_os.buffer.end()) 66 | ); 67 | 68 | ASSERT_TRUE(ws_conn.state() & ws_connection::k_read_frame_header); 69 | 70 | ASSERT_EQ("/index?subscribed=true&query=(true)", ws_conn.uri()); 71 | 72 | // Check we can send frames of size < 125 73 | mock_os.buffer.clear(); 74 | 75 | EXPECT_CALL(async_fd, ready_read()).WillRepeatedly(Return(false)); 76 | 77 | auto word_64 = make_word(64); 78 | ws_conn.send_frame(word_64); 79 | ws_conn.callback(async_fd); 80 | 81 | ASSERT_EQ(0x81, mock_os.buffer[0]); 82 | ASSERT_EQ(0x40, mock_os.buffer[1]); 83 | ASSERT_EQ(word_64.c_str(), 84 | std::string(mock_os.buffer.begin() + 2, mock_os.buffer.end())); 85 | 86 | // Check we can send frames of 125 < size < 2**16 87 | mock_os.buffer.clear(); 88 | 89 | EXPECT_CALL(async_fd, ready_read()).WillRepeatedly(Return(false)); 90 | 91 | auto word_256 = make_word(256); 92 | ws_conn.send_frame(word_256); 93 | ws_conn.callback(async_fd); 94 | 95 | ASSERT_EQ(0x81, mock_os.buffer[0]); 96 | ASSERT_EQ(0x7e, mock_os.buffer[1]); 97 | ASSERT_EQ(0x01, mock_os.buffer[2]); 98 | ASSERT_EQ(0x00, mock_os.buffer[3]); 99 | ASSERT_EQ(word_256.c_str(), 100 | std::string(mock_os.buffer.begin() + 4, mock_os.buffer.end())); 101 | 102 | // Check we can send frames of size > 2**16 103 | mock_os.buffer.clear(); 104 | 105 | auto word_100_000 = make_word(100000); 106 | ws_conn.send_frame(word_100_000); 107 | ws_conn.callback(async_fd); 108 | 109 | ASSERT_EQ(0x81, mock_os.buffer[0]); 110 | ASSERT_EQ(0x7F, mock_os.buffer[1]); 111 | ASSERT_EQ(0x86, mock_os.buffer[2]); 112 | ASSERT_EQ(0xA0, mock_os.buffer[3]); 113 | ASSERT_EQ(word_100_000.c_str(), 114 | std::string(mock_os.buffer.begin() + 10, mock_os.buffer.end())); 115 | } 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /src/predicates/predicates.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | namespace predicates { 8 | 9 | const unsigned int k_default_ttl = 60; 10 | 11 | predicate_t above_eq(const double value) { 12 | return PRED(above_eq(e, value)); 13 | } 14 | 15 | predicate_t above(const double value) { 16 | return PRED(above(e, value)); 17 | } 18 | 19 | predicate_t under_eq(const double value) { 20 | return PRED(under_eq(e, value)); 21 | } 22 | 23 | predicate_t under(const double value) { 24 | return PRED(under(e, value)); 25 | } 26 | 27 | predicate_t state(const std::string state) { 28 | return PRED(e.state() == state); 29 | } 30 | 31 | predicate_t service(const std::string service) { 32 | return PRED(e.service() == service); 33 | } 34 | 35 | predicate_t match(const std::string key, const std::string value) { 36 | return PRED(match(e, key, value)); 37 | } 38 | 39 | predicate_t match_any(const std::string key, 40 | const std::vector values) 41 | { 42 | return PRED( 43 | std::find(values.begin(), values.end(), e.value_to_str(key)) 44 | != values.end() 45 | ); 46 | } 47 | 48 | predicate_t match_re(const std::string key, const std::string value) { 49 | return PRED(match_re(e, key, value)); 50 | } 51 | 52 | predicate_t match_re_any(const std::string key, 53 | const std::vector values) 54 | { 55 | return [=](e_t e) { 56 | 57 | for (const auto & val : values) { 58 | if (match_re(e, key, val)) { 59 | return true; 60 | } 61 | } 62 | 63 | return false; 64 | }; 65 | } 66 | 67 | predicate_t match_like(const std::string key, const std::string value) { 68 | return PRED(match_like(e, key, value)); 69 | } 70 | 71 | predicate_t match_like_any(const std::string key, 72 | const std::vector values) 73 | { 74 | return [=](e_t e) { 75 | 76 | for (const auto & val : values) { 77 | if (match_like(e, key, val)) { 78 | return true; 79 | } 80 | } 81 | 82 | return false; 83 | }; 84 | } 85 | 86 | predicate_t default_true() { 87 | return [](e_t){ return true; }; 88 | } 89 | 90 | bool tagged_any(e_t e, const tags_t& tags) { 91 | for (auto &tag: tags) { 92 | if (e.has_tag(tag)) { 93 | return true; 94 | } 95 | } 96 | return false; 97 | } 98 | 99 | bool tagged_all(e_t e, const tags_t& tags) { 100 | for (auto &tag: tags) { 101 | if (!e.has_tag(tag)) { 102 | return false; 103 | } 104 | } 105 | return true; 106 | } 107 | 108 | bool expired(e_t e) { 109 | auto ttl = e.has_ttl() ? e.ttl() : k_default_ttl; 110 | 111 | if (e.state() == "expired") { 112 | return true; 113 | } 114 | 115 | if (g_core->sched().unix_time() < e.time()) { 116 | return false; 117 | } 118 | 119 | return (g_core->sched().unix_time() - e.time() > ttl); 120 | } 121 | 122 | bool above_eq(e_t e, const double value) { 123 | return (e.metric() >= value); 124 | } 125 | 126 | bool above(e_t e, const double value) { 127 | return (e.metric() > value); 128 | } 129 | 130 | bool under_eq(e_t e, const double value) { 131 | return (e.metric() <= value); 132 | } 133 | 134 | bool under(e_t e, const double value) { 135 | return (e.metric() < value); 136 | } 137 | 138 | bool match(e_t e, const std::string key, const std::string value) { 139 | 140 | return e.value_to_str(key) == value; 141 | 142 | } 143 | 144 | bool match_re(e_t e, const std::string key, const std::string value) { 145 | 146 | return match_regex(e.value_to_str(key), value); 147 | 148 | } 149 | 150 | bool match_like(e_t e, const std::string key, const std::string value) { 151 | 152 | return ::match_like(e.value_to_str(key), value); 153 | 154 | } 155 | 156 | } 157 | -------------------------------------------------------------------------------- /src/riemann_tcp_connection.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace { 10 | 11 | std::vector generate_msg_ok() 12 | { 13 | riemann::Msg msg_ok; 14 | msg_ok.set_ok(true); 15 | 16 | uint32_t nsize = htonl(msg_ok.ByteSize()); 17 | std::vector response (sizeof(nsize) + msg_ok.ByteSize()); 18 | 19 | memcpy(&response[0], static_cast(&nsize), sizeof(nsize)); 20 | 21 | CHECK(msg_ok.SerializeToArray(&response[sizeof(nsize)], msg_ok.ByteSize())) 22 | << "error serialazing msg_ok response"; 23 | 24 | return response; 25 | } 26 | 27 | const std::vector ok_response(generate_msg_ok()); 28 | 29 | bool add_ok_response(tcp_connection & connection) { 30 | VLOG(3) << "adding ok response with size: " << ok_response.size(); 31 | 32 | return connection.queue_write(&ok_response[0], ok_response.size()); 33 | } 34 | 35 | size_t msg_size(tcp_connection & connection) { 36 | 37 | uint32_t header; 38 | 39 | auto p = connection.r_buffer.linearize(); 40 | 41 | std::copy(p, p + 4, reinterpret_cast(&header)); 42 | 43 | connection.r_buffer.erase_begin(4); 44 | 45 | return ntohl(header); 46 | 47 | } 48 | 49 | 50 | } 51 | 52 | riemann_tcp_connection::riemann_tcp_connection( 53 | tcp_connection & tcp_connection, 54 | raw_msg_fn_t raw_msg_fn 55 | ) : 56 | tcp_connection_(tcp_connection), 57 | raw_msg_fn_(raw_msg_fn), 58 | reading_header_(true), 59 | protobuf_size_(0) 60 | { 61 | } 62 | 63 | void riemann_tcp_connection::callback(async_fd & async) { 64 | 65 | if (async.ready_read()) { 66 | read_cb(); 67 | } 68 | 69 | if (tcp_connection_.close_connection) { 70 | return; 71 | } 72 | 73 | if (async.ready_write()) { 74 | write_cb(); 75 | } 76 | } 77 | 78 | void riemann_tcp_connection::read_cb() { 79 | 80 | if (!tcp_connection_.read()) { 81 | 82 | return; 83 | 84 | } 85 | 86 | size_t bytes; 87 | 88 | do { 89 | 90 | bytes = tcp_connection_.read_bytes(); 91 | 92 | if (reading_header_) { 93 | 94 | read_header(); 95 | 96 | } else { 97 | 98 | read_message(); 99 | 100 | } 101 | 102 | } while (bytes != tcp_connection_.read_bytes()); 103 | 104 | } 105 | 106 | void riemann_tcp_connection::write_cb() { 107 | 108 | VLOG(3) << "write_cb(): " << tcp_connection_.pending_write(); 109 | 110 | if (!tcp_connection_.pending_write()) { 111 | return; 112 | } 113 | 114 | tcp_connection_.write(); 115 | 116 | } 117 | 118 | void riemann_tcp_connection::read_header() { 119 | 120 | VLOG(3) << "reading header"; 121 | 122 | if (tcp_connection_.read_bytes() < 4) { 123 | return; 124 | } 125 | 126 | protobuf_size_ = msg_size(tcp_connection_); 127 | 128 | if (protobuf_size_ + 4 > tcp_connection_.buff_size) { 129 | LOG(ERROR) << "msg too big: " << protobuf_size_; 130 | tcp_connection_.close_connection = true; 131 | return; 132 | } 133 | 134 | reading_header_ = false; 135 | } 136 | 137 | void riemann_tcp_connection::read_message() { 138 | 139 | if (tcp_connection_.read_bytes() < protobuf_size_) { 140 | return; 141 | } 142 | 143 | /* We have a complete message */ 144 | 145 | if (!add_ok_response(tcp_connection_)) { 146 | VLOG(3) << "write buffer is full"; 147 | tcp_connection_.close_connection = true; 148 | return; 149 | } 150 | 151 | 152 | std::vector msg(protobuf_size_); 153 | 154 | auto p = tcp_connection_.r_buffer.linearize(); 155 | 156 | std::copy(p, p + protobuf_size_, msg.begin()); 157 | 158 | tcp_connection_.r_buffer.erase_begin(protobuf_size_); 159 | 160 | /* Process message */ 161 | raw_msg_fn_(std::move(msg)); 162 | 163 | /* State transtion */ 164 | reading_header_ = true; 165 | 166 | } 167 | -------------------------------------------------------------------------------- /src/rule_tester_util.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "rule_tester_util.h" 5 | #include 6 | 7 | namespace { 8 | bool compare_time_events(const Event & left, const Event & right) { 9 | return (left.time() < right.time()); 10 | } 11 | 12 | std::vector & stable_sort(std::vector & events) { 13 | std::stable_sort(begin(events), end(events), compare_time_events); 14 | return events; 15 | } 16 | 17 | Json::Value event_to_jsoncpp(const Event & e) { 18 | Json::Value event; 19 | event["host"] = Json::Value(e.host()); 20 | event["service"] = Json::Value(e.service()); 21 | event["description"] = Json::Value(e.description()); 22 | event["state"] = Json::Value(e.state()); 23 | event["metric"] = Json::Value(e.metric()); 24 | event["time"] = Json::Value::UInt64(e.time()); 25 | event["state"] = Json::Value(e.state()); 26 | 27 | Json::Value tags(Json::arrayValue); 28 | auto re = e.riemann_event(); 29 | for (auto i = 0; i < re.tags_size(); i++ ) { 30 | tags.append(re.tags(i)); 31 | } 32 | 33 | event["tags"] = tags; 34 | 35 | return event; 36 | } 37 | 38 | Json::Value index_event_to_json(mock_index_events_t idx_event) { 39 | Json::Value event; 40 | event["time"] = Json::Value::UInt64(static_cast(idx_event.first)); 41 | event["event"] = event_to_jsoncpp(idx_event.second); 42 | return event; 43 | } 44 | 45 | Json::Value index_to_json(std::vector idx_events) { 46 | Json::Value events(Json::arrayValue); 47 | for (const auto & idx_event: idx_events) { 48 | events.append(index_event_to_json(idx_event)); 49 | } 50 | return events; 51 | } 52 | 53 | Json::Value external_events_to_json(std::vector ex_event) { 54 | Json::Value events(Json::arrayValue); 55 | 56 | for (const auto & e: ex_event) { 57 | Json::Value event(Json::arrayValue); 58 | event.append(e.external_method); 59 | 60 | Json::Value ex_data(Json::objectValue); 61 | ex_data["time"] = Json::Value::UInt64(static_cast(e.time)); 62 | ex_data["message"]= e.message; 63 | ex_data["extra"] = e.extra; 64 | ex_data["event"] = event_to_jsoncpp(e.e); 65 | event.append(ex_data); 66 | 67 | events.append(event); 68 | } 69 | 70 | return events; 71 | } 72 | } 73 | 74 | 75 | std::vector json_to_events(const std::string json, bool & ok) { 76 | Json::Value root; 77 | Json::Reader reader; 78 | 79 | bool parsingSuccessful = reader.parse(json, root); 80 | if (!parsingSuccessful) { 81 | LOG(FATAL) << "Failed to parse json:" << reader.getFormattedErrorMessages(); 82 | ok = false; 83 | return {}; 84 | } 85 | 86 | std::vector events; 87 | for (const auto e: root) { 88 | Event event; 89 | for (const auto & m: e.getMemberNames()) { 90 | if (m != "tags") { 91 | if (e[m].isString()) { 92 | set_event_value(event, m, e[m].asString(), true); 93 | } else if (e[m].isDouble()) { 94 | set_event_value(event, m, e[m].asDouble(), true); 95 | } else if (e[m].isIntegral()) { 96 | set_event_value(event, m, e[m].asInt(), true); 97 | } 98 | } else { 99 | for (const auto & tag: e[m]) { 100 | event.add_tag(tag.asString()); 101 | } 102 | } 103 | } 104 | events.push_back(event); 105 | } 106 | ok = true; 107 | 108 | return stable_sort(events); 109 | } 110 | 111 | std::string results(std::vector index_events, 112 | std::vector external_events) 113 | { 114 | Json::Value result; 115 | Json::Value reports(Json::arrayValue); 116 | Json::Value index(Json::arrayValue); 117 | 118 | result["reports"] = external_events_to_json(external_events); 119 | result["index"] = index_to_json(index_events); 120 | 121 | return result.toStyledString(); 122 | } 123 | -------------------------------------------------------------------------------- /src/external/mailer_pool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | using namespace std::placeholders; 6 | 7 | namespace { 8 | 9 | struct mailer_extra { 10 | const std::string server; 11 | const std::string from; 12 | const std::vector to; 13 | }; 14 | 15 | typedef std::pair, size_t> payload_t; 16 | 17 | size_t payload_source(void *ptr, size_t size, size_t nmemb, void *userp) 18 | { 19 | 20 | if((size == 0) || (nmemb == 0) || ((size * nmemb) < 1)) { 21 | return 0; 22 | } 23 | 24 | auto payload_data = static_cast(userp); 25 | 26 | size_t copied = payload_data->second; 27 | size_t left = payload_data->first.size() - copied; 28 | size_t to_copy = std::min(size * nmemb, left); 29 | 30 | if (to_copy > 0) { 31 | 32 | memcpy(ptr, static_cast(&payload_data->first[copied]), to_copy); 33 | payload_data->second += to_copy; 34 | 35 | return to_copy; 36 | 37 | } else { 38 | return 0; 39 | } 40 | 41 | } 42 | 43 | std::vector payload_text(const mailer_extra extra, const Event & e) { 44 | 45 | std::string to = "unknown"; 46 | 47 | if (!extra.to.empty()) { 48 | to = extra.to[0]; 49 | } 50 | 51 | std::string subject = e.host() + " " + e.service() + " is " + e.state(); 52 | 53 | if (e.has_metric()) { 54 | subject += " (" + e.metric_to_str() + ")"; 55 | } 56 | 57 | std::string payload = "To: " + to + "\r\n" + 58 | "From: " + extra.from + "\r\n" + 59 | "Subject: " + subject + "\r\n" + 60 | "\r\n" + 61 | e.json_str(); 62 | 63 | return std::vector(payload.begin(), payload.end()); 64 | } 65 | 66 | } 67 | 68 | mailer_pool::mailer_pool(const size_t thread_num, const bool enable_debug) 69 | : 70 | curl_pool_( 71 | thread_num, 72 | std::bind(&mailer_pool::curl_event, this, _1, _2, _3) 73 | ), 74 | enable_debug_(enable_debug) 75 | { 76 | } 77 | 78 | void mailer_pool::push_event(const std::string server, const std::string from, 79 | const std::vector to, 80 | const Event & event) 81 | { 82 | VLOG(3) << "push_event() server: " << server << " from: " << from; 83 | 84 | mailer_extra extra = {server, from, to}; 85 | curl_pool_.push_event(event, extra); 86 | 87 | } 88 | 89 | void mailer_pool::curl_event(const queued_event_t queued_event, 90 | const std::shared_ptr easy, 91 | std::function & clean_fn) 92 | { 93 | 94 | const auto extra = boost::any_cast(queued_event.extra); 95 | const auto & e = queued_event.event; 96 | 97 | std::shared_ptr server( 98 | new std::string("smtp://" + extra.server)); 99 | 100 | std::shared_ptr from(new std::string(extra.from)); 101 | 102 | struct curl_slist *recipients = NULL; 103 | for (const auto & to : extra.to) { 104 | recipients = curl_slist_append(recipients, to.c_str()); 105 | } 106 | 107 | auto payload = std::make_shared( 108 | payload_t({payload_text(extra, e), 0})); 109 | 110 | curl_easy_setopt(easy.get(), CURLOPT_URL, server->c_str()); 111 | curl_easy_setopt(easy.get(), CURLOPT_MAIL_FROM, from->c_str()); 112 | curl_easy_setopt(easy.get(), CURLOPT_MAIL_RCPT, recipients); 113 | curl_easy_setopt(easy.get(), CURLOPT_UPLOAD, 1L); 114 | curl_easy_setopt(easy.get(), CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); 115 | curl_easy_setopt(easy.get(), CURLOPT_READFUNCTION, payload_source); 116 | curl_easy_setopt(easy.get(), CURLOPT_READDATA, payload.get()); 117 | curl_easy_setopt(easy.get(), CURLOPT_VERBOSE, enable_debug_); 118 | 119 | clean_fn = [=]() 120 | { 121 | UNUSED_VAR(server); 122 | UNUSED_VAR(from); 123 | UNUSED_VAR(payload); 124 | if (recipients) { 125 | curl_slist_free_all(recipients); 126 | } 127 | }; 128 | 129 | } 130 | 131 | void mailer_pool::stop() { 132 | VLOG(3) << "stop()"; 133 | 134 | curl_pool_.stop(); 135 | } 136 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | FIND_PACKAGE(Threads REQUIRED) 2 | 3 | SET(CMAKE_CXX_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") 4 | SET(CMAKE_C_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") 5 | # Code coverage options 6 | #ADD_DEFINITIONS( -fprofile-arcs -ftest-coverage -g -O0) 7 | 8 | SET(CAVALIERI_TEST_SRCS 9 | ${CMAKE_SOURCE_DIR}/src/scheduler/mock_scheduler.cpp 10 | ${CMAKE_SOURCE_DIR}/src/common/event.cpp 11 | ${CMAKE_SOURCE_DIR}/src/core/core.cpp 12 | ${CMAKE_SOURCE_DIR}/src/core/mock_core.cpp 13 | ${CMAKE_SOURCE_DIR}/src/index/mock_index.cpp 14 | ${CMAKE_SOURCE_DIR}/src/external/mock_external.cpp 15 | ${CMAKE_SOURCE_DIR}/src/os/mock_os_functions.cpp 16 | ${CMAKE_SOURCE_DIR}/src/os/os_functions.cpp 17 | ${CMAKE_SOURCE_DIR}/src/transport/tcp_connection.cpp 18 | ${CMAKE_SOURCE_DIR}/src/riemann_tcp_connection.cpp 19 | ${CMAKE_SOURCE_DIR}/src/transport/ws_connection.cpp 20 | ${CMAKE_SOURCE_DIR}/src/transport/ws_util.cpp 21 | ${CMAKE_SOURCE_DIR}/src/predicates/predicates.cpp 22 | ${CMAKE_SOURCE_DIR}/src/rules_loader.cpp 23 | ${CMAKE_SOURCE_DIR}/src/streams/lib.cpp 24 | ${CMAKE_SOURCE_DIR}/src/streams/stream_infra.cpp 25 | ${CMAKE_SOURCE_DIR}/src/streams/stream_functions.cpp 26 | ${CMAKE_SOURCE_DIR}/src/streams/stream_functions_lock.cpp 27 | ${CMAKE_SOURCE_DIR}/src/instrumentation/instrumentation.cpp 28 | ${CMAKE_SOURCE_DIR}/src/instrumentation/rate.cpp 29 | ${CMAKE_SOURCE_DIR}/src/instrumentation/reservoir.cpp 30 | ${CMAKE_SOURCE_DIR}/src/instrumentation/gauge.cpp 31 | ${CMAKE_SOURCE_DIR}/src/instrumentation/mem.cpp 32 | ${CMAKE_SOURCE_DIR}/src/pub_sub/pub_sub.cpp 33 | ${CMAKE_SOURCE_DIR}/src/folds/folds.cpp 34 | ${CMAKE_SOURCE_DIR}/src/index/real_index.cpp 35 | ${CMAKE_SOURCE_DIR}/src/pub_sub/pub_sub.cpp 36 | ${CMAKE_SOURCE_DIR}/src/util/util.cpp 37 | ${CMAKE_SOURCE_DIR}/src/query/driver.cpp 38 | ${CMAKE_SOURCE_DIR}/src/query/scanner.cpp 39 | ${CMAKE_SOURCE_DIR}/src/query/parser.cpp 40 | ${CMAKE_SOURCE_DIR}/src/query/expression.cpp 41 | ${CMAKE_SOURCE_DIR}/src/rules/common.cpp 42 | ) 43 | 44 | SET(CAVALIERI_PROTOFILES 45 | ${CMAKE_SOURCE_DIR}/src/proto.proto 46 | ) 47 | 48 | PROTOBUF_GENERATE_CPP(ProtoSources ProtoHeaders ${CAVALIERI_PROTOFILES}) 49 | 50 | INCLUDE_DIRECTORIES( 51 | ${GTEST_INCLUDE_DIRS} 52 | ${CMAKE_CURRENT_BINARY_DIR} 53 | ${CMAKE_BINARY_DIR} 54 | ${CAVALIERI_HDRS} 55 | ${PROTOBUF_INCLUDE_DIR} 56 | ${Glog_INCLUDE_DIR} 57 | ${LibEv_INCLUDE_DIR} 58 | ${ThirdParty_INCLUDE_DIRS} 59 | ${JsonCpp_INCLUDE_DIRS} 60 | ) 61 | 62 | ADD_EXECUTABLE( 63 | test_plan 64 | ${CAVALIERI_TEST_SRCS} 65 | ${ProtoSources} 66 | ${ProtoHeaders} 67 | ${CMAKE_SOURCE_DIR}/tests/test_plan.cpp 68 | ) 69 | 70 | #GTEST_ADD_TESTS( 71 | # test_plan 72 | # ${CMAKE_SOURCE_DIR}/tests/basic_test_case.hpp 73 | # ${CMAKE_SOURCE_DIR}/tests/query_grammar_test_case.hpp 74 | # ${CMAKE_SOURCE_DIR}/tests/mock_scheduler_test_case.hpp 75 | # ${CMAKE_SOURCE_DIR}/tests/streams_test_case.hpp 76 | # ${CMAKE_SOURCE_DIR}/tests/folds_test_case.hpp 77 | # ${CMAKE_SOURCE_DIR}/tests/atom_test_case.hpp 78 | # ${CMAKE_SOURCE_DIR}/tests/rules_common_test_case.hpp 79 | # ${CMAKE_SOURCE_DIR}/tests/tcp_connection_test_case.hpp 80 | # ${CMAKE_SOURCE_DIR}/tests/ws_connection_test_case.hpp 81 | # ${CMAKE_SOURCE_DIR}/tests/riemann_tcp_connection_test_case.hpp 82 | # ${CMAKE_SOURCE_DIR}/tests/pubsub_test_case.hpp 83 | # ${CMAKE_SOURCE_DIR}/tests/index_test_case.hpp 84 | # ) 85 | 86 | TARGET_LINK_LIBRARIES( 87 | test_plan 88 | gcov 89 | ${GTEST_BOTH_LIBRARIES} 90 | ${CMAKE_THREAD_LIBS_INIT} 91 | ${PROTOBUF_LIBRARY} 92 | ${LibEv_LIBRARIES} 93 | ${Glog_LIBRARIES} 94 | ${JsonCpp_LIBRARIES} 95 | ssl 96 | crypto 97 | curl 98 | pthread 99 | tbb 100 | ${ThirdParty_LIBRARIES} 101 | dl 102 | boost_filesystem 103 | boost_system 104 | ) 105 | 106 | add_dependencies(test_plan cds) 107 | add_dependencies(test_plan googlemock) 108 | 109 | 110 | INCLUDE(CodeCoverage) 111 | 112 | SETUP_TARGET_FOR_COVERAGE( 113 | test_plan_coverage 114 | test_plan 115 | coverage 116 | ) 117 | -------------------------------------------------------------------------------- /src/query/stack.hh: -------------------------------------------------------------------------------- 1 | // A Bison parser, made by GNU Bison 3.0.2. 2 | 3 | // Stack handling for Bison parsers in C++ 4 | 5 | // Copyright (C) 2002-2013 Free Software Foundation, Inc. 6 | 7 | // This program is free software: you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by 9 | // the Free Software Foundation, either version 3 of the License, or 10 | // (at your option) any later version. 11 | 12 | // This program is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | 17 | // You should have received a copy of the GNU General Public License 18 | // along with this program. If not, see . 19 | 20 | // As a special exception, you may create a larger work that contains 21 | // part or all of the Bison parser skeleton and distribute that work 22 | // under terms of your choice, so long as that work isn't itself a 23 | // parser generator using the skeleton or a modified version thereof 24 | // as a parser skeleton. Alternatively, if you modify or redistribute 25 | // the parser skeleton itself, you may (at your option) remove this 26 | // special exception, which will cause the skeleton and the resulting 27 | // Bison output files to be licensed under the GNU General Public 28 | // License without this special exception. 29 | 30 | // This special exception was added by the Free Software Foundation in 31 | // version 2.2 of Bison. 32 | 33 | /** 34 | ** \file stack.hh 35 | ** Define the queryparser::stack class. 36 | */ 37 | 38 | #ifndef YY_QUERYPARSER_STACK_HH_INCLUDED 39 | # define YY_QUERYPARSER_STACK_HH_INCLUDED 40 | 41 | # include 42 | 43 | 44 | namespace queryparser { 45 | #line 46 "stack.hh" // stack.hh:133 46 | template > 47 | class stack 48 | { 49 | public: 50 | // Hide our reversed order. 51 | typedef typename S::reverse_iterator iterator; 52 | typedef typename S::const_reverse_iterator const_iterator; 53 | 54 | stack () 55 | : seq_ () 56 | { 57 | } 58 | 59 | stack (unsigned int n) 60 | : seq_ (n) 61 | { 62 | } 63 | 64 | inline 65 | T& 66 | operator[] (unsigned int i) 67 | { 68 | return seq_[seq_.size () - 1 - i]; 69 | } 70 | 71 | inline 72 | const T& 73 | operator[] (unsigned int i) const 74 | { 75 | return seq_[seq_.size () - 1 - i]; 76 | } 77 | 78 | /// Steal the contents of \a t. 79 | /// 80 | /// Close to move-semantics. 81 | inline 82 | void 83 | push (T& t) 84 | { 85 | seq_.push_back (T()); 86 | operator[](0).move (t); 87 | } 88 | 89 | inline 90 | void 91 | pop (unsigned int n = 1) 92 | { 93 | for (; n; --n) 94 | seq_.pop_back (); 95 | } 96 | 97 | void 98 | clear () 99 | { 100 | seq_.clear (); 101 | } 102 | 103 | inline 104 | typename S::size_type 105 | size () const 106 | { 107 | return seq_.size (); 108 | } 109 | 110 | inline 111 | const_iterator 112 | begin () const 113 | { 114 | return seq_.rbegin (); 115 | } 116 | 117 | inline 118 | const_iterator 119 | end () const 120 | { 121 | return seq_.rend (); 122 | } 123 | 124 | private: 125 | stack (const stack&); 126 | stack& operator= (const stack&); 127 | /// The wrapped container. 128 | S seq_; 129 | }; 130 | 131 | /// Present a slice of the top of a stack. 132 | template > 133 | class slice 134 | { 135 | public: 136 | slice (const S& stack, unsigned int range) 137 | : stack_ (stack) 138 | , range_ (range) 139 | { 140 | } 141 | 142 | inline 143 | const T& 144 | operator [] (unsigned int i) const 145 | { 146 | return stack_[range_ - i]; 147 | } 148 | 149 | private: 150 | const S& stack_; 151 | unsigned int range_; 152 | }; 153 | 154 | 155 | } // queryparser 156 | #line 157 "stack.hh" // stack.hh:133 157 | 158 | #endif // !YY_QUERYPARSER_STACK_HH_INCLUDED 159 | -------------------------------------------------------------------------------- /src/query/parser.yy: -------------------------------------------------------------------------------- 1 | %{ /*** C/C++ Declarations ***/ 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "expression.h" 8 | 9 | %} 10 | 11 | /*** yacc/bison Declarations ***/ 12 | 13 | /* Require bison 2.3 or later */ 14 | %require "2.3" 15 | 16 | /* add debug output code to generated parser. disable this for release 17 | * versions. */ 18 | %debug 19 | 20 | /* start symbol is named "start" */ 21 | %start start 22 | 23 | /* write out a header file containing the token defines */ 24 | %defines 25 | 26 | /* use newer C++ skeleton file */ 27 | %skeleton "lalr1.cc" 28 | 29 | /* namespace to enclose parser in */ 30 | %name-prefix="queryparser" 31 | 32 | %define "parser_class_name" "Parser" 33 | 34 | %locations 35 | %initial-action 36 | { 37 | // initialize the initial location object 38 | @$.begin.filename = @$.end.filename = &driver.streamname; 39 | }; 40 | 41 | %parse-param { class driver & driver } 42 | 43 | /* verbose error messages */ 44 | %error-verbose 45 | 46 | %union { 47 | int integerVal; 48 | double doubleVal; 49 | std::string * stringVal; 50 | std::string * unquotedstringVal; 51 | class query_node * querynode; 52 | } 53 | 54 | %token END 0 "end of file" 55 | %token EOL "end of line" 56 | %token INTEGER "integer" 57 | %token DOUBLE "double" 58 | %token STRING "string" 59 | %token EQUAL "equal" 60 | %token GREATER "greater" 61 | %token GREATER_EQ "greater_eq" 62 | %token LESSER "lesser" 63 | %token LESSER_EQ "lesser_eq" 64 | %token LIKE "like" 65 | %token UNQUOTEDSTRING "unquotedstring" 66 | 67 | %token TRUE 68 | %token TAGGED 69 | %token NIL 70 | %left OR AND NOT 71 | 72 | %type all action expr 73 | %type operator 74 | 75 | %{ 76 | 77 | #include "driver.h" 78 | #include "scanner.h" 79 | 80 | #undef yylex 81 | #define yylex driver.lexer->lex 82 | 83 | %} 84 | 85 | %% /*** Grammar Rules ***/ 86 | 87 | all : '(' TRUE ')' 88 | { 89 | $$ = new query_true(); 90 | } 91 | | TRUE 92 | { 93 | $$ = new query_true(); 94 | } 95 | 96 | operator: EQUAL 97 | { 98 | $$ = $1; 99 | } 100 | | 101 | LIKE 102 | { 103 | $$ = $1; 104 | } 105 | | GREATER 106 | { 107 | $$ = $1; 108 | } 109 | | GREATER_EQ 110 | { 111 | $$ = $1; 112 | } 113 | | LESSER 114 | { 115 | $$ = $1; 116 | } 117 | | LESSER_EQ 118 | { 119 | $$ = $1; 120 | } 121 | 122 | action : TAGGED EQUAL STRING 123 | { 124 | delete $2; 125 | $$ = new query_tagged($3); 126 | } 127 | | TAGGED STRING 128 | { 129 | $$ = new query_tagged($2); 130 | } 131 | | UNQUOTEDSTRING EQUAL NIL 132 | { 133 | delete $2; 134 | $$ = new query_field($1); 135 | } 136 | 137 | | UNQUOTEDSTRING operator STRING 138 | { 139 | $$ = new query_field($1, $3, $2); 140 | } 141 | | UNQUOTEDSTRING operator INTEGER 142 | { 143 | $$ = new query_field($1, $3, $2); 144 | } 145 | | UNQUOTEDSTRING operator DOUBLE 146 | { 147 | $$ = new query_field($1, $3, $2); 148 | } 149 | 150 | expr: '(' expr ')' 151 | { 152 | $$ = $2; 153 | } 154 | | expr AND expr 155 | { 156 | $$ = new query_and($1, $3); 157 | } 158 | | expr OR expr 159 | { 160 | $$ = new query_or($1, $3); 161 | } 162 | | NOT expr 163 | { 164 | $$ = new query_not($2); 165 | } 166 | | action 167 | { 168 | $$ = $1; 169 | } 170 | 171 | start : all 172 | { 173 | driver.query.set_expression($1); 174 | } 175 | | expr 176 | { 177 | driver.query.set_expression($1); 178 | } 179 | 180 | %% 181 | 182 | void queryparser::Parser::error(const Parser::location_type& l, 183 | const std::string& m) 184 | { 185 | driver.error(l, m); 186 | } 187 | --------------------------------------------------------------------------------