├── tests ├── 0014-http-posts.objdeps ├── 0015-kafka-producer.objdeps ├── 0005-rb_decoder.objdeps ├── 0007-mac-partitioner.objdeps ├── 0011-rb_http2k-limits.objdeps ├── 0002-meraki.objdeps ├── 0008-http2k-malformed-messages.objdeps ├── 0000-MSE8.objdeps ├── 0001-MSE10.objdeps ├── 0009-http2k-url.objdeps ├── 0010-http2k-organization_enrichment.objdeps ├── 0012-http2k-reload.objdeps ├── 0013-http2k-clients-reports.objdeps ├── 0006-meraki-null-fields.objdeps ├── 0003-MSE8_listener_enrich.objdeps ├── 0004-MSE10_listener_enrich.objdeps ├── assertion_handler.h ├── display_coverage.sh ├── Makefile.tests ├── rb_meraki_tests.h ├── rb_mse_tests.h ├── rb_http2k_tests.c ├── 0009-http2k-url.c ├── assertion_handler.c ├── 0007-mac-partitioner.c ├── 0010-http2k-organization_enrichment.c ├── 0011-rb_http2k-limits.c ├── rb_json_tests.c └── 0006-meraki-null-fields.c ├── src ├── decoder │ ├── mse │ │ ├── Makefile.mk │ │ └── rb_mse.h │ ├── meraki │ │ ├── Makefile.mk │ │ └── rb_meraki.h │ ├── Makefile.mk │ └── rb_http2k │ │ ├── Makefile.mk │ │ ├── rb_http2k_sync_common.h │ │ ├── tommyds │ │ ├── LICENSE │ │ ├── tommylist.c │ │ ├── tommyhash.h │ │ └── tommyhash.c │ │ ├── rb_http2k_curl_handler.h │ │ ├── uuid_database.c │ │ ├── rb_database.h │ │ ├── rb_http2k_parser.h │ │ ├── uuid_database.h │ │ ├── rb_http2k_sensors_database.h │ │ ├── rb_http2k_decoder.h │ │ ├── rb_database.c │ │ ├── rb_http2k_sync_thread.h │ │ └── rb_http2k_curl_handler.c ├── listener │ ├── Makefile.mk │ ├── http.h │ └── socket.h ├── Makefile.mk ├── engine │ ├── Makefile.mk │ ├── parse.h │ ├── engine.h │ ├── rb_addr.h │ ├── engine.c │ ├── rb_addr.c │ ├── n2kafka.c │ └── global_config.h └── util │ ├── Makefile.mk │ ├── rb_json.h │ ├── rb_mac.h │ ├── pair.c │ ├── pair.h │ ├── in_addr_list.h │ ├── kafka_message_list.c │ ├── kafka_message_list.h │ ├── rb_json.c │ ├── util.h │ ├── topic_database.h │ ├── rb_time.h │ ├── rb_mac.c │ ├── in_addr_list.c │ ├── rb_timer.h │ ├── topic_database.c │ └── kafka.h ├── version.h ├── configs_example ├── n2kafka_config_debug.json ├── n2kafka_config_response.json ├── n2kafka_config.json.default ├── n2kafka_config_meraki.json ├── n2kafka_config.json ├── n2kafka_config_rbhttp.json ├── n2kafka_config_mse.json ├── n2kafka_tests_http.json └── n2kafka_tests_tcp.json ├── n2kafka.service ├── mklove ├── modules │ ├── configure.good_cflags │ ├── configure.gitversion │ ├── configure.socket │ ├── configure.builtin │ ├── configure.host │ ├── configure.atomics │ └── configure.cc └── Makefile.base ├── .gitignore ├── packaging └── rpm │ ├── Makefile │ └── n2kafka.spec ├── README.md ├── .travis.yml ├── Makefile ├── configure.n2kafka └── configure /tests/0014-http-posts.objdeps: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/0015-kafka-producer.objdeps: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/decoder/mse/Makefile.mk: -------------------------------------------------------------------------------- 1 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),rb_mse.c) 2 | -------------------------------------------------------------------------------- /version.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | const char *n2kafka_revision; 4 | const char *n2kafka_version; 5 | -------------------------------------------------------------------------------- /src/decoder/meraki/Makefile.mk: -------------------------------------------------------------------------------- 1 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),rb_meraki.c) 2 | 3 | -------------------------------------------------------------------------------- /configs_example/n2kafka_config_debug.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": 1, 3 | "proto": "tcp", 4 | "port": 2057, 5 | "threads": 20 6 | } 7 | -------------------------------------------------------------------------------- /src/listener/Makefile.mk: -------------------------------------------------------------------------------- 1 | THIS_SRCS := \ 2 | http.c \ 3 | socket.c \ 4 | 5 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS)) 6 | 7 | THIS_SRCS := -------------------------------------------------------------------------------- /configs_example/n2kafka_config_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "debug": 1, 3 | "proto": "tcp", 4 | "port": 5050, 5 | "threads": 20, 6 | "response": "response.xml" 7 | } 8 | -------------------------------------------------------------------------------- /src/decoder/Makefile.mk: -------------------------------------------------------------------------------- 1 | SUBDIRS := \ 2 | rb_http2k \ 3 | mse \ 4 | meraki 5 | 6 | include $(addprefix $(CURRENT_N2KAFKA_DIR), $(addsuffix /Makefile.mk, $(SUBDIRS))) 7 | SUBDIRS := -------------------------------------------------------------------------------- /src/Makefile.mk: -------------------------------------------------------------------------------- 1 | SUBDIRS := \ 2 | decoder \ 3 | engine \ 4 | listener \ 5 | util \ 6 | 7 | include $(addprefix $(CURRENT_N2KAFKA_DIR), $(addsuffix /Makefile.mk,$(SUBDIRS))) 8 | 9 | SUBDIRS := -------------------------------------------------------------------------------- /src/engine/Makefile.mk: -------------------------------------------------------------------------------- 1 | THIS_SRCS := \ 2 | engine.c \ 3 | global_config.c \ 4 | n2kafka.c \ 5 | rb_addr.c \ 6 | 7 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS)) 8 | 9 | THIS_SRCS := 10 | -------------------------------------------------------------------------------- /n2kafka.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Network socket to Kafka messaging gateway 3 | 4 | [Service] 5 | ExecStart=/usr/bin/n2kafka /etc/n2kafka_config.json 6 | Restart=always 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /src/util/Makefile.mk: -------------------------------------------------------------------------------- 1 | THIS_SRCS := \ 2 | in_addr_list.c \ 3 | kafka.c \ 4 | kafka_message_list.c \ 5 | pair.c \ 6 | rb_json.c \ 7 | rb_mac.c \ 8 | topic_database.c \ 9 | rb_timer.c \ 10 | 11 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS)) 12 | 13 | THIS_SRCS := 14 | -------------------------------------------------------------------------------- /configs_example/n2kafka_config.json.default: -------------------------------------------------------------------------------- 1 | { 2 | "listeners": [{ 3 | "proto": "http", 4 | "port": 2057, 5 | "mode": "epoll", 6 | "num_threads": 2 7 | }], 8 | "brokers": "localhost", 9 | "topic": "n2kafka_test", 10 | "rdkafka.socket.max.fails": "3", 11 | "rdkafka.socket.keepalive.enable": "true" 12 | } 13 | 14 | -------------------------------------------------------------------------------- /mklove/modules/configure.good_cflags: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Provides some known-good CFLAGS 4 | # Sets: 5 | # CFLAGS 6 | # CXXFLAGS 7 | # CPPFLAGS 8 | 9 | 10 | function checks { 11 | mkl_mkvar_append CPPFLAGS CPPFLAGS \ 12 | "-Wall -Wsign-compare -Wfloat-equal -Wpointer-arith" 13 | 14 | if [[ $MKL_WANT_WERROR = "y" ]]; then 15 | mkl_mkvar_append CPPFLAGS CPPFLAGS \ 16 | "-Werror" 17 | fi 18 | } 19 | -------------------------------------------------------------------------------- /src/decoder/rb_http2k/Makefile.mk: -------------------------------------------------------------------------------- 1 | THIS_SRCS := \ 2 | rb_http2k_decoder.c \ 3 | rb_database.c \ 4 | uuid_database.c \ 5 | rb_http2k_parser.c \ 6 | rb_http2k_sensors_database.c \ 7 | rb_http2k_sync_thread.c \ 8 | rb_http2k_curl_handler.c \ 9 | rb_http2k_organizations_database.c \ 10 | tommyds/tommyhash.c \ 11 | tommyds/tommyhashdyn.c \ 12 | tommyds/tommylist.c \ 13 | 14 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),$(THIS_SRCS)) 15 | -------------------------------------------------------------------------------- /mklove/modules/configure.gitversion: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Sets version variable from git information. 4 | # Optional arguments: 5 | # "as" 6 | # VARIABLE_NAME 7 | # 8 | # Example: Set version in variable named "MYVERSION": 9 | # mkl_require gitversion as MYVERSION 10 | 11 | if [[ $1 == "as" ]]; then 12 | __MKL_GITVERSION_VARNAME="$2" 13 | else 14 | __MKL_GITVERSION_VARNAME="VERSION" 15 | fi 16 | 17 | function checks { 18 | mkl_allvar_set "gitversion" "$__MKL_GITVERSION_VARNAME" "$(git describe --abbrev=6 --tags HEAD --always)" 19 | } 20 | -------------------------------------------------------------------------------- /mklove/modules/configure.socket: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Provides proper compiler flags for socket support, e.g. socket(3). 4 | 5 | function checks { 6 | 7 | local src=" 8 | #include 9 | #include 10 | #include 11 | void foo (void); 12 | void foo (void) { 13 | int s = socket(0, 0, 0); 14 | close(s); 15 | }" 16 | if ! mkl_compile_check socket "" cont CC "" "$src"; then 17 | if mkl_compile_check --ldflags="-lsocket -lnsl" socket_nsl "" fail CC "" "$src"; then 18 | mkl_mkvar_append socket_nsl LIBS "-lsocket -lnsl" 19 | fi 20 | fi 21 | } 22 | -------------------------------------------------------------------------------- /configs_example/n2kafka_config_meraki.json: -------------------------------------------------------------------------------- 1 | { 2 | "listeners": [{ 3 | "proto": "http", 4 | "port": 2057, 5 | "mode": "epoll", 6 | "num_threads": 20 7 | }, { 8 | "proto": "http", 9 | "port": 2056, 10 | "tcp_keepalive": true, 11 | "num_threads": 40, 12 | "decode_as": "meraki" 13 | }], 14 | "brokers": "eugeniodev", 15 | "topic": "rb_flow", 16 | "rdkafka.socket.max.fails": "3", 17 | "rdkafka.socket.keepalive.enable": "true", 18 | "blacklist": ["192.168.101.3"], 19 | "meraki-secrets": { 20 | "r3dB0rder": { 21 | "sensor_name": "MSE_testing", 22 | "sensor_id": 255 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Libraries 8 | *.lib 9 | *.a 10 | 11 | # Shared objects (inc. Windows DLLs) 12 | *.dll 13 | *.so 14 | *.so.* 15 | *.dylib 16 | 17 | # Executables 18 | *.exe 19 | *.out 20 | *.app 21 | *.i*86 22 | *.x86_64 23 | *.hex 24 | 25 | # Dependendes 26 | *.d 27 | 28 | # Coverage 29 | *.gcda 30 | *.gcno 31 | 32 | # mklove 33 | Makefile.config 34 | config.cache 35 | config.h 36 | config.log 37 | config.log.old 38 | version.c 39 | 40 | # Tests 41 | /tests/*.test 42 | /tests/*.xml 43 | /tests/coverage.info 44 | /tests/out 45 | 46 | # Binary 47 | n2kafka 48 | 49 | #build 50 | /packaging/rpm/SOURCES/ 51 | /packaging/rpm/pkgs 52 | -------------------------------------------------------------------------------- /tests/0005-rb_decoder.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0007-mac-partitioner.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0011-rb_http2k-limits.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0002-meraki.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0008-http2k-malformed-messages.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0000-MSE8.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0001-MSE10.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0009-http2k-url.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0010-http2k-organization_enrichment.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0012-http2k-reload.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0013-http2k-clients-reports.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0006-meraki-null-fields.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0003-MSE8_listener_enrich.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /tests/0004-MSE10_listener_enrich.objdeps: -------------------------------------------------------------------------------- 1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 2 | -------------------------------------------------------------------------------- /src/engine/parse.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | -------------------------------------------------------------------------------- /tests/assertion_handler.h: -------------------------------------------------------------------------------- 1 | #include "stdio.h" 2 | #include "stdlib.h" 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | // Public structs 9 | struct assertion_e { 10 | TAILQ_ENTRY(assertion_e) tailq; 11 | char *str; 12 | }; 13 | 14 | struct value_e { 15 | TAILQ_ENTRY(value_e) tailq; 16 | char *str; 17 | size_t len; 18 | }; 19 | 20 | // Private structs 21 | struct assertion_handler_s; 22 | 23 | // Functions 24 | struct assertion_handler_s *assertion_handler_new(); 25 | void assertion_handler_push_assertion( 26 | struct assertion_handler_s *assertion_handler, 27 | struct assertion_e *assertion); 28 | void assertion_handler_push_value(struct assertion_handler_s *assertion_handler, 29 | struct value_e *value); 30 | int assertion_handler_assert(struct assertion_handler_s *assertion_handler); 31 | void assertion_handler_destroy(struct assertion_handler_s *assertion_handler); 32 | -------------------------------------------------------------------------------- /src/engine/engine.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include 22 | #include 23 | 24 | extern int do_shutdown; 25 | 26 | void main_loop(); 27 | -------------------------------------------------------------------------------- /src/util/rb_json.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** Copyright (c) 2014, Eneo Tecnologia 4 | ** Author: Eugenio Perez 5 | ** All rights reserved. 6 | ** 7 | ** This program is free software; you can redistribute it and/or modify 8 | ** it under the terms of the GNU Affero General Public License as 9 | ** published by the Free Software Foundation, either version 3 of the 10 | ** License, or (at your option) any later version. 11 | ** 12 | ** This program is distributed in the hope that it will be useful, 13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | ** GNU Affero General Public License for more details. 16 | ** 17 | ** You should have received a copy of the GNU General Public License 18 | ** along with this program. If not, see . 19 | */ 20 | 21 | #pragma once 22 | 23 | #include 24 | 25 | int json_object_update_missing_copy(json_t *dst, const json_t *src); 26 | -------------------------------------------------------------------------------- /src/engine/rb_addr.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | const char *sockaddr2str(char *buf, size_t buf_size, struct sockaddr *sockaddr); 26 | -------------------------------------------------------------------------------- /src/listener/http.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #ifdef HAVE_LIBMICROHTTPD 22 | #include "engine/global_config.h" 23 | struct http_handler; 24 | struct json_t; 25 | struct listener *create_http_listener(struct json_t *config, 26 | decoder_callback cb,int cb_flags,void *cb_opaque); 27 | #endif 28 | -------------------------------------------------------------------------------- /src/engine/engine.c: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #include "config.h" 20 | #include "util/util.h" 21 | 22 | #include "engine.h" 23 | #include "parse.h" 24 | #include "util/kafka.h" 25 | #include "global_config.h" 26 | 27 | #ifdef HAVE_LIBMICROHTTPD 28 | #include "listener/http.h" 29 | #endif 30 | 31 | int do_shutdown = 0; 32 | 33 | -------------------------------------------------------------------------------- /src/util/rb_mac.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** Copyright (c) 2014, Eneo Tecnologia 4 | ** Author: Eugenio Perez 5 | ** All rights reserved. 6 | ** 7 | ** This program is free software; you can redistribute it and/or modify 8 | ** it under the terms of the GNU Affero General Public License as 9 | ** published by the Free Software Foundation, either version 3 of the 10 | ** License, or (at your option) any later version. 11 | ** 12 | ** This program is distributed in the hope that it will be useful, 13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | ** GNU Affero General Public License for more details. 16 | ** 17 | ** You should have received a copy of the GNU General Public License 18 | ** along with this program. If not, see . 19 | */ 20 | 21 | #pragma once 22 | #include 23 | 24 | /// MAX mac can return parse_mac 25 | #define MAX_MAC 0xffffffffffffL 26 | 27 | #define valid_mac(mac) (mac <= MAX_MAC) 28 | #define INVALID_MAC (MAX_MAC+1) 29 | 30 | // error if return 0xFFFFFFFFFFFFFFFFL 31 | uint64_t parse_mac(const char *mac); 32 | -------------------------------------------------------------------------------- /tests/display_coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COLUMNS=89 4 | INPUT="coverage.out" 5 | 6 | # echo "$INPUT" 7 | BRANCH_COVERAGE=$(grep branches coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1) 8 | LINES_COVERAGE=$(grep lines coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1) 9 | FUNCTIONS_COVERAGE=$(grep functions coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1) 10 | 11 | COVERAGE=$(echo "$BRANCH_COVERAGE" | cut -d '.' -f 1) 12 | if [ $COVERAGE -gt 89 ]; then 13 | COLOR="\e[1m\e[32m" 14 | elif [ $COVERAGE -gt 69 ]; then 15 | COLOR="\e[1m\e[33m" 16 | else 17 | COLOR="\e[1m\e[31m" 18 | fi 19 | 20 | TITLE="CODE COVERAGE" 21 | CONTENT=$(printf "\e[1mLINES: %s%% | FUNCTIONS: %s%% | BRANCH: %s%%\e[0m\n" "$LINES_COVERAGE" "$FUNCTIONS_COVERAGE" "$BRANCH_COVERAGE") 22 | 23 | printf "$COLOR=====================================================================================================\e[0m\n" 24 | printf "$COLOR%*s\e[0m\n" $(((${#TITLE}+$COLUMNS)/2)) "$TITLE" 25 | printf "$COLOR\n" 26 | printf "$COLOR%*s\e[0m\n" $(((${#CONTENT}+$COLUMNS)/2)) "$CONTENT" 27 | printf "$COLOR=====================================================================================================\e[0m\n" 28 | -------------------------------------------------------------------------------- /src/listener/socket.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include "engine/global_config.h" 22 | struct http_handler; 23 | struct json_t; 24 | struct listener *create_socket_listener(struct json_t *config, 25 | decoder_callback callback,int callback_flags,void *callback_opaque); 26 | 27 | #define create_tcp_listener create_socket_listener 28 | #define create_udp_listener create_socket_listener 29 | -------------------------------------------------------------------------------- /src/util/pair.c: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #include "pair.h" 20 | 21 | #include 22 | 23 | void add_key_value_pair(keyval_list_t *list,struct pair *pair) { 24 | TAILQ_INSERT_TAIL(list,pair,entry); 25 | } 26 | 27 | const char *valueof(const keyval_list_t *list,const char *key) { 28 | struct pair *pair = NULL; 29 | TAILQ_FOREACH(pair, list, entry) { 30 | if(0 == strcmp(key,pair->key)) { 31 | return pair->value; 32 | } 33 | } 34 | 35 | return NULL; 36 | } -------------------------------------------------------------------------------- /src/util/pair.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include 22 | 23 | struct pair { 24 | const char *key; 25 | const char *value; 26 | TAILQ_ENTRY(pair) entry; 27 | }; 28 | 29 | typedef TAILQ_HEAD(,pair) keyval_list_t; 30 | #define keyval_list_initializer TAILQ_HEAD_INITIALIZER 31 | #define keyval_list_init TAILQ_INIT 32 | 33 | void add_key_value_pair(keyval_list_t *list,struct pair *pair); 34 | const char *valueof(const keyval_list_t *list,const char *key); 35 | -------------------------------------------------------------------------------- /src/decoder/rb_http2k/rb_http2k_sync_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** Copyright (c) 2014, Eneo Tecnologia 4 | ** Author: Eugenio Perez 5 | ** All rights reserved. 6 | ** 7 | ** This program is free software; you can redistribute it and/or modify 8 | ** This program is free software; you can redistribute it and/or modify 9 | ** it under the terms of the GNU Affero General Public License as 10 | ** published by the Free Software Foundation, either version 3 of the 11 | ** License, or (at your option) any later version. 12 | ** 13 | ** This program is distributed in the hope that it will be useful, 14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | ** GNU Affero General Public License for more details. 17 | ** 18 | ** You should have received a copy of the GNU General Public License 19 | ** along with this program. If not, see . 20 | */ 21 | 22 | #pragma once 23 | 24 | static const char MONITOR_MSG_TIMESTAMP_KEY[] = "timestamp"; 25 | static const char MONITOR_MSG_MONITOR_KEY[] = "monitor"; 26 | static const char MONITOR_MSG_VALUE_KEY[] = "value"; 27 | static const char MONITOR_MSG_ORGANIZATION_UUID_KEY[] = "organization_uuid"; 28 | static const char MONITOR_MSG_N2KAFKA_ID_KEY[] = "n2kafka_id"; 29 | -------------------------------------------------------------------------------- /src/util/in_addr_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include 22 | #include 23 | 24 | typedef struct in_addr_list_s in_addr_list_t; /* FW DECLARATION */ 25 | 26 | /// Init a sockaddr_in list. 27 | in_addr_list_t *in_addr_list_new(); 28 | 29 | /// Add an address to list. 30 | void in_addr_list_add(in_addr_list_t *list,const struct in_addr *addr); 31 | 32 | /// Check if an addr is in list. 33 | int in_addr_list_contains(const in_addr_list_t *list,const struct in_addr *addr); 34 | 35 | /// Deallocate a list. 36 | void in_addr_list_done(in_addr_list_t *list); 37 | -------------------------------------------------------------------------------- /src/decoder/rb_http2k/tommyds/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 18 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 21 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 22 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 23 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /src/util/kafka_message_list.c: -------------------------------------------------------------------------------- 1 | 2 | #include "kafka_message_list.h" 3 | 4 | #include 5 | #include 6 | 7 | struct rd_kafka_message_queue_elm_s { 8 | /** Kafka message */ 9 | rd_kafka_message_t msg; 10 | /** List entry */ 11 | TAILQ_ENTRY(rd_kafka_message_queue_elm_s) list_entry; 12 | }; 13 | 14 | void rd_kafka_msg_q_init(rd_kafka_message_queue_t *q) { 15 | q->count = 0; 16 | TAILQ_INIT(&q->list); 17 | } 18 | 19 | int rd_kafka_msg_q_add(rd_kafka_message_queue_t *q, 20 | const rd_kafka_message_t *msg) { 21 | rd_kafka_message_queue_elm_t *elm = malloc(sizeof(*elm)); 22 | if(elm) { 23 | ++q->count; 24 | memcpy(&elm->msg,msg,sizeof(msg[0])); 25 | TAILQ_INSERT_TAIL(&q->list,elm,list_entry); 26 | } 27 | 28 | return elm != NULL; 29 | } 30 | 31 | static void rd_kafka_msg_q_dump0(rd_kafka_message_queue_t *q, 32 | rd_kafka_message_t *msgs) { 33 | 34 | rd_kafka_message_queue_elm_t *elm = NULL; 35 | size_t i = 0; 36 | 37 | while((elm = TAILQ_FIRST(&q->list))) { 38 | TAILQ_REMOVE(&q->list,elm,list_entry); 39 | if(msgs) { 40 | memcpy(&msgs[i++],&elm->msg,sizeof(msgs[0])); 41 | } 42 | free(elm); 43 | } 44 | 45 | q->count = 0; 46 | } 47 | 48 | void rd_kafka_msg_q_dump(rd_kafka_message_queue_t *q, 49 | rd_kafka_message_t *msgs) { 50 | 51 | rd_kafka_msg_q_dump0(q,msgs); 52 | } 53 | 54 | void rd_kafka_msg_q_clean(rd_kafka_message_queue_t *q) { 55 | rd_kafka_msg_q_dump0(q,NULL); 56 | } -------------------------------------------------------------------------------- /src/util/kafka_message_list.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | struct rd_kafka_message_queue_elm_s; 7 | 8 | /** Queue element with one rdkafka message */ 9 | typedef struct rd_kafka_message_queue_elm_s rd_kafka_message_queue_elm_t; 10 | 11 | /** Kafka message queue */ 12 | typedef struct rd_kafka_message_queue_s { 13 | /** Number of elements */ 14 | size_t count; 15 | /** Actual list */ 16 | TAILQ_HEAD(,rd_kafka_message_queue_elm_s) list; 17 | } rd_kafka_message_queue_t; 18 | 19 | #define rd_kafka_msg_q_size(q) ((q)->count) 20 | 21 | /** Init a message queue 22 | @param q Queue */ 23 | void rd_kafka_msg_q_init(rd_kafka_message_queue_t *q); 24 | 25 | /** Add a message to the message queue 26 | @param q Queue 27 | @param msg Message 28 | @note The message will be copied 29 | @return 1 if ok, 0 if no memory avalable */ 30 | int rd_kafka_msg_q_add(rd_kafka_message_queue_t *q, 31 | const rd_kafka_message_t *msg); 32 | 33 | /** Dump messages to an allocated messages array. It is 34 | suppose to be able to hold as many messages as 35 | rd_kafka_msg_q_size(q) 36 | @param q Queue 37 | @param msgs Messages to dump to 38 | @note after this call, queue will be empty */ 39 | void rd_kafka_msg_q_dump(rd_kafka_message_queue_t *q, 40 | rd_kafka_message_t *msgs); 41 | 42 | /** Discards all messages in queue 43 | @param q Queue 44 | */ 45 | void rd_kafka_msg_q_clean(rd_kafka_message_queue_t *q); -------------------------------------------------------------------------------- /packaging/rpm/Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE_NAME?= n2kafka 2 | 3 | #VERSION?= $(shell git describe --tags --long | sed 's/-/_/g') 4 | VERSION?= $(shell git describe --abbrev=6 --tags HEAD --always | sed 's/-/_/g') 5 | 6 | BUILD_NUMBER?= 1 7 | 8 | MOCK_CONFIG?=default 9 | 10 | RESULT_DIR?=pkgs 11 | 12 | all: rpm 13 | 14 | 15 | SOURCES: 16 | mkdir -p SOURCES 17 | 18 | archive: SOURCES 19 | cd ../../ && \ 20 | git archive --prefix=$(PACKAGE_NAME)-$(VERSION)/ \ 21 | -o packaging/rpm/SOURCES/$(PACKAGE_NAME)-$(VERSION).tar.gz HEAD 22 | 23 | 24 | build_prepare: archive 25 | mkdir -p $(RESULT_DIR) 26 | rm -f $(RESULT_DIR)/$(PACKAGE_NAME)*.rpm 27 | 28 | 29 | srpm: build_prepare 30 | /usr/bin/mock \ 31 | -r $(MOCK_CONFIG) \ 32 | --define "__version $(VERSION)" \ 33 | --define "__release $(BUILD_NUMBER)" \ 34 | --resultdir=$(RESULT_DIR) \ 35 | --buildsrpm \ 36 | --spec=n2kafka.spec \ 37 | --sources=SOURCES 38 | @echo "======= Source RPM now available in $(RESULT_DIR) =======" 39 | 40 | rpm: srpm 41 | /usr/bin/mock \ 42 | -r $(MOCK_CONFIG) \ 43 | --define "__version $(VERSION)"\ 44 | --define "__release $(BUILD_NUMBER)"\ 45 | --resultdir=$(RESULT_DIR) \ 46 | --rebuild $(RESULT_DIR)/$(PACKAGE_NAME)*.src.rpm 47 | @echo "======= Binary RPMs now available in $(RESULT_DIR) =======" 48 | 49 | clean: 50 | rm -rf SOURCES pkgs 51 | 52 | distclean: clean 53 | rm -f build.log root.log state.log available_pkgs installed_pkgs \ 54 | *.rpm *.tar.gz 55 | 56 | -------------------------------------------------------------------------------- /src/util/rb_json.c: -------------------------------------------------------------------------------- 1 | /* 2 | ** 3 | ** Copyright (c) 2014, Eneo Tecnologia 4 | ** Author: Eugenio Perez 5 | ** All rights reserved. 6 | ** 7 | ** This program is free software; you can redistribute it and/or modify 8 | ** it under the terms of the GNU Affero General Public License as 9 | ** published by the Free Software Foundation, either version 3 of the 10 | ** License, or (at your option) any later version. 11 | ** 12 | ** This program is distributed in the hope that it will be useful, 13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | ** GNU Affero General Public License for more details. 16 | ** 17 | ** You should have received a copy of the GNU General Public License 18 | ** along with this program. If not, see . 19 | */ 20 | 21 | #include "rb_json.h" 22 | 23 | #include 24 | 25 | int json_object_update_missing_copy(json_t *dst, const json_t *src) { 26 | const char *key=NULL; 27 | const json_t *value = NULL; 28 | /* We promise that we will not modify this */ 29 | json_t *my_src = NULL; 30 | 31 | if(!json_is_object(src) || !json_is_object(dst)) 32 | return -1; 33 | 34 | memcpy(&my_src,&src,sizeof(my_src)); 35 | 36 | json_object_foreach(my_src,key,value) { 37 | if(NULL == json_object_get(dst,key)){ 38 | json_t *new_json = json_deep_copy(value); 39 | json_object_set_new_nocheck(dst,key,new_json); 40 | } 41 | } 42 | 43 | return 0; 44 | } -------------------------------------------------------------------------------- /packaging/rpm/n2kafka.spec: -------------------------------------------------------------------------------- 1 | Name: n2kafka 2 | Version: %{__version} 3 | Release: %{__release}%{?dist} 4 | 5 | License: GNU AGPLv3 6 | URL: https://gitlab.redborder.lan/dfernandez.ext/n2kafka/repository/archive.tar.gz?ref=redborder 7 | Source0: %{name}-%{version}.tar.gz 8 | 9 | BuildRequires: gcc librd-devel json-c-devel librdkafka-devel libev-devel libmicrohttpd-devel jansson-devel >= 2.7 yajl-devel >= 2.1.0 libcurl-devel >= 7.48 10 | 11 | Summary: Network messages to json/kafka gateway 12 | Group: Development/Libraries/C and C++ 13 | Requires: librd0 json-c librdkafka libev libmicrohttpd jansson >= 2.7 yajl >= 2.1.0 libcurl >= 7.48 14 | %description 15 | %{summary} 16 | 17 | %prep 18 | %setup -qn %{name}-%{version} 19 | 20 | %build 21 | export CFLAGS=" -fcommon" 22 | ./configure --prefix=/usr 23 | make 24 | 25 | %install 26 | DESTDIR=%{buildroot} make install 27 | mkdir -p %{buildroot}/usr/share/n2kafka 28 | install -D -m 644 n2kafka.service %{buildroot}/usr/lib/systemd/system/n2kafka.service 29 | install -D -m 644 configs_example/n2kafka_config.json.default %{buildroot}/usr/share/n2kafka 30 | 31 | %clean 32 | rm -rf %{buildroot} 33 | 34 | %post -p /sbin/ldconfig 35 | %postun -p /sbin/ldconfig 36 | 37 | %files 38 | %defattr(755,root,root) 39 | /usr/bin/n2kafka 40 | %defattr(644,root,root) 41 | /usr/lib/systemd/system/n2kafka.service 42 | /usr/share/n2kafka/n2kafka_config.json.default 43 | 44 | %changelog 45 | * Wed May 11 2016 Juan J. Prieto - 1.0-1 46 | - first spec version 47 | 48 | 49 | -------------------------------------------------------------------------------- /tests/Makefile.tests: -------------------------------------------------------------------------------- 1 | include ../mklove/Makefile.base 2 | 3 | CC ?= cc 4 | TEST_PROGS := ${TEST_SRCS:%.c=%.test} 5 | XML_FILES := ${TEST_SRCS:%.c=%.test.xml} ${TEST_SRCS:%.c=%.test.mem.xml} \ 6 | ${TEST_SRCS:%.c=%.drd.xml} ${TEST_SRCS:%.c=%.test.helgrind.xml} 7 | COV_FILES := ${TEST_SRCS:%.c=%.gcda} ${TEST_SRCS:%.c=%.gcno} 8 | CFLAGS += -g 9 | CFLAGS += -fstack-protector -DFORTIFY_SOURCE=2 --param=ssp-buffer-size=4 -Wformat 10 | CFLAGS += -Wall -Wfloat-equal -Wpointer-arith -O0 \ 11 | -I$(TOPDIR) -I$(TOPDIR)/src 12 | 13 | CPPFLAGS := $(filter-out -I. -I./src,$(CPPFLAGS)) 14 | 15 | LIBS += -lcmocka 16 | 17 | VALGRIND_SUP = --suppressions=valgrind.suppressions 18 | 19 | COVERAGE_INFO = coverage.info 20 | COVERAGE_OUTPUT_DIRECTORY = out 21 | 22 | .PHONY = coverage 23 | .PRECIOUS = $(TEST_PROGS) 24 | 25 | # Profiling 26 | #CFLAGS += -O0 -pg 27 | #LDFLAGS += -pg 28 | 29 | all: test 30 | 31 | %.test: %.c %.objdeps 32 | @printf "\033[1m\033[33mBuilding $@\033[1m\033[0m\n" 33 | @$(CC) $(CFLAGS) $(CPPFLAGS) $< `cat $(subst .c,.objdeps,$<)` -o $@ $(LDFLAGS) $(LIBS) 34 | 35 | coverage: test 36 | @lcov --gcov-tool=gcov -q \ 37 | --rc lcov_branch_coverage=1 --capture \ 38 | --directory ../ --output-file ${COVERAGE_INFO} 39 | @genhtml --branch-coverage ${COVERAGE_INFO} --output-directory ${COVERAGE_OUTPUT_DIRECTORY} > coverage.out 40 | @./display_coverage.sh 41 | 42 | test: $(TEST_PROGS) 43 | @./run_tests.sh 44 | 45 | clean: 46 | rm -f $(TEST_PROGS) $(XML_FILES) $(COV_FILES) $(COVERAGE_INFO) 47 | rm -rf $(COVERAGE_OUTPUT_DIRECTORY) 48 | -------------------------------------------------------------------------------- /src/util/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #pragma once 20 | 21 | #include "librd/rdlog.h" 22 | 23 | #include 24 | 25 | #define RB_UNUSED __attribute__((unused)) 26 | 27 | #ifdef likely 28 | #undef likely 29 | #endif 30 | #define likely(x) __builtin_expect(!!(x), 1) 31 | 32 | #ifdef unlikely 33 | #undef unlikely 34 | #endif 35 | #define unlikely(x) __builtin_expect(!!(x), 0) 36 | 37 | #define rblog(x...) rdlog(x) 38 | 39 | #define fatal(msg...) do{rblog(LOG_ERR,msg);exit(1);}while(0) 40 | 41 | #define swap_ptrs(p1,p2) do{void *aux = p1;p1 = p2;p2 = aux;}while(0) 42 | 43 | static inline char *mystrerror(int _errno,char *buffer,size_t buffer_size){ 44 | strerror_r(_errno,buffer,buffer_size); 45 | return buffer; 46 | } 47 | 48 | -------------------------------------------------------------------------------- /configs_example/n2kafka_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "listeners": [{ 3 | "proto": "http", 4 | "port": 2057, 5 | "mode": "epoll", 6 | "num_threads": 20, 7 | "decode_as": "MSE", 8 | "max_time_threshold": 3600 9 | }, { 10 | "proto": "tcp", 11 | "port": 2056, 12 | "tcp_keepalive": true, 13 | "num_threads": 40, 14 | "decode_as": "MSE", 15 | "max_time_threshold": 3600 16 | }], 17 | "brokers": "eugeniodev", 18 | "topic": "rb_flow", 19 | "rdkafka.socket.max.fails": "3", 20 | "rdkafka.socket.keepalive.enable": "true", 21 | "blacklist": ["192.168.101.3"], 22 | "mse-sensors": [{ 23 | "stream": "stream_test", 24 | "enrichment": { 25 | "sensor_name": "MSE_testing", 26 | "sensor_id": 255 27 | } 28 | }, { 29 | "stream": "streamstream2", 30 | "enrichment": { 31 | "sensor_name": "MSE_testing223", 32 | "sensor_id": 256 33 | } 34 | }, { 35 | "stream": "streamsss", 36 | "enrichment": { 37 | "sensor_name": "MSE_testing4", 38 | "sensor_id": 257, 39 | "market": "Spain", 40 | "organization": "Adamo" 41 | } 42 | }, { 43 | "stream": "streamtest", 44 | "enrichment": { 45 | "sensor_name": "Mse_test", 46 | "sensor_id": 258 47 | } 48 | }, { 49 | "stream": "StreamTest", 50 | "enrichment": { 51 | "sensor_name": "MsE_test", 52 | "sensor_id": 308 53 | } 54 | }, { 55 | "stream": "khgfkghf", 56 | "enrichment": { 57 | "sensor_name": "njgdjhdgf", 58 | "sensor_id": 310 59 | } 60 | }] 61 | } 62 | -------------------------------------------------------------------------------- /src/engine/rb_addr.c: -------------------------------------------------------------------------------- 1 | /* 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L. 3 | ** Author: Eugenio Perez 4 | ** 5 | ** This program is free software; you can redistribute it and/or modify 6 | ** it under the terms of the GNU Affero General Public License as 7 | ** published by the Free Software Foundation, either version 3 of the 8 | ** License, or (at your option) any later version. 9 | ** 10 | ** This program is distributed in the hope that it will be useful, 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ** GNU Affero General Public License for more details. 14 | ** 15 | ** You should have received a copy of the GNU General Public License 16 | ** along with this program. If not, see . 17 | */ 18 | 19 | #include "rb_addr.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | const char *sockaddr2str(char *buf, size_t buf_size, struct sockaddr *sockaddr) { 26 | char errbuf[BUFSIZ]; 27 | 28 | const void *addr_buf = NULL; 29 | const char *ret = NULL; 30 | 31 | switch(sockaddr->sa_family) { 32 | case AF_INET: 33 | addr_buf = &((struct sockaddr_in *)sockaddr)->sin_addr; 34 | break; 35 | case AF_INET6: 36 | addr_buf = &((struct sockaddr_in6 *)sockaddr)->sin6_addr; 37 | break; 38 | default: 39 | break; 40 | } 41 | 42 | if(NULL == addr_buf) { 43 | errno = EAFNOSUPPORT; 44 | } else { 45 | ret = inet_ntop(sockaddr->sa_family, addr_buf,buf,buf_size); 46 | } 47 | 48 | if(NULL == ret) { 49 | strerror_r(errno,errbuf,sizeof(errbuf)); 50 | rdlog(LOG_ERR,"Can't print client address: %s",errbuf); 51 | } 52 | 53 | return ret; 54 | } 55 | -------------------------------------------------------------------------------- /configs_example/n2kafka_config_rbhttp.json: -------------------------------------------------------------------------------- 1 | { 2 | "listeners": [{ 3 | "proto": "http", 4 | "port": 2057, 5 | "mode": "epoll", 6 | "num_threads": 20, 7 | "redborder_uri": true, 8 | "decode_as": "rb_http2k" 9 | }], 10 | "brokers": "kafka", 11 | "n2kafka_id": "n2kafka_test", 12 | "rdkafka.socket.max.fails": "3", 13 | "rdkafka.socket.keepalive.enable": "true", 14 | "blacklist": ["192.168.101.3"], 15 | "rb_http2k_config": { 16 | "sensors_uuids": { 17 | "abc": { 18 | "enrichment": { 19 | "a": 1, 20 | "b": "c", 21 | "d": true, 22 | "e": null 23 | }, 24 | "organization_uuid":"abc_org" 25 | }, 26 | "def": { 27 | "enrichment": { 28 | "f": 1, 29 | "g": "w", 30 | "h": false, 31 | "i": null 32 | }, 33 | "organization_uuid":"def_org" 34 | } 35 | }, 36 | "organizations_uuids": { 37 | "abc_org": { 38 | "enrichment": { 39 | "a_org":10 40 | }, 41 | "limits": { 42 | "bytes": 10240 43 | } 44 | }, 45 | "def_org": { 46 | "enrichment": { 47 | "b_org":20 48 | }, 49 | "limits": { 50 | "bytes": 20480 51 | } 52 | } 53 | }, 54 | "organizations_sync": { 55 | "put_url": "http://localhost:80/", 56 | "topics": ["rb_monitor","rb_event"], 57 | "interval_s": 5, 58 | "clean_on": { 59 | "timestamp_s_mod":5, 60 | "timestamp_s_offset":4 61 | } 62 | }, 63 | "topics": { 64 | "rb_flow": { 65 | "partition_key": "client_mac", 66 | "partition_algo": "mac" 67 | }, 68 | "rb_event": {} 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # n2kafka 2 | 3 | [![Build Status](https://travis-ci.org/redBorder/n2kafka.svg?branch=develop)](https://travis-ci.org/redBorder/n2kafka) 4 | [![Coverage Status](https://coveralls.io/repos/github/redBorder/n2kafka/badge.svg?branch=develop)](https://coveralls.io/github/redBorder/n2kafka?branch=develop) 5 | 6 | Network to kafka translator. It (currently) support conversion from tcp/udp raw 7 | sockets and HTTP POST to kafka messages, doing message-processing if you need 8 | to. 9 | 10 | # Setup 11 | To use it, you only need to do a typical `./configure && make && make install` 12 | 13 | # Usage 14 | ## Basic usage 15 | 16 | In order to send raw tcp messages from port `2056`, to `mymessages` topic, using 17 | `40` threads, you need to use this config file: 18 | ```json 19 | { 20 | "listeners":[ 21 | {"proto":"tcp","port":2056,"num_threads":40} 22 | ], 23 | "brokers":"localhost", 24 | "topic":"mymessages" 25 | } 26 | ``` 27 | 28 | And launch `n2kafka` using `./n2kafka `. You can also use `udp` and 29 | `http` as proto values. If you want to listen in different ports, you can add as 30 | many listeners as you want. 31 | 32 | ## Recommended config parameters 33 | You can also use this parameters in config json root to improve n2kafka 34 | behavior: 35 | - `"blacklist":["192.168.101.3"]`, that will ignore requests of this directions 36 | (useful for load balancers) 37 | - All 38 | [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). 39 | options. If a config option starts with `rdkafka.