├── .gitignore
├── .travis.yml
├── LICENSE
├── Makefile
├── README.md
├── configs_example
    ├── n2kafka_config.json
    ├── n2kafka_config.json.default
    ├── n2kafka_config_debug.json
    ├── n2kafka_config_meraki.json
    ├── n2kafka_config_mse.json
    ├── n2kafka_config_rbhttp.json
    ├── n2kafka_config_response.json
    ├── n2kafka_tests_http.json
    └── n2kafka_tests_tcp.json
├── configure
├── configure.n2kafka
├── mklove
    ├── Makefile.base
    └── modules
    │   ├── configure.atomics
    │   ├── configure.base
    │   ├── configure.builtin
    │   ├── configure.cc
    │   ├── configure.gitversion
    │   ├── configure.good_cflags
    │   ├── configure.host
    │   └── configure.socket
├── n2kafka.service
├── packaging
    └── rpm
    │   ├── Makefile
    │   └── n2kafka.spec
├── src
    ├── Makefile.mk
    ├── decoder
    │   ├── Makefile.mk
    │   ├── meraki
    │   │   ├── Makefile.mk
    │   │   ├── rb_meraki.c
    │   │   └── rb_meraki.h
    │   ├── mse
    │   │   ├── Makefile.mk
    │   │   ├── rb_mse.c
    │   │   └── rb_mse.h
    │   └── rb_http2k
    │   │   ├── Makefile.mk
    │   │   ├── rb_database.c
    │   │   ├── rb_database.h
    │   │   ├── rb_http2k_curl_handler.c
    │   │   ├── rb_http2k_curl_handler.h
    │   │   ├── rb_http2k_decoder.c
    │   │   ├── rb_http2k_decoder.h
    │   │   ├── rb_http2k_organizations_database.c
    │   │   ├── rb_http2k_organizations_database.h
    │   │   ├── rb_http2k_parser.c
    │   │   ├── rb_http2k_parser.h
    │   │   ├── rb_http2k_sensors_database.c
    │   │   ├── rb_http2k_sensors_database.h
    │   │   ├── rb_http2k_sync_common.h
    │   │   ├── rb_http2k_sync_thread.c
    │   │   ├── rb_http2k_sync_thread.h
    │   │   ├── tommyds
    │   │       ├── LICENSE
    │   │       ├── tommychain.h
    │   │       ├── tommyhash.c
    │   │       ├── tommyhash.h
    │   │       ├── tommyhashdyn.c
    │   │       ├── tommyhashdyn.h
    │   │       ├── tommylist.c
    │   │       ├── tommylist.h
    │   │       └── tommytypes.h
    │   │   ├── uuid_database.c
    │   │   └── uuid_database.h
    ├── engine
    │   ├── Makefile.mk
    │   ├── engine.c
    │   ├── engine.h
    │   ├── global_config.c
    │   ├── global_config.h
    │   ├── n2kafka.c
    │   ├── parse.h
    │   ├── rb_addr.c
    │   └── rb_addr.h
    ├── listener
    │   ├── Makefile.mk
    │   ├── http.c
    │   ├── http.h
    │   ├── socket.c
    │   └── socket.h
    └── util
    │   ├── Makefile.mk
    │   ├── in_addr_list.c
    │   ├── in_addr_list.h
    │   ├── kafka.c
    │   ├── kafka.h
    │   ├── kafka_message_list.c
    │   ├── kafka_message_list.h
    │   ├── pair.c
    │   ├── pair.h
    │   ├── rb_json.c
    │   ├── rb_json.h
    │   ├── rb_mac.c
    │   ├── rb_mac.h
    │   ├── rb_time.h
    │   ├── rb_timer.c
    │   ├── rb_timer.h
    │   ├── topic_database.c
    │   ├── topic_database.h
    │   └── util.h
├── tests
    ├── 0000-MSE8.c
    ├── 0000-MSE8.objdeps
    ├── 0001-MSE10.c
    ├── 0001-MSE10.objdeps
    ├── 0002-meraki.c
    ├── 0002-meraki.objdeps
    ├── 0003-MSE8_listener_enrich.c
    ├── 0003-MSE8_listener_enrich.objdeps
    ├── 0004-MSE10_listener_enrich.c
    ├── 0004-MSE10_listener_enrich.objdeps
    ├── 0005-rb_decoder.c
    ├── 0005-rb_decoder.objdeps
    ├── 0006-meraki-null-fields.c
    ├── 0006-meraki-null-fields.objdeps
    ├── 0007-mac-partitioner.c
    ├── 0007-mac-partitioner.objdeps
    ├── 0008-http2k-malformed-messages.c
    ├── 0008-http2k-malformed-messages.objdeps
    ├── 0009-http2k-url.c
    ├── 0009-http2k-url.objdeps
    ├── 0010-http2k-organization_enrichment.c
    ├── 0010-http2k-organization_enrichment.objdeps
    ├── 0011-rb_http2k-limits.c
    ├── 0011-rb_http2k-limits.objdeps
    ├── 0012-http2k-reload.c
    ├── 0012-http2k-reload.objdeps
    ├── 0013-http2k-clients-reports.c
    ├── 0013-http2k-clients-reports.objdeps
    ├── 0014-http-posts.c
    ├── 0014-http-posts.objdeps
    ├── 0015-kafka-producer.c
    ├── 0015-kafka-producer.objdeps
    ├── Makefile.tests
    ├── assertion_handler.c
    ├── assertion_handler.h
    ├── display_coverage.sh
    ├── rb_http2k_tests.c
    ├── rb_json_tests.c
    ├── rb_meraki_tests.h
    ├── rb_mse_tests.h
    ├── run_tests.sh
    └── valgrind.suppressions
└── version.h


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Object files
 2 | *.o
 3 | *.ko
 4 | *.obj
 5 | *.elf
 6 | 
 7 | # Libraries
 8 | *.lib
 9 | *.a
10 | 
11 | # Shared objects (inc. Windows DLLs)
12 | *.dll
13 | *.so
14 | *.so.*
15 | *.dylib
16 | 
17 | # Executables
18 | *.exe
19 | *.out
20 | *.app
21 | *.i*86
22 | *.x86_64
23 | *.hex
24 | 
25 | # Dependendes
26 | *.d
27 | 
28 | # Coverage
29 | *.gcda
30 | *.gcno
31 | 
32 | # mklove
33 | Makefile.config
34 | config.cache
35 | config.h
36 | config.log
37 | config.log.old
38 | version.c
39 | 
40 | # Tests
41 | /tests/*.test
42 | /tests/*.xml
43 | /tests/coverage.info
44 | /tests/out
45 | 
46 | # Binary
47 | n2kafka
48 | 
49 | #build
50 | /packaging/rpm/SOURCES/
51 | /packaging/rpm/pkgs
52 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | sudo: required
 2 | 
 3 | language: c
 4 | 
 5 | notifications:
 6 |   email: false
 7 | 
 8 | branches:
 9 |   only:
10 |     - master
11 |     - develop
12 |     - /^[0-9]+\.[0-9]+\.[0-9]+/
13 | 
14 | services:
15 |   - docker
16 | 
17 | install:
18 |   - sudo apt-get install -y docker-engine
19 |   - pip install --user cpp-coveralls
20 | 
21 | script:
22 |   - docker network create --subnet=172.26.0.0/24 test
23 |   - docker run -d --net test --ip 172.26.0.2 --name zookeeper wurstmeister/zookeeper
24 |   - docker run -d --net test --ip 172.26.0.3 --name kafka -e KAFKA_ADVERTISED_HOST_NAME="172.26.0.3" -e KAFKA_ADVERTISED_PORT="9092" -e KAFKA_ZOOKEEPER_CONNECT="172.26.0.2:2181" -v /var/run/docker.sock:/var/run/docker.sock wurstmeister/kafka
25 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka ./configure
26 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka make
27 |   - docker run -v $(pwd):/app -e CFLAGS=-w --link kafka --net test redborder/build-dockerfiles:n2kafka make tests
28 | 
29 | after_success:
30 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka make clean
31 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka ./configure --enable-coverage
32 |   - docker run -v $(pwd):/app -e CFLAGS=-w --link kafka --net test redborder/build-dockerfiles:n2kafka make coverage
33 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka lcov --remove tests/coverage.info 'tests/*' '/usr/*' --output-file tests/coverage.info
34 |   - docker run -v $(pwd):/app redborder/build-dockerfiles:n2kafka lcov --list tests/coverage.info
35 |   - coveralls --exclude /usr/lib --exclude tests --gcov-options '\-lp'
36 | 
37 | before_deploy:
38 |   - tar -cvzf n2kafka-${TRAVIS_TAG}.tar.gz n2kafka
39 | 
40 | deploy:
41 |   provider: releases
42 |   api_key:
43 |     secure: dTyNSU5oSYX9mc5u25b3Hxzui0nNp88z1IVaxAK7A1qROld3S9GT4PbM5jzM/kYrHrpbzGeV7NUzh/WyAorgtYa2W5waJyTA1WhTx2Vn2NUOcm5TZ1KiOHxK05zeIsTMVjzEIRXGj1xsoQ1f9dwzlBh1cMx5jSDk4E3+B/YKFgQ=
44 |   file: n2kafka-${TRAVIS_TAG}.tar.gz
45 |   on:
46 |     tags: true
47 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
  1 | BIN=	n2kafka
  2 | 
  3 | SRCS=   version.c
  4 | OBJS = $(SRCS:.c=.o)
  5 | TESTS_C = $(sort $(wildcard tests/0*.c))
  6 | 
  7 | TESTS = $(TESTS_C:.c=.test)
  8 | TESTS_OBJS = $(TESTS:.test=.o)
  9 | TESTS_CHECKS_XML = $(TESTS_C:.c=.xml)
 10 | TESTS_MEM_XML = $(TESTS_C:.c=.mem.xml)
 11 | TESTS_HELGRIND_XML = $(TESTS_C:.c=.helgrind.xml)
 12 | TESTS_DRD_XML = $(TESTS_C:.c=.drd.xml)
 13 | TESTS_VALGRIND_XML = $(TESTS_MEM_XML) $(TESTS_HELGRIND_XML) $(TESTS_DRD_XML)
 14 | TESTS_XML = $(TESTS_CHECKS_XML) $(TESTS_VALGRIND_XML)
 15 | 
 16 | all: $(BIN)
 17 | 
 18 | CURRENT_N2KAFKA_DIR = $(dir $(lastword $(MAKEFILE_LIST)))
 19 | 
 20 | include src/Makefile.mk
 21 | include mklove/Makefile.base
 22 | 
 23 | .PHONY: version.c tests coverage
 24 | 
 25 | version.c:
 26 | 	@rm -f $@
 27 | 	@echo "const char *n2kafka_revision=\"`git describe --abbrev=6 --dirty --tags --always`\";" >> $@
 28 | 	@echo 'const char *n2kafka_version="1.0.0";' >> $@
 29 | 
 30 | install: bin-install
 31 | 
 32 | clean: bin-clean
 33 | 	rm -f $(TESTS) $(TESTS_OBJS) $(TESTS_XML) $(COV_FILES)
 34 | 
 35 | COV_FILES = $(foreach ext,gcda gcno, $(SRCS:.c=.$(ext)) $(TESTS_C:.c=.$(ext)))
 36 | 
 37 | VALGRIND ?= valgrind
 38 | SUPPRESSIONS_FILE ?= tests/valgrind.suppressions
 39 | ifneq ($(wildcard $(SUPPRESSIONS_FILE)),)
 40 | SUPPRESSIONS_VALGRIND_ARG = --suppressions=$(SUPPRESSIONS_FILE)
 41 | endif
 42 | 
 43 | .PHONY: tests checks memchecks drdchecks helchecks coverage check_coverage
 44 | 
 45 | run_tests = tests/run_tests.sh $(1) $(TESTS_C:.c=)
 46 | run_valgrind = $(VALGRIND) --tool=$(1) $(SUPPRESSIONS_VALGRIND_ARG) --xml=yes \
 47 | 					--xml-file=$(2) $(3) >/dev/null 2>&1
 48 | 
 49 | tests: $(TESTS_XML)
 50 | 	@$(call run_tests, -cvdh)
 51 | 
 52 | checks: $(TESTS_CHECKS_XML)
 53 | 	@$(call run_tests,-c)
 54 | 
 55 | memchecks: $(TESTS_VALGRIND_XML)
 56 | 	@$(call run_tests,-v)
 57 | 
 58 | drdchecks: $(TESTS_DRD_XML)
 59 | 	@$(call run_tests,-d)
 60 | 
 61 | helchecks: $(TESTS_HELGRIND_XML)
 62 | 	@$(call run_tests,-h)
 63 | 
 64 | tests/%.mem.xml: tests/%.test
 65 | 	@echo -e '\033[0;33m Checking memory:\033[0m $<'
 66 | 	-@$(call run_valgrind,memcheck,"$@","./$<")
 67 | 
 68 | tests/%.helgrind.xml: tests/%.test
 69 | 	@echo -e '\033[0;33m Testing concurrency [HELGRIND]:\033[0m $<'
 70 | 	-@$(call run_valgrind,helgrind,"$@","./$<")
 71 | 
 72 | tests/%.drd.xml: tests/%.test
 73 | 	@echo -e '\033[0;33m Testing concurrency [DRD]:\033[0m $<'
 74 | 	-@$(call run_valgrind,drd,"$@","./$<")
 75 | 
 76 | tests/%.xml: tests/%.test
 77 | 	@echo -e '\033[0;33m Testing:\033[0m $<'
 78 | 	@CMOCKA_XML_FILE="$@" CMOCKA_MESSAGE_OUTPUT=XML "./$<" >/dev/null 2>&1
 79 | 
 80 | tests/%.test: CPPFLAGS := -I. $(CPPFLAGS)
 81 | tests/%.test: tests/%.o $(filter-out src/engine/n2kafka.o,$(OBJS))
 82 | 	@echo -e '\033[0;33m Building: $@ \033[0m'
 83 | 	@$(CC) $(CPPFLAGS) $(LDFLAGS) $< $(shell cat $(@:.test=.objdeps)) -o $@ $(LIBS) -lcmocka
 84 | 
 85 | check_coverage:
 86 | 	@( if [[ "x$(WITH_COVERAGE)" == "xn" ]]; then \
 87 | 	echo -e "$(MKL_RED) You need to configure using --enable-coverage"; \
 88 | 	echo -n "$(MKL_CLR_RESET)"; \
 89 | 	false; \
 90 | 	fi)
 91 | 
 92 | COVERAGE_INFO ?= coverage.info
 93 | COVERAGE_OUTPUT_DIRECTORY ?= coverage.out.html
 94 | COV_VALGRIND ?= valgrind
 95 | COV_GCOV ?= gcov
 96 | COV_LCOV ?= lcov
 97 | 
 98 | coverage: check_coverage $(TESTS)
 99 | 	( for test in $(TESTS); do ./$$test; done )
100 | 	$(COV_LCOV) --gcov-tool=$(COV_GCOV) -q \
101 |                 --rc lcov_branch_coverage=1 --capture \
102 |                 --directory ./ --output-file ${COVERAGE_INFO}
103 | 	genhtml --branch-coverage ${COVERAGE_INFO} --output-directory \
104 | 				${COVERAGE_OUTPUT_DIRECTORY} > coverage.out
105 | 	# ./display_coverage.sh
106 | 
107 | -include $(DEPS)
108 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # n2kafka
 2 | 
 3 | [![Build Status](https://travis-ci.org/redBorder/n2kafka.svg?branch=develop)](https://travis-ci.org/redBorder/n2kafka)
 4 | [![Coverage Status](https://coveralls.io/repos/github/redBorder/n2kafka/badge.svg?branch=develop)](https://coveralls.io/github/redBorder/n2kafka?branch=develop)
 5 | 
 6 | Network to kafka translator. It (currently) support conversion from tcp/udp raw
 7 | sockets and HTTP POST to kafka messages, doing message-processing if you need
 8 | to.
 9 | 
10 | # Setup
11 | To use it, you only need to do a typical `./configure && make && make install`
12 | 
13 | # Usage
14 | ## Basic usage
15 | 
16 | In order to send raw tcp messages from port `2056`, to `mymessages` topic, using
17 | `40` threads, you need to use this config file:
18 | ```json
19 | {
20 | 	"listeners":[
21 | 		{"proto":"tcp","port":2056,"num_threads":40}
22 | 	],
23 | 	"brokers":"localhost",
24 | 	"topic":"mymessages"
25 | }
26 | ```
27 | 
28 | And launch `n2kafka` using `./n2kafka <config_file>`. You can also use `udp` and
29 | `http` as proto values. If you want to listen in different ports, you can add as
30 | many listeners as you want.
31 | 
32 | ## Recommended config parameters
33 | You can also use this parameters in config json root to improve n2kafka
34 | behavior:
35 | - `"blacklist":["192.168.101.3"]`, that will ignore requests of this directions
36 |   (useful for load balancers)
37 | - All
38 |   [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md).
39 |   options. If a config option starts with `rdkafka.<option>`, `<option>` will be
40 |   passed directly to librdkafka config, so you can use whatever config you need.
41 |   Recommended options are:
42 |   * `"rdkafka.socket.max.fails":"3"`
43 |   * `"rdkafka.socket.keepalive.enable":"true"`
44 |   * `"delivery.report.only.error":"true"`
45 | 
46 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "listeners": [{
 3 |     "proto": "http",
 4 |     "port": 2057,
 5 |     "mode": "epoll",
 6 |     "num_threads": 20,
 7 |     "decode_as": "MSE",
 8 |     "max_time_threshold": 3600
 9 |   }, {
10 |     "proto": "tcp",
11 |     "port": 2056,
12 |     "tcp_keepalive": true,
13 |     "num_threads": 40,
14 |     "decode_as": "MSE",
15 |     "max_time_threshold": 3600
16 |   }],
17 |   "brokers": "eugeniodev",
18 |   "topic": "rb_flow",
19 |   "rdkafka.socket.max.fails": "3",
20 |   "rdkafka.socket.keepalive.enable": "true",
21 |   "blacklist": ["192.168.101.3"],
22 |   "mse-sensors": [{
23 |     "stream": "stream_test",
24 |     "enrichment": {
25 |       "sensor_name": "MSE_testing",
26 |       "sensor_id": 255
27 |     }
28 |   }, {
29 |     "stream": "streamstream2",
30 |     "enrichment": {
31 |       "sensor_name": "MSE_testing223",
32 |       "sensor_id": 256
33 |     }
34 |   }, {
35 |     "stream": "streamsss",
36 |     "enrichment": {
37 |       "sensor_name": "MSE_testing4",
38 |       "sensor_id": 257,
39 |       "market": "Spain",
40 |       "organization": "Adamo"
41 |     }
42 |   }, {
43 |     "stream": "streamtest",
44 |     "enrichment": {
45 |       "sensor_name": "Mse_test",
46 |       "sensor_id": 258
47 |     }
48 |   }, {
49 |     "stream": "StreamTest",
50 |     "enrichment": {
51 |       "sensor_name": "MsE_test",
52 |       "sensor_id": 308
53 |     }
54 |   }, {
55 |     "stream": "khgfkghf",
56 |     "enrichment": {
57 |       "sensor_name": "njgdjhdgf",
58 |       "sensor_id": 310
59 |     }
60 |   }]
61 | }
62 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config.json.default:
--------------------------------------------------------------------------------
 1 | {
 2 |   "listeners": [{
 3 |     "proto": "http",
 4 |     "port": 2057,
 5 |     "mode": "epoll",
 6 |     "num_threads": 2
 7 |   }],
 8 |   "brokers": "localhost",
 9 |   "topic": "n2kafka_test",
10 |   "rdkafka.socket.max.fails": "3",
11 |   "rdkafka.socket.keepalive.enable": "true"
12 | }
13 | 
14 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config_debug.json:
--------------------------------------------------------------------------------
1 | {
2 |   "debug": 1,
3 |   "proto": "tcp",
4 |   "port": 2057,
5 |   "threads": 20
6 | }
7 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config_meraki.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "listeners": [{
 3 |     "proto": "http",
 4 |     "port": 2057,
 5 |     "mode": "epoll",
 6 |     "num_threads": 20
 7 |   }, {
 8 |     "proto": "http",
 9 |     "port": 2056,
10 |     "tcp_keepalive": true,
11 |     "num_threads": 40,
12 |     "decode_as": "meraki"
13 |   }],
14 |   "brokers": "eugeniodev",
15 |   "topic": "rb_flow",
16 |   "rdkafka.socket.max.fails": "3",
17 |   "rdkafka.socket.keepalive.enable": "true",
18 |   "blacklist": ["192.168.101.3"],
19 |   "meraki-secrets": {
20 |     "r3dB0rder": {
21 |       "sensor_name": "MSE_testing",
22 |       "sensor_id": 255
23 |     }
24 |   }
25 | }
26 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config_mse.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "listeners": [{
 3 |     "proto": "http",
 4 |     "port": 2050,
 5 |     "mode": "epoll",
 6 |     "num_threads": 20,
 7 |     "decode_as": "MSE",
 8 |     "max_time_offset": 3600,
 9 |     "max_time_offset_warning_wait": 3600,
10 |     "topic":"rb_event"
11 |   }, {
12 |     "proto": "http",
13 |     "port": 2051,
14 |     "tcp_keepalive": true,
15 |     "num_threads": 40,
16 |     "decode_as": "MSE",
17 |     "max_time_offset": 3600,
18 |     "max_time_offset_warning_wait": 3600,
19 |     "topic":"rb_flow"
20 |   }],
21 |   "brokers": "eugeniodev",
22 |   "topic": "rb_flow",
23 |   "rdkafka.socket.max.fails": "3",
24 |   "rdkafka.socket.keepalive.enable": "true",
25 |   "blacklist": ["192.168.101.3"],
26 |   "mse-sensors": [{
27 |     "stream": "stream_test",
28 |     "enrichment": {
29 |       "sensor_name": "MSE_testing",
30 |       "sensor_id": 255
31 |     }
32 |   }, {
33 |     "stream": "streamstream2",
34 |     "enrichment": {
35 |       "sensor_name": "MSE_testing223",
36 |       "sensor_id": 256
37 |     }
38 |   }, {
39 |     "stream": "streamsss",
40 |     "enrichment": {
41 |       "sensor_name": "MSE_testing4",
42 |       "sensor_id": 257,
43 |       "market": "Spain",
44 |       "organization": "Adamo"
45 |     }
46 |   }, {
47 |     "stream": "streamtest",
48 |     "enrichment": {
49 |       "sensor_name": "Mse_test",
50 |       "sensor_id": 258
51 |     }
52 |   }, {
53 |     "stream": "StreamTest",
54 |     "enrichment": {
55 |       "sensor_name": "MsE_test",
56 |       "sensor_id": 308
57 |     }
58 |   }, {
59 |     "stream": "khgfkghf",
60 |     "enrichment": {
61 |       "sensor_name": "njgdjhdgf",
62 |       "sensor_id": 310
63 |     }
64 |   }, {
65 |     "stream": "*",
66 |     "enrichment": {
67 |       "sensor_name": "diego_test",
68 |       "sensor_id": 310
69 |     }
70 |   }]
71 | 
72 | }
73 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config_rbhttp.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "listeners": [{
 3 |     "proto": "http",
 4 |     "port": 2057,
 5 |     "mode": "epoll",
 6 |     "num_threads": 20,
 7 |     "redborder_uri": true,
 8 |     "decode_as": "rb_http2k"
 9 |   }],
10 |   "brokers": "kafka",
11 |   "n2kafka_id": "n2kafka_test",
12 |   "rdkafka.socket.max.fails": "3",
13 |   "rdkafka.socket.keepalive.enable": "true",
14 |   "blacklist": ["192.168.101.3"],
15 |   "rb_http2k_config": {
16 |     "sensors_uuids": {
17 |       "abc": {
18 | 	"enrichment": {
19 |           "a": 1,
20 |           "b": "c",
21 |           "d": true,
22 |           "e": null
23 |         },
24 |         "organization_uuid":"abc_org"
25 |       },
26 |       "def": {
27 | 	"enrichment": {
28 |           "f": 1,
29 |           "g": "w",
30 |           "h": false,
31 |           "i": null
32 |         },
33 |         "organization_uuid":"def_org"
34 |       }
35 |     },
36 |     "organizations_uuids": {
37 |       "abc_org": {
38 |         "enrichment": {
39 |           "a_org":10
40 |         },
41 |         "limits": {
42 |           "bytes": 10240
43 |         }
44 |       },
45 |       "def_org": {
46 |         "enrichment": {
47 |           "b_org":20
48 |         },
49 |         "limits": {
50 |           "bytes": 20480
51 |         }
52 |       }
53 |     },
54 |     "organizations_sync": {
55 |       "put_url": "http://localhost:80/",
56 |       "topics": ["rb_monitor","rb_event"],
57 |       "interval_s": 5,
58 |       "clean_on": {
59 |         "timestamp_s_mod":5,
60 |         "timestamp_s_offset":4
61 |       }
62 |     },
63 |     "topics": {
64 |       "rb_flow": {
65 |         "partition_key": "client_mac",
66 |         "partition_algo": "mac"
67 |       },
68 |       "rb_event": {}
69 |     }
70 |   }
71 | }
72 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_config_response.json:
--------------------------------------------------------------------------------
1 | {
2 |   "debug": 1,
3 |   "proto": "tcp",
4 |   "port": 5050,
5 |   "threads": 20,
6 |   "response": "response.xml"
7 | }
8 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_tests_http.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "listeners": [{
 3 |         "proto": "http",
 4 |         "port": 2057,
 5 |         "mode": "epoll",
 6 |         "num_threads": 20,
 7 |         "redborder_uri": true
 8 |     }],
 9 |     "brokers": "kafka",
10 |     "topic": "rb_flow",
11 |     "n2kafka_id": "n2kafka_test",
12 |     "rdkafka.socket.max.fails": "3",
13 |     "rdkafka.socket.keepalive.enable": "true",
14 |     "blacklist": ["192.168.101.3"],
15 |     "rb_http2k_config": {
16 |         "sensors_uuids": {
17 |             "abc": {
18 |                 "enrichment": {
19 |                     "a": 1,
20 |                     "b": "c",
21 |                     "d": true,
22 |                     "e": null
23 |                 },
24 |                 "organization_uuid": "abc_org"
25 |             },
26 |             "def": {
27 |                 "enrichment": {
28 |                     "f": 1,
29 |                     "g": "w",
30 |                     "h": false,
31 |                     "i": null
32 |                 },
33 |                 "organization_uuid": "def_org"
34 |             }
35 |         },
36 |         "organizations_uuids": {
37 |             "abc_org": {
38 |                 "enrichment": {
39 |                     "a_org": 10
40 |                 },
41 |                 "limits": {
42 |                     "bytes": 10240
43 |                 }
44 |             },
45 |             "def_org": {
46 |                 "enrichment": {
47 |                     "b_org": 20
48 |                 },
49 |                 "limits": {
50 |                     "bytes": 20480
51 |                 }
52 |             }
53 |         },
54 |         "topics": {
55 |             "rb_flow": {
56 |                 "partition_key": "client_mac",
57 |                 "partition_algo": "mac"
58 |             },
59 |             "rb_event": {}
60 |         }
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/configs_example/n2kafka_tests_tcp.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "listeners": [{
 3 |         "proto": "tcp",
 4 |         "port": 2056,
 5 |         "tcp_keepalive": true,
 6 |         "num_threads": 40,
 7 |         "max_time_threshold": 3600
 8 |     }],
 9 |     "brokers": "kafka",
10 |     "topic": "rb_flow",
11 |     "n2kafka_id": "n2kafka_test",
12 |     "rdkafka.socket.max.fails": "3",
13 |     "rdkafka.socket.keepalive.enable": "true",
14 |     "blacklist": ["192.168.101.3"],
15 |     "rb_http2k_config": {
16 |         "sensors_uuids": {
17 |             "abc": {
18 |                 "enrichment": {
19 |                     "a": 1,
20 |                     "b": "c",
21 |                     "d": true,
22 |                     "e": null
23 |                 },
24 |                 "organization_uuid": "abc_org"
25 |             },
26 |             "def": {
27 |                 "enrichment": {
28 |                     "f": 1,
29 |                     "g": "w",
30 |                     "h": false,
31 |                     "i": null
32 |                 },
33 |                 "organization_uuid": "def_org"
34 |             }
35 |         },
36 |         "organizations_uuids": {
37 |             "abc_org": {
38 |                 "enrichment": {
39 |                     "a_org": 10
40 |                 },
41 |                 "limits": {
42 |                     "bytes": 10240
43 |                 }
44 |             },
45 |             "def_org": {
46 |                 "enrichment": {
47 |                     "b_org": 20
48 |                 },
49 |                 "limits": {
50 |                     "bytes": 20480
51 |                 }
52 |             }
53 |         },
54 |         "topics": {
55 |             "rb_flow": {
56 |                 "partition_key": "client_mac",
57 |                 "partition_algo": "mac"
58 |             },
59 |             "rb_event": {}
60 |         }
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/configure:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | 
  4 | MKL_CONFIGURE_ARGS="$0 $*"
  5 | 
  6 | # Load base module
  7 | source mklove/modules/configure.base
  8 | 
  9 | # Read some special command line options right away that must be known prior to
 10 | # sourcing modules.
 11 | mkl_in_list "$*" "--no-download" && MKL_NO_DOWNLOAD=1
 12 | # Disable downloads when --help is used to avoid blocking calls.
 13 | mkl_in_list "$*" "--help" && MKL_NO_DOWNLOAD=1
 14 | mkl_in_list "$*" "--debug" && MKL_DEBUG=1
 15 | 
 16 | # Delete temporary Makefile and header files on exit.
 17 | trap "{ rm -f $MKL_OUTMK $MKL_OUTH; }" EXIT
 18 | 
 19 | 
 20 | 
 21 | ##
 22 | ## Load builtin modules
 23 | ##
 24 | 
 25 | # Builtin options, etc.
 26 | mkl_require builtin
 27 | 
 28 | # Host/target support
 29 | mkl_require host
 30 | 
 31 | # Compiler detection
 32 | mkl_require cc
 33 | 
 34 | 
 35 | # Load application provided modules (in current directory), if any.
 36 | for fname in configure.* ; do
 37 |     if [[ $fname = 'configure.*' ]]; then
 38 |         continue
 39 |     fi
 40 | 
 41 |     # Skip temporary files
 42 |     if [[ $fname = *~ ]]; then
 43 |         continue
 44 |     fi
 45 | 
 46 |     mkl_require $fname
 47 | done
 48 | 
 49 | 
 50 | 
 51 | 
 52 | ##
 53 | ## Argument parsing (options)
 54 | ##
 55 | ##
 56 | 
 57 | # Parse arguments
 58 | while [[ ! -z $@ ]]; do
 59 |     if [[ $1 != --* ]]; then
 60 |         mkl_err "Unknown non-option argument: $1"
 61 |         mkl_usage
 62 |         exit 1
 63 |     fi
 64 | 
 65 |     opt=${1#--}
 66 |     shift
 67 | 
 68 |     if [[ $opt = *=* ]]; then
 69 |         name="${opt%=*}"
 70 |         arg="${opt#*=}"
 71 |         eqarg=1
 72 |     else
 73 |         name="$opt"
 74 |         arg=""
 75 |         eqarg=0
 76 |     fi
 77 | 
 78 |     safeopt="$(mkl_env_esc $name)"
 79 | 
 80 |     if ! mkl_func_exists opt_$safeopt ; then
 81 |         mkl_err "Unknown option $opt"
 82 |         mkl_usage
 83 |         exit 1
 84 |     fi
 85 | 
 86 |     # Check if this option needs an argument.
 87 |     reqarg=$(mkl_meta_get "MKL_OPT_ARGS" "$(mkl_env_esc $name)")
 88 |     if [[ ! -z $reqarg ]]; then
 89 |         if [[ $eqarg == 0 && -z $arg ]]; then
 90 |             arg=$1
 91 |             shift
 92 | 
 93 |             if [[ -z $arg ]]; then
 94 |                 mkl_err "Missing argument to option --$name $reqarg"
 95 |                 exit 1
 96 |             fi
 97 |         fi
 98 |     else
 99 |         if [[ ! -z $arg ]]; then
100 |             mkl_err "Option --$name expects no argument"
101 |             exit 1
102 |         fi
103 |         arg=y
104 |     fi
105 | 
106 |     case $name in
107 |         re|reconfigure)
108 |             oldcmd=$(grep '^# configure exec: ' config.log | \
109 |                 sed -e 's/^\# configure exec: //')
110 |             if [[ -z $oldcmd ]]; then
111 |                 echo "No previous execution found in config.log"
112 |                 exit 1
113 |             fi
114 |             echo "Reconfiguring: $oldcmd"
115 |             exec $oldcmd
116 |             ;;
117 | 
118 |         list-modules)
119 |             echo "Modules loaded:"
120 |             for mod in $MKL_MODULES ; do
121 |                 echo "  $mod"
122 |             done
123 |             exit 0
124 |             ;;
125 | 
126 |         list-checks)
127 |             echo "Check functions in calling order:"
128 |             for mf in $MKL_CHECKS ; do
129 |                 mod=${mf%:*}
130 |                 func=${mf#*:}
131 |                 echo -e "${MKL_GREEN}From module $mod:$MKL_CLR_RESET"
132 |                 declare -f $func
133 |                 echo ""
134 |             done
135 |             exit 0
136 |             ;;
137 | 
138 |         update-modules)
139 |             fails=0
140 |             echo "Updating modules"
141 |             for mod in $MKL_MODULES ; do
142 |                 echo -n "Updating $mod..."
143 |                 if mkl_module_download "$mod" > /dev/null ; then
144 |                     echo -e "${MKL_GREEN}ok${MKL_CLR_RESET}"
145 |                 else
146 |                     echo -e "${MKL_RED}failed${MKL_CLR_RESET}"
147 |                     fails=$(expr $fails + 1)
148 |                 fi
149 |             done
150 |             exit $fails
151 |             ;;
152 | 
153 |         help)
154 |             mkl_usage
155 |             exit 0
156 |             ;;
157 | 
158 |         *)
159 |             opt_$safeopt $arg || exit 1
160 |             mkl_var_append MKL_OPTS_SET "$safeopt"
161 |             ;;
162 |     esac
163 | done
164 | 
165 | if [[ ! -z $MKL_CLEAN ]]; then
166 |     mkl_clean
167 |     exit 0
168 | fi
169 | 
170 | # Move away previous log file
171 | [[ -f $MKL_OUTDBG ]] && mv $MKL_OUTDBG ${MKL_OUTDBG}.old
172 | 
173 | 
174 | # Create output files
175 | echo "# configure exec: $0 $*" >> $MKL_OUTDBG
176 | echo "# On $(date)" >> $MKL_OUTDBG
177 | 
178 | rm -f $MKL_OUTMK $MKL_OUTH
179 | mkl_write_mk "# Automatically generated by $0 $*"
180 | mkl_write_h "// Automatically generated by $0 $*"
181 | mkl_write_h "#pragma once"
182 | 
183 | 
184 | # Load cache file
185 | mkl_cache_read
186 | 
187 | # Run checks
188 | mkl_checks_run
189 | 
190 | # Check accumulated failures, will not return on failure.
191 | mkl_check_fails
192 | 
193 | # Generate outputs
194 | mkl_generate
195 | 
196 | # Summarize what happened
197 | mkl_summary
198 | 
199 | # Write cache file
200 | mkl_cache_write
201 | 
202 | 
203 | echo ""
204 | echo "Now type 'make' to build"
205 | trap - EXIT
206 | exit 0
207 | 


--------------------------------------------------------------------------------
/configure.n2kafka:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | 
  4 | mkl_require good_cflags must pass
  5 | mkl_require socket must pass
  6 | mkl_require gitversion as GITVERSION
  7 | mkl_require atomics must pass
  8 | 
  9 | mkl_meta_set "description" "name"      "n2kafka"
 10 | mkl_meta_set "description" "oneline"   "The Network to kafka conversion service"
 11 | mkl_meta_set "description" "long"      "Consume messages from tcp/udp/http and send them via kafka"
 12 | mkl_meta_set "description" "copyright" "Copyright (c) 2014-2016 Eneo Tecnologia S.L."
 13 | 
 14 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-DFORTIFY_SOURCE=2"
 15 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-fno-strict-aliasing" # Needed by libev
 16 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-Wunused -Wcast-qual -Wextra -Wsign-conversion -Wshadow"
 17 | #mkl_mkvar_append CPPFLAGS CPPFLAGS "-ftrack-macro-expansion=2"
 18 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-Wcast-align -Wdisabled-optimization -Winit-self -Wlogical-op"
 19 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-Wmissing-include-dirs -Wstrict-overflow=5 -Wundef -Wswitch-default"
 20 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-Wmissing-declarations -Wdisabled-optimization"
 21 | 
 22 | # This will make the life easier
 23 | mkl_mkvar_append CPPFLAGS CPPFLAGS "-I. -I./src"
 24 | 
 25 | mkl_toggle_option "Feature" WITH_HTTP "--enable-http" "HTTP support using libmicrohttpd" "y"
 26 | mkl_toggle_option "Debug" WITH_COVERAGE "--enable-coverage" "Coverage build" "n"
 27 | 
 28 | function checks_libmicrohttpd {
 29 |     # quick hack to avoid bad formatted libmicrohttpd header. With this macro
 30 |     # defined, the header will know that we are using GCC at libmicrohttp
 31 |     # 0.9.44
 32 |     mkl_mkvar_append CPPFLAGS CPPFLAGS "-D__clang_major__"
 33 |     mkl_meta_set "libmicrohttpd" "desc" \
 34 |         "library embedding HTTP server functionality"
 35 |     mkl_meta_set "libmicrohttpd" "deb" "libmicrohttpd-dev"
 36 |     mkl_lib_check "libmicrohttpd" "" fail CC "-lmicrohttpd" \
 37 |         "#include <microhttpd.h>"
 38 |     mkl_define_set "Have libmicrohttpd library" "HAVE_LIBMICROHTTPD" "1"
 39 | }
 40 | 
 41 | function checks {
 42 |     mkl_meta_set "librd" "desc" "Magnus Edenhill's librd is available at http://github.com/edenhill/librd"
 43 |     mkl_lib_check --static=-lrd "librd" "" fail CC "-lrd -lpthread -lz -lrt" \
 44 |        "#include <librd/rd.h>"
 45 | 
 46 |     # Check that librdkafka is available, and allow to link it statically.
 47 |     mkl_meta_set "librdkafka" "desc" \
 48 |       "Magnus Edenhill's librdkafka is available at http://github.com/edenhill/librdkafka"
 49 |     mkl_meta_set "librdkafka" "deb" "librdkafka-dev"
 50 |     mkl_lib_check --static=-lrdkafka "librdkafka" "" fail CC "-lrdkafka -lpthread -lz" \
 51 |        "#include <librdkafka/rdkafka.h>
 52 |        #if RD_KAFKA_VERSION < 0x00090100
 53 |        #error Need librdkafka version >0.9.1
 54 |        #endif"
 55 | 
 56 |     mkl_meta_set "libev" "desc" "A high performance full-featured event loop written in C"
 57 |     mkl_meta_set "libev" "deb" "libev-dev"
 58 |     mkl_lib_check "libev" "" fail CC "-lev" "#include <ev.h>"
 59 | 
 60 |     # -libmicrohttpd required if HTTP enabled
 61 |     if [[ "x$WITH_HTTP" == "xy" ]]; then
 62 |         checks_libmicrohttpd
 63 |     fi
 64 | 
 65 |     mkl_meta_set "libjansson" "desc" "C library for encoding, decoding and manipulating JSON data"
 66 |     mkl_meta_set "libjansson" "deb" "libjansson-dev"
 67 |     mkl_lib_check --static=-ljansson "libjansson" "" fail CC "-ljansson" \
 68 |        "#include <jansson.h>
 69 |        #if JANSSON_VERSION_HEX < 0x020700
 70 |        #error Need jansson library version >2.7
 71 |        #endif"
 72 | 
 73 |     # TODO include some function that is only in v2, because CentOS bring us v1
 74 |     mkl_meta_set "yajl" "desc" "Small event-driven (SAX-style) JSON parser"
 75 |     mkl_lib_check --static=-lyajl "yajl" "" fail CC "-lyajl" \
 76 |        "#include <yajl.h>"
 77 | 
 78 |     # Check that libcurl is available, and allow to link it statically.
 79 |     mkl_meta_set "lcurl" "desc" "Free and easy-to-use client-side URL transfer library"
 80 |     mkl_meta_set "lcurl" "deb" "libcurl4-gnutls-dev"
 81 |     mkl_lib_check --static=-lcurl "libcurl" "" fail CC "-lcurl -lpthread -lz" \
 82 |        "#include <curl/curl.h>"
 83 | 
 84 |     # Check libcurl is new enough
 85 |     mkl_meta_set "curlversion" "name" "libcurl MAX_TOTAL_CONNECTIONS"
 86 |     mkl_meta_set "curlversion" "desc" "libcurl 7.45.0 or later is required for the Max connections option"
 87 |     mkl_compile_check "curlversion" "" fail CC "" \
 88 |         "#include <curl/curl.h>
 89 |         static int foo __attribute__((unused)) = CURLMOPT_MAX_TOTAL_CONNECTIONS;"
 90 | 
 91 |     if [[ "x$WITHOUT_OPTIMIZATION" != "xy" || "x$WITH_COVERAGE" != "xy" ]]; then
 92 |       mkl_mkvar_append CPPFLAGS CPPFLAGS "-DNDEBUG"
 93 |     fi
 94 | 
 95 |     if [[ "x$WITH_COVERAGE" != "xn" ]]; then
 96 |       mkl_mkvar_append CPPFLAGS CPPFLAGS "--coverage"
 97 |       mkl_mkvar_append LDFLAGS LDFLAGS "--coverage"
 98 |     fi
 99 | }
100 | 


--------------------------------------------------------------------------------
/mklove/Makefile.base:
--------------------------------------------------------------------------------
  1 | # Base Makefile providing various standard targets
  2 | # Part of mklove suite but may be used independently.
  3 | 
  4 | MKL_RED=\033[031m
  5 | MKL_GREEN=\033[032m
  6 | MKL_YELLOW=\033[033m
  7 | MKL_BLUE=\033[034m
  8 | MKL_CLR_RESET=\033[0m
  9 | 
 10 | DEPS=		$(OBJS:%.o=%.d)
 11 | 
 12 | # TOPDIR is "TOPDIR/mklove/../" i.e., TOPDIR.
 13 | # We do it with two dir calls instead of /.. to support mklove being symlinked.
 14 | MKLOVE_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
 15 | TOPDIR = $(MKLOVE_DIR:mklove/=.)
 16 | 
 17 | # Silence lousy default ARFLAGS (rv)
 18 | ARFLAGS=
 19 | 
 20 | -include $(TOPDIR)/Makefile.config
 21 | 
 22 | INSTALL?=		install
 23 | INSTALL_PROGRAM?=	$(INSTALL)
 24 | INSTALL_DATA?=		$(INSTALL) -m 644
 25 | 
 26 | prefix?=	/usr/local
 27 | exec_prefix?=	$(prefix)
 28 | bindir?=	$(exec_prefix)/bin
 29 | sbindir?=	$(exec_prefix)/sbin
 30 | libexecdir?=	$(exec_prefix)/libexec/  # append PKGNAME on install
 31 | datarootdir?=	$(prefix)/share
 32 | datadir?=	$(datarootdir)		 # append PKGNAME on install
 33 | sysconfdir?=	$(prefix)/etc
 34 | sharedstatedir?=$(prefix)/com
 35 | localestatedir?=$(prefix)/var
 36 | runstatedir?=	$(localestatedir)/run
 37 | includedir?=	$(prefix)/include
 38 | docdir?=	$(datarootdir)/doc/$(PKGNAME)
 39 | infodir?=	$(datarootdir)/info
 40 | libdir?=	$(prefix)/lib
 41 | localedir?=	$(datarootdir)/locale
 42 | mandir?=	$(datarootdir)/man
 43 | man1dir?=	$(mandir)/man1
 44 | man2dir?=	$(mandir)/man2
 45 | man3dir?=	$(mandir)/man3
 46 | man4dir?=	$(mandir)/man4
 47 | man5dir?=	$(mandir)/man5
 48 | man6dir?=	$(mandir)/man6
 49 | man7dir?=	$(mandir)/man7
 50 | man8dir?=	$(mandir)/man8
 51 | 
 52 | 
 53 | # Checks that mklove is set up and ready for building
 54 | mklove-check:
 55 | 	@if [ ! -f "$(TOPDIR)/Makefile.config" ]; then \
 56 | 	 	echo "$(MKL_RED)$(TOPDIR)/Makefile.config missing: please run ./configure$(MKL_CLR_RESET)" ; \
 57 | 		exit 1 ; \
 58 | 	fi
 59 | 
 60 | %.o: %.c
 61 | 	$(CC) -MD -MP $(CPPFLAGS) $(CFLAGS) -c $< -o $@
 62 | 
 63 | %.o: %.cpp
 64 | 	$(CXX) -MD -MP $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@
 65 | 
 66 | 
 67 | lib: $(LIBNAME).so.$(LIBVER) $(LIBNAME).a
 68 | 
 69 | $(LIBNAME).so.$(LIBVER): $(OBJS) $(LIBNAME).lds
 70 | 	@echo "$(MKL_YELLOW)Creating shared library $@$(MKL_CLR_RESET)"
 71 | 	$(CC) $(LDFLAGS) $(LIB_LDFLAGS) $(OBJS) -o $@ $(LIBS)
 72 | 
 73 | $(LIBNAME).a:	$(OBJS)
 74 | 	@echo "$(MKL_YELLOW)Creating static library $@$(MKL_CLR_RESET)"
 75 | 	$(AR) rcs$(ARFLAGS) $@ $(OBJS)
 76 | 
 77 | 
 78 | $(BIN): $(OBJS)
 79 | 	@echo "$(MKL_YELLOW)Creating program $@$(MKL_CLR_RESET)"
 80 | 	$(CC) $(CPPFLAGS) $(LDFLAGS) $(OBJS) -o $@ $(LIBS)
 81 | 
 82 | 
 83 | file-check:
 84 | 	@(echo "$(MKL_YELLOW)Checking $(LIBNAME) integrity$(MKL_CLR_RESET)" ; \
 85 | 	 RET=true ; \
 86 | 	 for f in $(CHECK_FILES) ; do \
 87 | 		printf "%-30s " $$f ; \
 88 | 		if [ -f "$$f" ]; then \
 89 | 			echo "$(MKL_GREEN)OK$(MKL_CLR_RESET)"; \
 90 | 		else \
 91 | 			echo "$(MKL_RED)MISSING$(MKL_CLR_RESET)"; \
 92 | 			RET=false ; \
 93 | 		fi; \
 94 | 	done ; \
 95 | 	$$($$RET))
 96 | 
 97 | 
 98 | lib-install:
 99 | 	@echo "$(MKL_YELLOW)Install $(LIBNAME) to $$DESTDIR$(prefix)$(MKL_CLR_RESET)"
100 | 	$(INSTALL) -d $$DESTDIR$(includedir)/$(PKGNAME) ; \
101 | 	$(INSTALL) -d $$DESTDIR$(libdir) ; \
102 | 	$(INSTALL) $(HDRS) $$DESTDIR$(includedir)/$(PKGNAME) ; \
103 | 	$(INSTALL) $(LIBNAME).a $$DESTDIR$(libdir) ; \
104 | 	$(INSTALL) $(LIBNAME).so.$(LIBVER) $$DESTDIR$(libdir) ; \
105 | 	(cd $$DESTDIR$(libdir) && ln -sf $(LIBNAME).so.$(LIBVER) $(LIBNAME).so)
106 | 
107 | lib-uninstall:
108 | 	@echo "$(MKL_YELLOW)Uninstall $(LIBNAME) from $$DESTDIR$(prefix)$(MKL_CLR_RESET)"
109 | 	for hdr in $(HDRS) ; do \
110 | 		rm -f $$DESTDIR$(includedir)/$(PKGNAME)/$$hdr ; done
111 | 	rm -f $$DESTDIR$(libdir)/$(LIBNAME).a
112 | 	rm -f $$DESTDIR$(libdir)/$(LIBNAME).so.$(LIBVER)
113 | 	rm -f $$DESTDIR$(libdir)/$(LIBNAME).so
114 | 	rmdir $$DESTDIR$(includedir)/$(PKGNAME) || true
115 | 
116 | 
117 | bin-install:
118 | 	@echo "$(MKL_YELLOW)Install $(BIN) to $$DESTDIR$(prefix)$(MKL_CLR_RESET)"
119 | 	$(INSTALL) -d $$DESTDIR$(bindir) && \
120 | 	$(INSTALL) $(BIN) $$DESTDIR$(bindir) 
121 | 
122 | bin-uninstall:
123 | 	@echo "$(MKL_YELLOW)Uninstall $(BIN) from $$DESTDIR$(prefix)$(MKL_CLR_RESET)"
124 | 	rm -f $$DESTDIR$(bindir)/$(BIN)
125 | 
126 | 
127 | generic-clean:
128 | 	rm -f $(OBJS) $(DEPS)
129 | 
130 | lib-clean: generic-clean
131 | 	rm -f $(LIBNAME)*.a $(LIBNAME)*.so $(LIBNAME)*.so.$(LIBVER) \
132 | 		$(LIBNAME).lds
133 | 
134 | bin-clean: generic-clean
135 | 	rm -f $(BIN)
136 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.atomics:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | # Checks for atomic ops:
  4 | #  compiler builtin (__sync_..) and portable libatomic's (__atomic_..)
  5 | # Will also provide abstraction by defining the prefix to use.
  6 | #
  7 | # Sets:
  8 | #  HAVE_ATOMICS
  9 | #  HAVE_ATOMICS_32
 10 | #  HAVE_ATOMICS_64
 11 | #  HAVE_ATOMICS_32_ATOMIC   __atomic interface
 12 | #  HAVE_ATOMICS_32_SYNC     __sync interface
 13 | #  HAVE_ATOMICS_64_ATOMIC   __atomic interface
 14 | #  HAVE_ATOMICS_64_SYNC     __sync interface
 15 | #  WITH_LIBATOMIC
 16 | #  LIBS
 17 | #
 18 | #  ATOMIC_OP(OP1,OP2,PTR,VAL)
 19 | #  ATOMIC_OP32(OP1,OP2,PTR,VAL)
 20 | #  ATOMIC_OP64(OP1,OP2,PTR,VAL)
 21 | #   where op* is 'add,sub,fetch'
 22 | #   e.g:  ATOMIC_OP32(add, fetch, &i, 10)
 23 | #         becomes __atomic_add_fetch(&i, 10, ..) or
 24 | #                 __sync_add_and_fetch(&i, 10)
 25 | #
 26 | 
 27 | function checks {
 28 | 
 29 | 
 30 |     # We prefer the newer __atomic stuff, but 64-bit atomics might
 31 |     # require linking with -latomic, so we need to perform these tests
 32 |     # in the proper order:
 33 |     #   __atomic 32
 34 |     #   __atomic 32 -latomic
 35 |     #   __sync 32
 36 |     #
 37 |     #   __atomic 64
 38 |     #   __atomic 64 -latomic
 39 |     #   __sync 64
 40 | 
 41 |     local _libs=
 42 |     local _a32="__atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)"
 43 |     local _a64="__atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)"
 44 | 
 45 |     # 32-bit:
 46 |     # Try fully builtin __atomic
 47 |     if ! mkl_compile_check __atomic_32 HAVE_ATOMICS_32 cont CC "" \
 48 |         "
 49 | #include <inttypes.h>
 50 | int32_t foo (int32_t i); int32_t foo (int32_t i) {
 51 |   return __atomic_add_fetch(&i, 1, __ATOMIC_SEQ_CST);
 52 | }"
 53 |         then
 54 |         # Try __atomic with -latomic
 55 |         if mkl_compile_check --ldflags="-latomic" __atomic_32_lib HAVE_ATOMICS_32 \
 56 |             cont CC "" \
 57 |             "
 58 | #include <inttypes.h>
 59 | int32_t foo (int32_t i); int32_t foo (int32_t i) {
 60 |   return __atomic_add_fetch(&i, 1, __ATOMIC_SEQ_CST);
 61 | }"
 62 |         then
 63 |             _libs="-latomic"
 64 |             mkl_allvar_set "__atomic_32_lib" "HAVE_ATOMICS_32_ATOMIC" "y"
 65 |         else
 66 |             # Try __sync interface
 67 |             if mkl_compile_check __sync_32 HAVE_ATOMICS_32 disable CC "" \
 68 |                 "
 69 | #include <inttypes.h>
 70 | int32_t foo (int32_t i); int32_t foo (int32_t i) {
 71 |   return __sync_add_and_fetch(&i, 1);
 72 | }"
 73 |                 then
 74 |                 _a32="__sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)"
 75 |                 mkl_allvar_set "__sync_32" "HAVE_ATOMICS_32_SYNC" "y"
 76 |             else
 77 |                 _a32=""
 78 |             fi
 79 |         fi
 80 |     else
 81 |         mkl_allvar_set "__atomic_32" "HAVE_ATOMICS_32_ATOMIC" "y"
 82 |     fi
 83 | 
 84 | 
 85 |     if [[ ! -z $_a32 ]]; then
 86 |         mkl_define_set "atomic_32" "ATOMIC_OP32(OP1,OP2,PTR,VAL)" "code:$_a32"
 87 |     fi
 88 | 
 89 | 
 90 | 
 91 |     # 64-bit:
 92 |     # Try fully builtin __atomic
 93 |     if ! mkl_compile_check __atomic_64 HAVE_ATOMICS_64 cont CC "" \
 94 |         "
 95 | #include <inttypes.h>
 96 | int64_t foo (int64_t i); int64_t foo (int64_t i) {
 97 |   return __atomic_add_fetch(&i, 1, __ATOMIC_SEQ_CST);
 98 | }"
 99 |         then
100 |         # Try __atomic with -latomic
101 |         if mkl_compile_check --ldflags="-latomic" __atomic_64_lib HAVE_ATOMICS_64 \
102 |             cont CC "" \
103 |             "
104 | #include <inttypes.h>
105 | int64_t foo (int64_t i); int64_t foo (int64_t i) {
106 |   return __atomic_add_fetch(&i, 1, __ATOMIC_SEQ_CST);
107 | }"
108 |         then
109 |             _libs="-latomic"
110 |             mkl_allvar_set "__atomic_64_lib" "HAVE_ATOMICS_64_ATOMIC" "y"
111 |         else
112 |             # Try __sync interface
113 |             if mkl_compile_check __sync_64 HAVE_ATOMICS_64 disable CC "" \
114 |                 "
115 | #include <inttypes.h>
116 | int64_t foo (int64_t i); int64_t foo (int64_t i) {
117 |   return __sync_add_and_fetch(&i, 1);
118 | }"
119 |                 then
120 |                 _a64="__sync_ ## OP1 ## _and_ ## OP2 (PTR, VAL)"
121 |                 mkl_allvar_set "__sync_64" "HAVE_ATOMICS_64_SYNC" "y"
122 |             else
123 |                 _a64=""
124 |             fi
125 |         fi
126 |     else
127 |         mkl_allvar_set "__atomic_64" "HAVE_ATOMICS_64_ATOMIC" "y"
128 |     fi
129 | 
130 | 
131 |     if [[ ! -z $_a64 ]]; then
132 |         mkl_define_set "atomic_64" "ATOMIC_OP64(OP1,OP2,PTR,VAL)" "code:$_a64"
133 | 
134 |         # Define generic ATOMIC() macro identical to 64-bit atomics"
135 |         mkl_define_set "atomic_64" "ATOMIC_OP(OP1,OP2,PTR,VAL)" "code:$_a64"
136 |     fi
137 | 
138 | 
139 |     if [[ ! -z $_libs ]]; then
140 |         mkl_mkvar_append LDFLAGS LDFLAGS "-Wl,--as-needed"
141 |         mkl_mkvar_append LIBS LIBS "$_libs"
142 |     fi
143 | 
144 | }
145 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.builtin:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | #
 3 | # mklove builtin checks and options
 4 | # Sets:
 5 | #  prefix, etc..
 6 | 
 7 | 
 8 | mkl_option "Standard" prefix "--prefix=PATH" \
 9 |     "Install arch-independent files in PATH" "/usr/local"
10 | mkl_option "Standard" exec_prefix "--exec-prefix=PATH" \
11 |     "Install arch-dependent files in PATH" "\$prefix"
12 | mkl_option "Standard" bindir "--bindir=PATH" "User executables" "\$exec_prefix/bin"
13 | mkl_option "Standard" sbindir "--sbindir=PATH" "System admin executables" \
14 |     "\$exec_prefix/sbin"
15 | mkl_option "Standard" libexecdir "--libexecdir=PATH" "Program executables" \
16 |     "\$exec_prefix/libexec"
17 | mkl_option "Standard" datadir "--datadir=PATH" "Read-only arch-independent data" \
18 |     "\$prefix/share"
19 | mkl_option "Standard" sysconfdir "--sysconfdir=PATH" "Configuration data" \
20 |     "\$prefix/etc"
21 | mkl_option "Standard" sharedstatedir "--sharedstatedir=PATH" \
22 |     "Modifiable arch-independent data" "\$prefix/com"
23 | mkl_option "Standard" localstatedir "--localstatedir=PATH" \
24 |     "Modifiable local state data" "\$prefix/var"
25 | mkl_option "Standard" libdir "--libdir=PATH" "Libraries" "\$exec_prefix/lib"
26 | mkl_option "Standard" includedir "--includedir=PATH" "C/C++ header files" \
27 |     "\$prefix/include"
28 | mkl_option "Standard" infodir "--infodir=PATH" "Info documentation" "\$prefix/info"
29 | mkl_option "Standard" mandir "--mandir=PATH" "Manual pages" "\$prefix/man"
30 | 
31 | mkl_option "Configure tool" "" "--list-modules" "List loaded mklove modules"
32 | mkl_option "Configure tool" "" "--list-checks" "List checks"
33 | mkl_option "Configure tool" env:MKL_FAILFATAL "--fail-fatal" "All failures are fatal"
34 | mkl_option "Configure tool" env:MKL_NOCACHE "--no-cache" "Dont use or generate config.cache"
35 | mkl_option "Configure tool" env:MKL_DEBUG "--debug" "Enable configure debugging"
36 | mkl_option "Configure tool" env:MKL_CLEAN "--clean" "Remove generated configure files"
37 | mkl_option "Configure tool" "" "--reconfigure" "Rerun configure with same arguments as last run"
38 | mkl_option "Configure tool" env:MKL_NO_DOWNLOAD "--no-download" "Disable downloads of required mklove modules"
39 | mkl_option "Configure tool" env:MKL_UPDATE_MODS "--update-modules" "Update modules from global repository"
40 | mkl_option "Configure tool" env:MKL_REPO_URL "--repo-url=URL_OR_PATH" "Override mklove modules repo URL" "$MKL_REPO_URL"
41 | mkl_option "Configure tool" "" "--help" "Show configure usage"
42 | 
43 | 
44 | mkl_toggle_option "Compatibility" "mk:MKL_MAINT_MODE" "--enable-maintainer-mode" "Maintainer mode (no-op)"
45 | 
46 | mkl_option "Configure tool" "mk:PROGRAM_PREFIX" "--program-prefix=PFX" "Program prefix"
47 | 
48 | mkl_option "Compatibility" "mk:DISABL_DEP_TRACK" "--disable-dependency-tracking" "Disable dependency tracking (no-op)"
49 | mkl_option "Compatibility" "mk:DISABL_SILENT_RULES" "--disable-silent-rules" "Verbose build output (no-op)"
50 | 
51 | 
52 | function checks {
53 | 
54 |     if [[ ! -z $libdir ]]; then
55 | 	mkl_mkvar_append "libdir" LDFLAGS "-L${libdir}"
56 |     fi
57 | 
58 |     if [[ ! -z $includedir ]]; then
59 | 	mkl_mkvar_append "includedir" CPPFLAGS "-I${includedir}"
60 |     fi
61 | 
62 | }
63 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.cc:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | # Compiler detection
  4 | # Sets:
  5 | #  CC, CXX, CFLAGS, CPPFLAGS, LDFLAGS, ARFLAGS, PKG_CONFIG, INSTALL, MBITS
  6 | 
  7 | 
  8 | mkl_require host
  9 | 
 10 | function checks {
 11 | 
 12 |     # C compiler
 13 |     mkl_meta_set "ccenv" "name" "C compiler from CC env"
 14 |     if ! mkl_command_check "ccenv" "WITH_CC" cont "$CC --version"; then
 15 |         if mkl_command_check "gcc" "WITH_GCC" cont "gcc --version"; then
 16 |             CC=gcc
 17 |         elif mkl_command_check "clang" "WITH_CLANG" cont "clang --version"; then
 18 |             CC=clang
 19 |         elif mkl_command_check "cc" "WITH_CC" fail "cc --version"; then
 20 |             CC=cc
 21 |         fi
 22 |     fi
 23 |     export CC="${CC}"
 24 |     mkl_mkvar_set CC CC "$CC"
 25 | 
 26 |     if [[ $MKL_CC_WANT_CXX == 1 ]]; then
 27 |     # C++ compiler
 28 |         mkl_meta_set "cxxenv" "name" "C++ compiler from CXX env"
 29 |         if ! mkl_command_check "cxxenv" "WITH_CXX" cont "$CXX --version" ; then
 30 |             mkl_meta_set "gxx" "name" "C++ compiler (g++)"
 31 |             mkl_meta_set "clangxx" "name" "C++ compiler (clang++)"
 32 |             mkl_meta_set "cxx" "name" "C++ compiler (c++)"
 33 |             if mkl_command_check "gxx" "WITH_GXX" cont "g++ --version"; then
 34 |                 CXX=g++
 35 |             elif mkl_command_check "clangxx" "WITH_CLANGXX" cont "clang++ --version"; then
 36 |                 CXX=clang++
 37 |             elif mkl_command_check "cxx" "WITH_CXX" fail "c++ --version"; then
 38 |                 CXX=c++
 39 |             fi
 40 |         fi
 41 |         export CXX="${CXX}"
 42 |         mkl_mkvar_set "CXX" CXX $CXX
 43 |     fi
 44 | 
 45 |     # Handle machine bits, if specified.
 46 |     if [[ ! -z "$MBITS" ]]; then
 47 | 	mkl_meta_set mbits_m name "mbits compiler flag (-m$MBITS)"
 48 | 	if mkl_compile_check mbits_m "" fail CC "-m$MBITS"; then
 49 | 	    mkl_mkvar_append CPPFLAGS CPPFLAGS "-m$MBITS"
 50 | 	    mkl_mkvar_append LDFLAGS LDFLAGS "-m$MBITS"
 51 | 	fi
 52 | 	if [[ -z "$ARFLAGS" && $MBITS == 64 && $MKL_DISTRO == "SunOS" ]]; then
 53 | 	    # Turn on 64-bit archives on SunOS
 54 | 	    mkl_mkvar_append ARFLAGS ARFLAGS "S"
 55 | 	fi
 56 |     fi
 57 | 
 58 |     # Provide prefix and checks for various other build tools.
 59 |     local t=
 60 |     for t in LD:ld NM:nm OBJDUMP:objdump STRIP:strip ; do
 61 |         local tenv=${t%:*}
 62 |         t=${t#*:}
 63 | 	local tval="${!tenv}"
 64 | 
 65 |         [[ -z $tval ]] && tval="$t"
 66 | 
 67 |         if mkl_prog_check "$t" "" disable "$tval" ; then
 68 |             if [[ $tval != ${!tenv} ]]; then
 69 | 		export "$tenv"="$tval"
 70 | 	    fi
 71 |             mkl_mkvar_set $tenv $tenv "$tval"
 72 |         fi
 73 |     done
 74 | 
 75 |     # Compiler and linker flags
 76 |     [[ ! -z $CFLAGS ]]   && mkl_mkvar_set "CFLAGS" "CFLAGS" "$CFLAGS"
 77 |     [[ ! -z $CPPFLAGS ]] && mkl_mkvar_set "CPPFLAGS" "CPPFLAGS" "$CPPFLAGS"
 78 |     [[ ! -z $CXXFLAGS ]] && mkl_mkvar_set "CXXFLAGS" "CXXFLAGS" "$CXXFLAGS"
 79 |     [[ ! -z $LDFLAGS ]]  && mkl_mkvar_set "LDFLAGS" "LDFLAGS" "$LDFLAGS"
 80 |     [[ ! -z $ARFLAGS ]]  && mkl_mkvar_set "ARFLAGS" "ARFLAGS" "$ARFLAGS"
 81 | 
 82 |     if [[ $MKL_NO_DEBUG_SYMBOLS != "y" ]]; then
 83 |         # Add debug symbol flag (-g)
 84 |         # OSX 10.9 requires -gstrict-dwarf for some reason.
 85 |         mkl_meta_set cc_g_dwarf name "debug symbols compiler flag (-g...)"
 86 |         if [[ $MKL_DISTRO == "osx" ]]; then
 87 |             if mkl_compile_check cc_g_dwarf "" cont CC "-gstrict-dwarf"; then
 88 |                 mkl_mkvar_append CPPFLAGS CPPFLAGS "-gstrict-dwarf"
 89 |             else
 90 |                 mkl_mkvar_append CPPFLAGS CPPFLAGS "-g"
 91 |             fi
 92 |         else
 93 |             mkl_mkvar_append CPPFLAGS CPPFLAGS "-g"
 94 |         fi
 95 |     fi
 96 | 
 97 | 
 98 |     # pkg-config
 99 |     if [ -z "$PKG_CONFIG" ]; then
100 |         PKG_CONFIG=pkg-config
101 |     fi
102 | 
103 |     if mkl_command_check "pkgconfig" "WITH_PKGCONFIG" cont "$PKG_CONFIG --version"; then
104 |         export PKG_CONFIG
105 |     fi
106 |     mkl_mkvar_set "pkgconfig" PKG_CONFIG $PKG_CONFIG
107 | 
108 |     [[ ! -z "$PKG_CONFIG_PATH" ]] && mkl_env_append PKG_CONFIG_PATH "$PKG_CONFIG_PATH"
109 | 
110 |     # install
111 |     if [ -z "$INSTALL" ]; then
112 | 	if [[ $MKL_DISTRO == "SunOS" ]]; then
113 | 	    mkl_meta_set ginstall name "GNU install"
114 | 	    if mkl_command_check ginstall "" ignore "ginstall --version"; then
115 | 		INSTALL=ginstall
116 | 	    else
117 | 		INSTALL=install
118 | 	    fi
119 |         else
120 |             INSTALL=install
121 | 	fi
122 |     fi
123 | 
124 |     if mkl_command_check "install" "WITH_INSTALL" cont "$INSTALL --version"; then
125 |         export INSTALL
126 |     fi
127 |     mkl_mkvar_set "install" INSTALL $INSTALL
128 | 
129 | 
130 |     # Enable profiling if desired
131 |     if [[ $WITH_PROFILING == y ]]; then
132 |         mkl_allvar_set "" "WITH_PROFILING" "y"
133 |         mkl_mkvar_append CPPFLAGS CPPFLAGS "-pg"
134 |         mkl_mkvar_append LDFLAGS LDFLAGS   "-pg"
135 |     fi
136 | 
137 |     # Optimization
138 |     if [[ $WITHOUT_OPTIMIZATION == n ]]; then
139 |         mkl_mkvar_append CPPFLAGS CPPFLAGS "-O2"
140 |     else
141 |         mkl_mkvar_append CPPFLAGS CPPFLAGS "-O0"
142 |     fi
143 | 
144 |     # Static linking
145 |     if [[ $WITH_STATIC_LINKING == y ]]; then
146 |         # LDFLAGS_STATIC is the LDFLAGS needed to enable static linking
147 |         # of sub-sequent libraries, while
148 |         # LDFLAGS_DYNAMIC is the LDFLAGS needed to enable dynamic linking.
149 |         mkl_mkvar_set staticlinking LDFLAGS_STATIC  "-Wl,-Bstatic"
150 |         mkl_mkvar_set staticlinking LDFLAGS_DYNAMIC "-Wl,-Bdynamic"
151 |     fi
152 | }
153 | 
154 | 
155 | mkl_option "Compiler" "env:CC" "--cc=CC" "Build using C compiler CC" "\$CC"
156 | mkl_option "Compiler" "env:CXX" "--cxx=CXX" "Build using C++ compiler CXX" "\$CXX"
157 | mkl_option "Compiler" "ARCH" "--arch=ARCH" "Build for architecture" "$(uname -m)"
158 | mkl_option "Compiler" "CPU" "--cpu=CPU" "Build and optimize for specific CPU" "generic"
159 | mkl_option "Compiler" "MBITS" "--mbits=BITS" "Machine bits (32 or 64)" ""
160 | 
161 | for n in CFLAGS CPPFLAGS CXXFLAGS LDFLAGS ARFLAGS; do
162 |     mkl_option "Compiler" "mk:$n" "--$n=$n" "Add $n flags"
163 | done
164 | 
165 | mkl_option "Compiler" "env:PKG_CONFIG_PATH" "--pkg-config-path" "Extra paths for pkg-config"
166 | 
167 | mkl_option "Compiler" "WITH_PROFILING" "--enable-profiling" "Enable profiling"
168 | mkl_option "Compiler" "WITH_STATIC_LINKING" "--enable-static" "Enable static linking"
169 | mkl_option "Compiler" "WITHOUT_OPTIMIZATION" "--disable-optimization" "Disable optimization flag to compiler" "n"
170 | mkl_option "Compiler" "env:MKL_NO_DEBUG_SYMBOLS" "--disable-debug-symbols" "Disable debugging symbols" "n"
171 | mkl_option "Compiler" "env:MKL_WANT_WERROR" "--enable-werror" "Enable compiler warnings as errors" "n"
172 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.gitversion:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | #
 3 | # Sets version variable from git information.
 4 | # Optional arguments:
 5 | #   "as"
 6 | #   VARIABLE_NAME
 7 | #
 8 | # Example: Set version in variable named "MYVERSION":
 9 | #   mkl_require gitversion as MYVERSION
10 | 
11 | if [[ $1 == "as" ]]; then
12 |     __MKL_GITVERSION_VARNAME="$2"
13 | else
14 |     __MKL_GITVERSION_VARNAME="VERSION"
15 | fi
16 | 
17 | function checks {
18 |     mkl_allvar_set "gitversion" "$__MKL_GITVERSION_VARNAME" "$(git describe --abbrev=6 --tags HEAD --always)"
19 | }
20 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.good_cflags:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | #
 3 | # Provides some known-good CFLAGS
 4 | # Sets:
 5 | #  CFLAGS
 6 | #  CXXFLAGS
 7 | #  CPPFLAGS
 8 | 
 9 | 
10 | function checks {
11 |     mkl_mkvar_append CPPFLAGS CPPFLAGS \
12 |         "-Wall -Wsign-compare -Wfloat-equal -Wpointer-arith"
13 | 
14 |     if [[ $MKL_WANT_WERROR = "y" ]]; then
15 |         mkl_mkvar_append CPPFLAGS CPPFLAGS \
16 |             "-Werror"
17 |     fi
18 | }
19 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.host:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | #
  3 | # Host OS support
  4 | # Sets:
  5 | #  HOST
  6 | #  BUILD
  7 | #  TARGET
  8 | 
  9 | # FIXME: No need for this right now
 10 | #mkl_require host_linux
 11 | #mkl_require host_osx
 12 | #mkl_require host_cygwin
 13 | 
 14 | #mkl_option "Cross-compilation" "mk:HOST_OS" "--host-os=osname" "Host OS (linux,osx,cygwin,..)" "auto"
 15 | 
 16 | 
 17 | # autoconf compatibility - does nothing at this point
 18 | mkl_option "Cross-compilation" "mk:HOST" "--host=HOST" "Configure to build programs to run on HOST (no-op)"
 19 | mkl_option "Cross-compilation" "mk:BUILD" "--build=BUILD" "Configure for building on BUILD (no-op)"
 20 | mkl_option "Cross-compilation" "mk:TARGET" "--target=TARGET" "Configure for building cross-toolkits for platform TARGET (no-op)"
 21 | 
 22 | function checks {
 23 |     # Try to figure out what OS/distro we are running on.
 24 |     mkl_check_begin "distro" "" "no-cache" "OS or distribution"
 25 | 
 26 |     # Try lsb_release
 27 |     local sys
 28 |     sys=$(lsb_release -is 2>/dev/null)
 29 |     if [[ $? -gt 0 ]]; then
 30 |         # That didnt work, try uname.
 31 |         local kn=$(uname -s)
 32 |         case $kn in
 33 |             Linux)
 34 |                 sys=Linux
 35 |                 ;;
 36 |             Darwin)
 37 |                 sys=osx
 38 |                 ;;
 39 |             CYGWIN*)
 40 |                 sys=Cygwin
 41 |                 ;;
 42 |             *)
 43 |                 sys="$kn"
 44 |                 ;;
 45 |         esac
 46 |     fi
 47 | 
 48 |     if [[ -z $sys ]]; then
 49 |         mkl_check_failed "distro" "" "ignore" ""
 50 |     else
 51 |         mkl_check_done "distro" "" "ignore" "ok" "$sys"
 52 |         mkl_mkvar_set "distro" "MKL_DISTRO" "$sys"
 53 |     fi
 54 | }
 55 | 
 56 | #function checks {
 57 | #    mkl_check_begin "host" "HOST_OS" "no-cache" "host OS"
 58 | #
 59 | #    #
 60 | #    # If --host-os=.. was not specified then this is most likely not a
 61 | #    # a cross-compilation and we can base the host-os on the native OS.
 62 | #    #
 63 | #    if [[ $HOST_OS != "auto" ]]; then
 64 | #        mkl_check_done "host" "HOST_OS" "cont" "ok" "$HOST_OS"
 65 | #        return 0
 66 | #    fi
 67 | #
 68 | #    kn=$(uname -s)
 69 | #    case $kn in
 70 | #        Linux)
 71 | #            hostos=linux
 72 | #            ;;
 73 | #        Darwin)
 74 | #            hostos=osx
 75 | #            ;;
 76 | #        CYGWIN*)
 77 | #            hostos=cygwin
 78 | #            ;;
 79 | #        *)
 80 | #            hostos="$(mkl_lower $kn)"
 81 | #            mkl_err  "Unknown host OS kernel name: $kn"
 82 | #            mkl_err0 "  Will attempt to load module host_$hostos anyway."
 83 | #            mkl_err0 "  Please consider writing a configure.host_$hostos"
 84 | #            ;;
 85 | #    esac
 86 | #
 87 | #    if ! mkl_require --try "host_$hostos"; then
 88 | #        # Module not found
 89 | #        mkl_check_done "host" "HOST_OS" "cont" "failed" "$kn?"
 90 | #    else
 91 | #        # Module loaded
 92 | #
 93 | #        if mkl_func_exists "host_${hostos}_setup" ; then
 94 | #            "host_${hostos}_setup"
 95 | #        fi
 96 | #
 97 | #        mkl_check_done "host" "HOST_OS" "cont" "ok" "$hostos"
 98 | #    fi
 99 | #
100 | #    # Set HOST_OS var even if probing failed.
101 | #    mkl_mkvar_set "host" "HOST_OS" "$hostos"
102 | #}
103 | 
104 | 


--------------------------------------------------------------------------------
/mklove/modules/configure.socket:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | #
 3 | # Provides proper compiler flags for socket support, e.g. socket(3).
 4 | 
 5 | function checks {
 6 | 
 7 |     local src="
 8 | #include <sys/types.h>
 9 | #include <sys/socket.h>
10 | #include <unistd.h>
11 | void foo (void);
12 | void foo (void) {
13 |    int s = socket(0, 0, 0);
14 |    close(s);
15 | }"
16 |     if ! mkl_compile_check socket "" cont CC "" "$src"; then
17 | 	if mkl_compile_check --ldflags="-lsocket -lnsl" socket_nsl "" fail CC "" "$src"; then
18 | 	    mkl_mkvar_append socket_nsl LIBS "-lsocket -lnsl"
19 | 	fi
20 |     fi
21 | }
22 | 


--------------------------------------------------------------------------------
/n2kafka.service:
--------------------------------------------------------------------------------
 1 | [Unit]
 2 | Description=Network socket to Kafka messaging gateway
 3 | 
 4 | [Service]
 5 | ExecStart=/usr/bin/n2kafka /etc/n2kafka_config.json
 6 | Restart=always
 7 | 
 8 | [Install]
 9 | WantedBy=multi-user.target
10 | 


--------------------------------------------------------------------------------
/packaging/rpm/Makefile:
--------------------------------------------------------------------------------
 1 | PACKAGE_NAME?=	n2kafka
 2 | 
 3 | #VERSION?=	$(shell git describe --tags --long | sed 's/-/_/g')
 4 | VERSION?=	$(shell git describe --abbrev=6 --tags HEAD --always | sed 's/-/_/g')
 5 | 
 6 | BUILD_NUMBER?= 1
 7 | 
 8 | MOCK_CONFIG?=default
 9 | 
10 | RESULT_DIR?=pkgs
11 | 
12 | all: rpm
13 | 
14 | 
15 | SOURCES:
16 | 	mkdir -p SOURCES
17 | 
18 | archive: SOURCES
19 | 	cd ../../ && \
20 | 	git archive --prefix=$(PACKAGE_NAME)-$(VERSION)/ \
21 | 		-o packaging/rpm/SOURCES/$(PACKAGE_NAME)-$(VERSION).tar.gz HEAD
22 | 
23 | 
24 | build_prepare: archive
25 | 	mkdir -p $(RESULT_DIR)
26 | 	rm -f $(RESULT_DIR)/$(PACKAGE_NAME)*.rpm
27 | 
28 | 
29 | srpm: build_prepare
30 | 	/usr/bin/mock \
31 | 		-r $(MOCK_CONFIG) \
32 | 		--define "__version $(VERSION)" \
33 | 		--define "__release $(BUILD_NUMBER)" \
34 | 		--resultdir=$(RESULT_DIR) \
35 | 		--buildsrpm \
36 | 		--spec=n2kafka.spec \
37 | 		--sources=SOURCES
38 | 	@echo "======= Source RPM now available in $(RESULT_DIR) ======="
39 | 
40 | rpm: srpm
41 | 	/usr/bin/mock \
42 | 		-r $(MOCK_CONFIG) \
43 | 		--define "__version $(VERSION)"\
44 | 		--define "__release $(BUILD_NUMBER)"\
45 | 		--resultdir=$(RESULT_DIR) \
46 | 		--rebuild $(RESULT_DIR)/$(PACKAGE_NAME)*.src.rpm
47 | 	@echo "======= Binary RPMs now available in $(RESULT_DIR) ======="
48 | 
49 | clean:
50 | 	rm -rf SOURCES pkgs
51 | 
52 | distclean: clean
53 | 	rm -f build.log root.log state.log available_pkgs installed_pkgs \
54 | 		*.rpm *.tar.gz
55 | 
56 | 


--------------------------------------------------------------------------------
/packaging/rpm/n2kafka.spec:
--------------------------------------------------------------------------------
 1 | Name:    n2kafka
 2 | Version: %{__version}
 3 | Release: %{__release}%{?dist}
 4 | 
 5 | License: GNU AGPLv3
 6 | URL: https://gitlab.redborder.lan/dfernandez.ext/n2kafka/repository/archive.tar.gz?ref=redborder
 7 | Source0: %{name}-%{version}.tar.gz
 8 | 
 9 | BuildRequires: gcc librd-devel json-c-devel librdkafka-devel libev-devel libmicrohttpd-devel jansson-devel >= 2.7 yajl-devel >= 2.1.0 libcurl-devel >= 7.48
10 | 
11 | Summary: Network messages to json/kafka gateway
12 | Group:   Development/Libraries/C and C++
13 | Requires: librd0 json-c librdkafka libev libmicrohttpd jansson >= 2.7 yajl >= 2.1.0 libcurl >= 7.48
14 | %description
15 | %{summary}
16 | 
17 | %prep
18 | %setup -qn %{name}-%{version}
19 | 
20 | %build
21 | export CFLAGS=" -fcommon"
22 | ./configure --prefix=/usr
23 | make
24 | 
25 | %install
26 | DESTDIR=%{buildroot} make install
27 | mkdir -p %{buildroot}/usr/share/n2kafka
28 | install -D -m 644 n2kafka.service %{buildroot}/usr/lib/systemd/system/n2kafka.service
29 | install -D -m 644 configs_example/n2kafka_config.json.default %{buildroot}/usr/share/n2kafka
30 | 
31 | %clean
32 | rm -rf %{buildroot}
33 | 
34 | %post -p /sbin/ldconfig
35 | %postun -p /sbin/ldconfig
36 | 
37 | %files
38 | %defattr(755,root,root)
39 | /usr/bin/n2kafka
40 | %defattr(644,root,root)
41 | /usr/lib/systemd/system/n2kafka.service
42 | /usr/share/n2kafka/n2kafka_config.json.default
43 | 
44 | %changelog
45 | * Wed May 11 2016 Juan J. Prieto <jjprieto@redborder.com> - 1.0-1
46 | - first spec version
47 | 
48 | 
49 | 


--------------------------------------------------------------------------------
/src/Makefile.mk:
--------------------------------------------------------------------------------
1 | SUBDIRS := \
2 | 	decoder \
3 | 	engine \
4 | 	listener \
5 | 	util \
6 | 
7 | include $(addprefix $(CURRENT_N2KAFKA_DIR), $(addsuffix /Makefile.mk,$(SUBDIRS)))
8 | 
9 | SUBDIRS :=


--------------------------------------------------------------------------------
/src/decoder/Makefile.mk:
--------------------------------------------------------------------------------
1 | SUBDIRS := \
2 | 	rb_http2k \
3 | 	mse \
4 | 	meraki
5 | 
6 | include $(addprefix $(CURRENT_N2KAFKA_DIR), $(addsuffix /Makefile.mk, $(SUBDIRS)))
7 | SUBDIRS :=


--------------------------------------------------------------------------------
/src/decoder/meraki/Makefile.mk:
--------------------------------------------------------------------------------
1 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),rb_meraki.c)
2 | 
3 | 


--------------------------------------------------------------------------------
/src/decoder/meraki/rb_meraki.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "util/pair.h"
24 | 
25 | #include <stdint.h>
26 | #include <string.h>
27 | #include <pthread.h>
28 | 
29 | /* All functions are thread-safe here, excepting free_valid_meraki_database */
30 | 
31 | struct json_t;
32 | struct meraki_database {
33 | 	/* Private */
34 | 	pthread_rwlock_t rwlock;
35 | 	struct json_t *root;
36 | };
37 | 
38 | static void init_meraki_database(struct meraki_database *db) __attribute__((unused));
39 | static void init_meraki_database(struct meraki_database *db) {
40 | 	pthread_rwlock_init(&db->rwlock,0);
41 | 	db->root = NULL;
42 | }
43 | 
44 | int meraki_opaque_creator(struct json_t *config,void **opaque);
45 | int meraki_opaque_reload(struct json_t *config,void *opaque);
46 | void meraki_opaque_destructor(void *opaque);
47 | 
48 | int parse_meraki_secrets(void *db, const struct json_t *meraki_object);
49 | 
50 | void meraki_database_done(struct meraki_database *db);
51 | 
52 | struct meraki_config {
53 | 	struct meraki_database database;
54 | };
55 | 
56 | void meraki_decode(char *buffer,size_t buf_size,
57 | 	const keyval_list_t *attrs,void *listener_callback_opaque,
58 | 	void **sessionp);
59 | 


--------------------------------------------------------------------------------
/src/decoder/mse/Makefile.mk:
--------------------------------------------------------------------------------
1 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),rb_mse.c)
2 | 


--------------------------------------------------------------------------------
/src/decoder/mse/rb_mse.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include <stdint.h>
24 | #include <string.h>
25 | #include <pthread.h>
26 | #include "util/pair.h"
27 | 
28 | /* All functions are thread-safe here, excepting free_valid_mse_database */
29 | 
30 | struct json_t;
31 | struct mse_database {
32 | 	/* Private */
33 | 	pthread_mutex_t warning_ht_lock;
34 | 	struct json_t *warning_ht;
35 | 	pthread_rwlock_t rwlock;
36 | 	struct json_t *root;
37 | };
38 | void init_mse_database(struct mse_database *db);
39 | int parse_mse_array(void *_db, const struct json_t *mse_array);
40 | void free_valid_mse_database(struct mse_database *db);
41 | 
42 | struct mse_config {
43 | 	struct mse_database database;
44 | };
45 | 
46 | int mse_opaque_creator(struct json_t *config, void **opaque);
47 | int mse_opaque_reload(struct json_t *config, void *opaque);
48 | void mse_opaque_done(void *opaque);
49 | void mse_decode(char *buffer, size_t buf_size,
50 |                 const keyval_list_t *keyval,
51 |                 void *listener_callback_opaque,
52 |                 void **sessionp);
53 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/Makefile.mk:
--------------------------------------------------------------------------------
 1 | THIS_SRCS := \
 2 | 	rb_http2k_decoder.c \
 3 | 	rb_database.c \
 4 | 	uuid_database.c \
 5 | 	rb_http2k_parser.c \
 6 | 	rb_http2k_sensors_database.c \
 7 | 	rb_http2k_sync_thread.c \
 8 | 	rb_http2k_curl_handler.c \
 9 | 	rb_http2k_organizations_database.c \
10 | 	tommyds/tommyhash.c \
11 | 	tommyds/tommyhashdyn.c \
12 | 	tommyds/tommylist.c \
13 | 
14 | SRCS := $(SRCS) $(addprefix $(CURRENT_N2KAFKA_DIR),$(THIS_SRCS))
15 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_database.c:
--------------------------------------------------------------------------------
  1 | /*
  2 | **
  3 | ** Copyright (c) 2014, Eneo Tecnologia
  4 | ** Author: Eugenio Perez <eupm90@gmail.com>
  5 | ** All rights reserved.
  6 | **
  7 | ** This program is free software; you can redistribute it and/or modify
  8 | ** This program is free software; you can redistribute it and/or modify
  9 | ** it under the terms of the GNU Affero General Public License as
 10 | ** published by the Free Software Foundation, either version 3 of the
 11 | ** License, or (at your option) any later version.
 12 | **
 13 | ** This program is distributed in the hope that it will be useful,
 14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
 15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 16 | ** GNU Affero General Public License for more details.
 17 | **
 18 | ** You should have received a copy of the GNU General Public License
 19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
 20 | */
 21 | 
 22 | #include "rb_database.h"
 23 | #include "util/topic_database.h"
 24 | 
 25 | #include <librd/rdlog.h>
 26 | 
 27 | #include <string.h>
 28 | #include <assert.h>
 29 | #include <stdlib.h>
 30 | #include <stdio.h>
 31 | #include <jansson.h>
 32 | #include <errno.h>
 33 | 
 34 | int init_rb_database(struct rb_database *db) {
 35 | 	char errbuf[BUFSIZ];
 36 | 
 37 | 	memset(db, 0, sizeof(*db));
 38 | 	const int pthread_rc = pthread_rwlock_init(&db->rwlock, 0);
 39 | 
 40 | 	if (pthread_rc != 0) {
 41 | 		strerror_r(errno, errbuf, sizeof(errbuf));
 42 | 		rdlog(LOG_ERR, "Can't start rwlock: %s", errbuf);
 43 | 		return pthread_rc;
 44 | 	}
 45 | 
 46 | 	const int odb_init = organizations_db_init(&db->organizations_db);
 47 | 	if (odb_init != 0) {
 48 | 		return odb_init;
 49 | 	}
 50 | 
 51 | 	return 0;
 52 | }
 53 | 
 54 | void free_valid_rb_database(struct rb_database *db) {
 55 | 	if (db) {
 56 | 		if (db->sensors_db) {
 57 | 			sensors_db_destroy(db->sensors_db);
 58 | 		}
 59 | 
 60 | 		if (db->topics_db) {
 61 | 			topics_db_done(db->topics_db);
 62 | 		}
 63 | 
 64 | 		organizations_db_done(&db->organizations_db);
 65 | 		pthread_rwlock_destroy(&db->rwlock);
 66 | 	}
 67 | }
 68 | 
 69 | int rb_http2k_database_get_topic_client(struct rb_database *db,
 70 | 		const char *topic, const char *sensor_uuid,
 71 | 		struct topic_s **topic_handler,
 72 | 		sensor_db_entry_t **client_enrichment) {
 73 | 	assert(db);
 74 | 	assert(topic_handler);
 75 | 	assert(client_enrichment);
 76 | 
 77 | 	pthread_rwlock_rdlock(&db->rwlock);
 78 | 	*topic_handler = topics_db_get_topic(db->topics_db, topic);
 79 | 
 80 | 	if(*topic_handler) {
 81 | 		*client_enrichment = sensors_db_get(db->sensors_db, sensor_uuid);
 82 | 	}
 83 | 	pthread_rwlock_unlock(&db->rwlock);
 84 | 
 85 | 	return NULL != *topic_handler && NULL != *client_enrichment;
 86 | }
 87 | 
 88 | int rb_http2k_validate_uuid(struct rb_database *db, const char *sensor_uuid) {
 89 | 	pthread_rwlock_rdlock(&db->rwlock);
 90 | 	const int ret = sensors_db_exists(db->sensors_db, sensor_uuid);
 91 | 	pthread_rwlock_unlock(&db->rwlock);
 92 | 
 93 | 	return ret;
 94 | }
 95 | 
 96 | int rb_http2k_validate_topic(struct rb_database *db, const char *topic) {
 97 |        pthread_rwlock_rdlock(&db->rwlock);
 98 |        const int ret = topics_db_topic_exists(db->topics_db,topic);
 99 |        pthread_rwlock_unlock(&db->rwlock);
100 | 
101 |        return ret;
102 | }
103 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_database.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "util/topic_database.h"
24 | #include "rb_http2k_sensors_database.h"
25 | #include "rb_http2k_organizations_database.h"
26 | 
27 | #include "util/rb_timer.h"
28 | 
29 | #include <pthread.h>
30 | #include <jansson.h>
31 | 
32 | struct rb_database {
33 | 	/* UUID enrichment read-only database */
34 | 	pthread_rwlock_t rwlock;
35 | 	/// sensors UUID database.
36 | 	sensors_db_t *sensors_db;
37 | 	/// Organizations database
38 | 	organizations_db_t organizations_db;
39 | 	/// Timer to send stats via rb_monitor topic
40 | 	struct topics_db *topics_db;
41 | 
42 | 	void *topics_memory;
43 | };
44 | 
45 | /** Initialized a rb_database
46 |   @param db database to init
47 |   @return 0 if success, !0 in other case
48 |   */
49 | int init_rb_database(struct rb_database *db);
50 | void free_valid_rb_database(struct rb_database *db);
51 | 
52 | /**
53 | 	Get sensor enrichment and topic of an specific database.
54 | 
55 | 	@param db Database to extract sensor and topic handler from
56 | 	@param topic Topic to search for
57 | 	@param sensor_uuid Sensor uuid to search for
58 | 	@param topic_handler Returned topic handler. Need to be freed with
59 | 	topic_decref
60 | 	@param sensor_info Returned sensor information. Need to be freed
61 | 	with sensor_db_entry_decref
62 | 	@return 0 if OK, !=0 in other case
63 | 	*/
64 | int rb_http2k_database_get_topic_client(struct rb_database *db,
65 | 	const char *topic, const char *sensor_uuid,
66 | 	struct topic_s **topic_handler, sensor_db_entry_t **sensor_info);
67 | 
68 | int rb_http2k_validate_uuid(struct rb_database *db,const char *uuid);
69 | int rb_http2k_validate_topic(struct rb_database *db,const char *topic);
70 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_curl_handler.c:
--------------------------------------------------------------------------------
  1 | /*
  2 | **
  3 | ** Copyright (c) 2014, Eneo Tecnologia
  4 | ** Author: Eugenio Perez <eupm90@gmail.com>
  5 | ** All rights reserved.
  6 | **
  7 | ** This program is free software; you can redistribute it and/or modify
  8 | ** This program is free software; you can redistribute it and/or modify
  9 | ** it under the terms of the GNU Affero General Public License as
 10 | ** published by the Free Software Foundation, either version 3 of the
 11 | ** License, or (at your option) any later version.
 12 | **
 13 | ** This program is distributed in the hope that it will be useful,
 14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
 15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 16 | ** GNU Affero General Public License for more details.
 17 | **
 18 | ** You should have received a copy of the GNU General Public License
 19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
 20 | */
 21 | 
 22 | #include "config.h"
 23 | #include "rb_http2k_curl_handler.h"
 24 | 
 25 | #include <util/util.h>
 26 | 
 27 | #include <librd/rdthread.h>
 28 | #include <librd/rdlog.h>
 29 | 
 30 | typedef struct rb_http2k_curl_handler_msg_s {
 31 | #ifndef NDEBUG
 32 | #define RB_HTTP2k_CURL_HANDLER_MSG 0xB112CA35B112CA35
 33 | 	uint64_t magic;
 34 | #endif
 35 | 	CURL *curl_handler;
 36 | } rb_http2k_curl_handler_msg_t;
 37 | 
 38 | static void assert_rb_http2k_curl_handler_ctx(
 39 | 					rb_http2k_curl_handler_t *handler) {
 40 | #ifdef RB_HTTP2K_CURL_HANDLER_MAGIC
 41 | 	assert(RB_HTTP2K_CURL_HANDLER_MAGIC == handler->magic);
 42 | #else
 43 | 	(void)handler;
 44 | #endif
 45 | }
 46 | 
 47 | /** Overwrite default write_callback, that prints out content by stdout */
 48 | static size_t write_callback(char *ptr, size_t size, size_t nmemb,
 49 | 							void *userdata) {
 50 | 	(void)ptr;
 51 | 	(void)userdata;
 52 | 
 53 | 	return size*nmemb;
 54 | }
 55 | 
 56 | /* Convenience function */
 57 | static CURL *my_rd_fifoq_pop_timedwait(rd_fifoq_t *fifoq, int timeout_ms) {
 58 | 	rd_fifoq_elm_t *rfqe = rd_fifoq_pop_timedwait(fifoq, timeout_ms);
 59 | 	CURL *ret = NULL;
 60 | 
 61 | 	if (rfqe) {
 62 | 		ret = rfqe->rfqe_ptr;
 63 | 		rd_fifoq_elm_release(fifoq, rfqe);
 64 | 	}
 65 | 
 66 | 	return ret;
 67 | }
 68 | 
 69 | static void curl_handler_add_curl(rb_http2k_curl_handler_t *handler,
 70 | 							CURL *curl_handler) {
 71 | 	rdlog(LOG_DEBUG, "Adding HTTP PUT %p", curl_handler);
 72 | 	curl_multi_add_handle(handler->curl_multi_handler, curl_handler);
 73 | }
 74 | 
 75 | /** Purge completed transfers
 76 |   @param handler Handler to purge
 77 |   @return Number of transfers cleaned
 78 |   */
 79 | static int curl_handler_purge_completed(rb_http2k_curl_handler_t *handler) {
 80 | 	CURLMsg *msg;
 81 | 	int cleaned = 0;
 82 | 	int msgs_in_queue;
 83 | 	while((msg = curl_multi_info_read(handler->curl_multi_handler,
 84 | 							&msgs_in_queue))) {
 85 | 		if(msg->msg == CURLMSG_DONE) {
 86 | 			CURL *e = msg->easy_handle;
 87 | 			rdlog(LOG_DEBUG, "HTTP PUT %p end", msg);
 88 | 			curl_multi_remove_handle(handler->curl_multi_handler,
 89 | 									e);
 90 | 			curl_easy_cleanup(e);
 91 | 			cleaned++;
 92 | 		} else {
 93 | 			rdlog(LOG_CRIT, "Unknown message returned!!");
 94 | 		}
 95 | 	}
 96 | 
 97 | 	return cleaned;
 98 | }
 99 | 
100 | static void curl_handler_entry_point0(rb_http2k_curl_handler_t *handler) {
101 | 	static int fifoq_pop_timeout_ms = 500;
102 | 	int my_still_running = 0;
103 | 	while(ATOMIC_OP(fetch,add,&handler->run,0)) {
104 | 		int still_running = 0;
105 | 		CURL *elm = my_rd_fifoq_pop_timedwait(&handler->msg_queue,
106 | 							fifoq_pop_timeout_ms);
107 | 		if (elm) {
108 | 			my_still_running++;
109 | 			curl_handler_add_curl(handler, elm);
110 | 		}
111 | 
112 | 		curl_multi_perform(handler->curl_multi_handler,
113 | 							&still_running);
114 | 
115 | 		if (still_running < my_still_running) {
116 | 			/* Some handlers require attention */
117 | 			const int cleaned = curl_handler_purge_completed(
118 | 								handler);
119 | 			my_still_running -= cleaned;
120 | 		}
121 | 	}
122 | 
123 | }
124 | 
125 | static void *curl_handler_entry_point(void *ctx) {
126 | 	rb_http2k_curl_handler_t *handler = ctx;
127 | 	assert_rb_http2k_curl_handler_ctx(handler);
128 | 	curl_handler_entry_point0(handler);
129 | 	return NULL;
130 | }
131 | 
132 | int rb_http2k_curl_handler_init(rb_http2k_curl_handler_t *handler,
133 | 							int max_msgs_size) {
134 | 	char err[BUFSIZ];
135 | #ifdef RB_HTTP2K_CURL_HANDLER_MAGIC
136 | 	handler->magic = RB_HTTP2K_CURL_HANDLER_MAGIC;
137 | #endif
138 | 	handler->run = 1;
139 | 
140 | 	rd_fifoq_init(&handler->msg_queue);
141 | 	rd_fifoq_set_max_size(&handler->msg_queue, max_msgs_size, 0);
142 | 	curl_global_init(CURL_GLOBAL_DEFAULT);
143 | 	handler->curl_multi_handler = curl_multi_init();
144 | 	if (NULL == handler->curl_multi_handler) {
145 | 		rdlog(LOG_ERR, "Couldn create curl_multi_handler.");
146 | 		return -1;
147 | 	}
148 | 
149 | 	const int pthread_create_rc = pthread_create(&handler->thread, NULL,
150 | 		curl_handler_entry_point, handler);
151 | 
152 | 	if (pthread_create_rc != 0) {
153 | 		rdlog(LOG_ERR, "Couldn't create thread: %s",
154 | 			mystrerror(errno, err, sizeof(err)));
155 | 		curl_multi_cleanup(handler->curl_multi_handler);
156 | 	}
157 | 
158 | 	return pthread_create_rc;
159 | }
160 | 
161 | void rb_http2k_curl_handler_done(rb_http2k_curl_handler_t *handler) {
162 | 	ATOMIC_OP(fetch,and,&handler->run,0);
163 | 	pthread_join(handler->thread, NULL);
164 | 	curl_multi_cleanup(handler->curl_multi_handler);
165 | 	curl_global_cleanup();
166 | 	rd_fifoq_destroy(&handler->msg_queue);
167 | }
168 | 
169 | static void curl_easy_set_http_put_method(CURL *curl_handler,
170 | 							const char *url) {
171 | 	curl_easy_setopt(curl_handler, CURLOPT_URL, url);
172 | 	curl_easy_setopt(curl_handler, CURLOPT_UPLOAD, 1);
173 | }
174 | 
175 | /// @TODO test full queue
176 | void rb_http2k_curl_handler_put_empty(rb_http2k_curl_handler_t *handler,
177 | 							const char *url) {
178 | 	CURL *curl_handler = curl_easy_init(), *purged = NULL;
179 | 	curl_easy_set_http_put_method(curl_handler, url);
180 | 	curl_easy_setopt(curl_handler, CURLOPT_WRITEFUNCTION, write_callback);
181 | 
182 | 	// Skip SSL verification
183 | 	curl_easy_setopt(curl_handler, CURLOPT_SSL_VERIFYPEER, 0L);
184 | 
185 | 	rdlog(LOG_DEBUG, "Adding curl handler %p to URL %s", curl_handler,
186 | 									url);
187 | 
188 | 	rd_fifoq_add_purge(&handler->msg_queue,curl_handler,&purged);
189 | 	if (purged) {
190 | 		char *purged_url = NULL;
191 | 		curl_easy_getinfo(purged, CURLINFO_EFFECTIVE_URL, &purged_url);
192 | 		rdlog(LOG_ERR, "Too much request queued. Freeing %s",
193 | 			purged_url ? purged_url : "older");
194 | 		curl_easy_cleanup(purged);
195 | 	}
196 | }
197 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_curl_handler.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** This program is free software; you can redistribute it and/or modify
 9 | ** it under the terms of the GNU Affero General Public License as
10 | ** published by the Free Software Foundation, either version 3 of the
11 | ** License, or (at your option) any later version.
12 | **
13 | ** This program is distributed in the hope that it will be useful,
14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 | ** GNU Affero General Public License for more details.
17 | **
18 | ** You should have received a copy of the GNU General Public License
19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
20 | */
21 | 
22 | #pragma once
23 | 
24 | #include <pthread.h>
25 | #include <librd/rdqueue.h>
26 | #include <curl/curl.h>
27 | 
28 | /** CURL handler to send PUT messages */
29 | typedef struct rb_http2k_curl_handler_s {
30 | #ifndef NDEBUG
31 | #define RB_HTTP2K_CURL_HANDLER_MAGIC 0xB112C3A1CB112C3AL
32 | 	uint64_t magic; 		///< Magic to assert coherence
33 | #endif
34 | 	volatile int run;		///< Keep running
35 | 	pthread_t thread;		///< Thread handler
36 | 	rd_fifoq_t msg_queue;		///< MSG queue
37 | 	CURLM *curl_multi_handler;	///< Curl handler
38 | } rb_http2k_curl_handler_t;
39 | 
40 | /** Creates a rb_http2k_curl_handler
41 |   @param handler Handler to start
42 |   @param max_msgs_size Maximum number of messages to accept
43 |   @return 0 if success, !0 in other case
44 |   */
45 | int rb_http2k_curl_handler_init(rb_http2k_curl_handler_t *handler,
46 | 							int max_msgs_size);
47 | 
48 | /** Free rb_curl handler resources
49 |   @param handler Handler to be freed
50 |   */
51 | void rb_http2k_curl_handler_done(rb_http2k_curl_handler_t *handler);
52 | 
53 | /** Send an empty PUT to a given URL
54 |   @param handler Handler to use to send PUT
55 |   @param URL URL to send it
56 |   @return 0 if OK, ENOBUFS if queue is full
57 |   */
58 | void rb_http2k_curl_handler_put_empty(rb_http2k_curl_handler_t *handler,
59 | 							const char *url);
60 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_decoder.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "rb_database.h"
24 | #include "rb_http2k_sync_thread.h"
25 | #include "rb_http2k_curl_handler.h"
26 | 
27 | #include "util/pair.h"
28 | 
29 | #include <librdkafka/rdkafka.h>
30 | #include <librd/rdavl.h>
31 | #include <librd/rdsysqueue.h>
32 | #include <jansson.h>
33 | 
34 | #include <stdint.h>
35 | #include <string.h>
36 | #include <pthread.h>
37 | 
38 | #ifndef NDEBUG
39 | /// MAGIC to check rb_config between void * conversions
40 | #define RB_CONFIG_MAGIC 0xbc01a1cbc01a1cL
41 | #endif
42 | 
43 | /* All functions are thread-safe here, excepting free_valid_mse_database */
44 | struct json_t;
45 | struct rb_config {
46 | #ifdef RB_CONFIG_MAGIC
47 | 	/// This value always have to be RB_CONFIG_MAGIC
48 | 	uint64_t magic;
49 | #endif
50 | 
51 | 	/// @TODO protect all char * members!
52 | 	struct {
53 | 		/// Sync id to difference between others n2kafka and this one
54 | 		char *n2kafka_id;
55 | 		/// Timer to send organization stats to monitor
56 | 		rb_timer_t *timer;
57 | 		/// Timer to clean stats
58 | 		rb_timer_t *clean_timer;
59 | 		/// Topics to send organization stats.
60 | 		struct rkt_array topics;
61 | 		/// Consumer ctx
62 | 		sync_thread_t thread;
63 | 		/// http related
64 | 		struct {
65 | 			/// CURL handler to send PUT when done
66 | 			rb_http2k_curl_handler_t curl_handler;
67 | 			/// Url to send PUT request.
68 | 			char *url;
69 | 		} http;
70 | 	} organizations_sync;
71 | 	struct rb_database database;
72 | };
73 | 
74 | #ifdef RB_CONFIG_MAGIC
75 | /// Checks that rb_config magic field has the right value
76 | #define assert_rb_config(cfg) do{ \
77 | 	assert(RB_CONFIG_MAGIC==(cfg)->magic);} while(0)
78 | #else
79 | #define assert_rb_config(cfg)
80 | #endif
81 | 
82 | int parse_rb_config(void *_db,const struct json_t *rb_config);
83 | /** Release all resources used */
84 | void rb_decoder_done(void *rb_config);
85 | /** Does nothing, since this decoder does not save anything related to
86 |     listener
87 |     */
88 | int rb_decoder_reload(void *_db, const struct json_t *rb_config);
89 | 
90 | int rb_opaque_creator(struct json_t *config,void **opaque);
91 | int rb_opaque_reload(struct json_t *config,void *opaque);
92 | void rb_opaque_done(void *opaque);
93 | void rb_decode(char *buffer,size_t buf_size,const keyval_list_t *props,
94 |                 void *listener_callback_opaque,void **decoder_sessionp);
95 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_parser.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** This program is free software; you can redistribute it and/or modify
 9 | ** it under the terms of the GNU Affero General Public License as
10 | ** published by the Free Software Foundation, either version 3 of the
11 | ** License, or (at your option) any later version.
12 | **
13 | ** This program is distributed in the hope that it will be useful,
14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 | ** GNU Affero General Public License for more details.
17 | **
18 | ** You should have received a copy of the GNU General Public License
19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
20 | */
21 | 
22 | #pragma once
23 | 
24 | #include "rb_http2k_sensors_database.h"
25 | 
26 | #include <yajl/yajl_parse.h>
27 | #include <yajl/yajl_gen.h>
28 | #include <util/kafka_message_list.h>
29 | #include <jansson.h>
30 | #include <util/pair.h>
31 | 
32 | /// @TODO many of the fields here could be a state machine
33 | /// @TODO separate parsing <-> not parsing fields
34 | /// @TODO could this be private?
35 | struct rb_session {
36 | 	/// Output generator.
37 | 	yajl_gen gen;
38 | 
39 | 	/// JSON handler
40 | 	yajl_handle handler;
41 | 
42 | 	/// Sensor information.
43 | 	sensor_db_entry_t *sensor;
44 | 
45 | 	/// Bookmark if we are skipping an object or array
46 | 	size_t object_array_parsing_stack;
47 | 
48 | 	/// Per POST business.
49 | 	const char *client_ip,*sensor_uuid,*topic,*kafka_partitioner_key;
50 | 
51 | 	/// Topid handler
52 | 	struct topic_s *topic_handler;
53 | 
54 | 	struct {
55 | #define CURRENT_KEY_OFFSET_NOT_SETTED -1
56 | 		/// current kafka message key offset
57 | 		int current_key_offset;
58 | 		size_t current_key_length;
59 | 		int valid;
60 | 	} message;
61 | 
62 | 	/// Message list in this call to decode()
63 | 	rd_kafka_message_queue_t msg_queue;
64 | 
65 | 	/// We are parsing value of kafka_partitioner_key
66 | 	int in_partition_key;
67 | 
68 | 	/// Skip next parsing value
69 | 	int skip_value;
70 | };
71 | 
72 | struct rb_config;
73 | struct rb_session *new_rb_session(struct rb_config *rb_config,
74 | 	                                const keyval_list_t *msg_vars);
75 | 
76 | int gen_jansson_object(yajl_gen gen, json_t *enrichment_data);
77 | 
78 | void free_rb_session(struct rb_session *sess);
79 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_sensors_database.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "uuid_database.h"
24 | #include "rb_http2k_organizations_database.h"
25 | 
26 | #include <jansson.h>
27 | 
28 | /// Sensors database entry
29 | typedef struct sensor_db_entry {
30 | 
31 | #ifndef NDEBUG
32 | 	/* Private data - do not access directly */
33 | 
34 | /// Magic to assert coherency.
35 | #define SENSOR_DB_ENTRY_MAGIC 0x05350DB305350DB3L
36 | 	/// Magic to assert coherency.
37 | 	uint64_t magic;
38 | #endif
39 | 
40 | 	/// Entry of uuid database
41 | 	uuid_entry_t uuid_entry;
42 | 
43 | 	/// Enrichment data
44 | 	json_t *enrichment;
45 | 
46 | 	/// Organization this sensor belongs to
47 | 	organization_db_entry_t *organization;
48 | 
49 | 	/// Reference counter
50 | 	uint64_t refcnt;
51 | } sensor_db_entry_t;
52 | 
53 | /** Obtains sensor uuid */
54 | #define sensor_db_entry_get_uuid(e) ((e)->uuid_entry.uuid)
55 | 
56 | /** Obtains sensor_db_entry enrichment information */
57 | #define sensor_db_entry_json_enrichment(e) ((e)->enrichment)
58 | 
59 | /** Obtains sensor organization */
60 | #define sensor_db_entry_organization(e) ((e)->organization);
61 | 
62 | /** Decrements uuid entry, signaling that we are not going to use it anymore */
63 | void sensor_db_entry_decref(sensor_db_entry_t *entry);
64 | 
65 | /** Sensors uuid database */
66 | struct sensors_db_s;
67 | typedef struct sensors_db_s sensors_db_t;
68 | 
69 | /** Creates a new database
70 |   @param uuids_config Configurations for each sensor
71 |   @param organizations_db Organizations db, each sensor belongs to one
72 |   @returns new database
73 |   */
74 | sensors_db_t *sensors_db_new(json_t *sensors_config,
75 | 					organizations_db_t *organizations_db);
76 | 
77 | /** Get an entry from sensor database.
78 |   @note Obtained entry need to be freed with sensor_db_entry_decref
79 |   @param db database
80 |   @param uuid sensor uuid
81 |   @returns sensor entry
82 |   */
83 | sensor_db_entry_t *sensors_db_get(sensors_db_t *db, const char *sensor_uuid);
84 | 
85 | /** Checks if an entry exists in uuid database.
86 |   @param db database
87 |   @param uuid sensor uuid
88 |   @returns 1 if sensor found, 0 ioc
89 |   */
90 | int sensors_db_exists(sensors_db_t *db, const char *sensor_uuid);
91 | 
92 | /** Destroy a sensor database
93 |   @param db Database to destroy
94 |   */
95 | void sensors_db_destroy(sensors_db_t *db);
96 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_sync_common.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** This program is free software; you can redistribute it and/or modify
 9 | ** it under the terms of the GNU Affero General Public License as
10 | ** published by the Free Software Foundation, either version 3 of the
11 | ** License, or (at your option) any later version.
12 | **
13 | ** This program is distributed in the hope that it will be useful,
14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 | ** GNU Affero General Public License for more details.
17 | **
18 | ** You should have received a copy of the GNU General Public License
19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
20 | */
21 | 
22 | #pragma once
23 | 
24 | static const char MONITOR_MSG_TIMESTAMP_KEY[] = "timestamp";
25 | static const char MONITOR_MSG_MONITOR_KEY[] = "monitor";
26 | static const char MONITOR_MSG_VALUE_KEY[] = "value";
27 | static const char MONITOR_MSG_ORGANIZATION_UUID_KEY[] = "organization_uuid";
28 | static const char MONITOR_MSG_N2KAFKA_ID_KEY[] = "n2kafka_id";
29 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/rb_http2k_sync_thread.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** This program is free software; you can redistribute it and/or modify
 9 | ** it under the terms of the GNU Affero General Public License as
10 | ** published by the Free Software Foundation, either version 3 of the
11 | ** License, or (at your option) any later version.
12 | **
13 | ** This program is distributed in the hope that it will be useful,
14 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 | ** GNU Affero General Public License for more details.
17 | **
18 | ** You should have received a copy of the GNU General Public License
19 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
20 | */
21 | 
22 | #pragma once
23 | 
24 | #include "rb_http2k_organizations_database.h"
25 | 
26 | #include <librdkafka/rdkafka.h>
27 | #include <stdint.h>
28 | #include <pthread.h>
29 | #include <semaphore.h>
30 | 
31 | /// Information shared between main thread and consumer thread
32 | typedef struct sync_thread_s {
33 | #ifndef NDEBUG
34 | #define SYNC_THREAD_MAGIC 0x5c3ada1c5c3ada1c
35 | 	/// Magic to assure coherence
36 | 	uint64_t magic;
37 | #endif
38 | 	/// Thread creation return code
39 | 	int creation_rc;
40 | 	/// Say when a thread should stop
41 | 	volatile int run;
42 | 	/** Clean interval to know how old can we accept reports. Sync interval
43 | 	    are delimited by timestamp T that match:
44 | 	    T%interval_s + offset == 0. So, if we receive a message that is not
45 | 	    in out own interval, we drop it.
46 | 	    */
47 | 	struct {
48 | 		/// Mutex to protect fields
49 | 		pthread_mutex_t mutex;
50 | 		/// Clean interval
51 | 		time_t interval_s;
52 | 		/// Offset on clean.
53 | 		time_t offset_s;
54 | 	} clean_interval;
55 | 	/** Consumer handler */
56 | 	rd_kafka_t *rk;
57 | 	/** Consumer handler base config */
58 | 	rd_kafka_conf_t *rk_conf;
59 | 	/// Thread to consume sync messages
60 | 	pthread_t thread;
61 | 	/// Organization's database we need to act over
62 | 	organizations_db_t *org_db;
63 | 	/// Semaphore to signal right status. Do not use out of
64 | 	/// sync_thread_init
65 | 	sem_t sem;
66 | } sync_thread_t;
67 | 
68 | /** Initializes a consumer topic context
69 |   @param rk Consumer handler
70 |   @param thread Context to initialize
71 |   @return 0 if success, !0 in other case
72 |   */
73 | int sync_thread_init(sync_thread_t *thread, rd_kafka_conf_t *rk,
74 | 						organizations_db_t *org_db);
75 | 
76 | /** De-initialize a consumer topic context
77 |   @param thread Context to free
78 |   */
79 | void sync_thread_done(sync_thread_t *thread);
80 | 
81 | /** Send an async request to consumer thread to update consumer topic. If
82 |     topic is the same as current, no signal is given */
83 | int update_sync_topic(sync_thread_t *thread, rd_kafka_topic_t *topic);
84 | 
85 | /** Update clean_interval. Thread will not accept messages that are in the
86 |   previous interval, i.e., that are supposed to be already cleaned.
87 |   @param thread Thread to update
88 |   @param interval_s Length of an interval
89 |   @param offset_s Offset from 0 that intervals start
90 |   */
91 | void update_sync_thread_clean_interval(sync_thread_t *thread,
92 | 					time_t interval_s, time_t offset_s);
93 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/tommyds/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2010, Andrea Mazzoleni. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions
 5 | are met:
 6 | 
 7 | 1. Redistributions of source code must retain the above copyright
 8 |    notice, this list of conditions and the following disclaimer.
 9 | 
10 | 2. Redistributions in binary form must reproduce the above copyright
11 |    notice, this list of conditions and the following disclaimer in the
12 |    documentation and/or other materials provided with the distribution.
13 | 
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
18 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
24 | POSSIBILITY OF SUCH DAMAGE.
25 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/tommyds/tommyhash.c:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved.
  3 |  *
  4 |  * Redistribution and use in source and binary forms, with or without
  5 |  * modification, are permitted provided that the following conditions
  6 |  * are met:
  7 |  *
  8 |  * 1. Redistributions of source code must retain the above copyright
  9 |  *    notice, this list of conditions and the following disclaimer.
 10 |  *
 11 |  * 2. Redistributions in binary form must reproduce the above copyright
 12 |  *    notice, this list of conditions and the following disclaimer in the
 13 |  *    documentation and/or other materials provided with the distribution.
 14 |  *
 15 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 16 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 17 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 18 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 19 |  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 20 |  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 21 |  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 22 |  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 23 |  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 24 |  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 25 |  * POSSIBILITY OF SUCH DAMAGE.
 26 |  */
 27 | 
 28 | #include "tommyhash.h"
 29 | 
 30 | /******************************************************************************/
 31 | /* hash */
 32 | 
 33 | tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr)
 34 | {
 35 | 	/* allow unaligned read on Intel x86 and x86_64 platforms */
 36 | #if defined(__i386__) || defined(_M_IX86) || defined(_X86_) || defined(__x86_64__) || defined(_M_X64)
 37 | 	/* defines from http://predef.sourceforge.net/ */
 38 | 	return *(const tommy_uint32_t*)ptr;
 39 | #else
 40 | 	const unsigned char* ptr8 = tommy_cast(const unsigned char*, ptr);
 41 | 	return ptr8[0] + ((tommy_uint32_t)ptr8[1] << 8) + ((tommy_uint32_t)ptr8[2] << 16) + ((tommy_uint32_t)ptr8[3] << 24);
 42 | #endif
 43 | }
 44 | 
 45 | #define tommy_rot(x, k) \
 46 | 	(((x) << (k)) | ((x) >> (32 - (k))))
 47 | 
 48 | #define tommy_mix(a, b, c) \
 49 | 	do { \
 50 | 		a -= c;  a ^= tommy_rot(c, 4);  c += b; \
 51 | 		b -= a;  b ^= tommy_rot(a, 6);  a += c; \
 52 | 		c -= b;  c ^= tommy_rot(b, 8);  b += a; \
 53 | 		a -= c;  a ^= tommy_rot(c, 16);  c += b; \
 54 | 		b -= a;  b ^= tommy_rot(a, 19);  a += c; \
 55 | 		c -= b;  c ^= tommy_rot(b, 4);  b += a; \
 56 | 	} while (0)
 57 | 
 58 | #define tommy_final(a, b, c) \
 59 | 	do { \
 60 | 		c ^= b; c -= tommy_rot(b, 14); \
 61 | 		a ^= c; a -= tommy_rot(c, 11); \
 62 | 		b ^= a; b -= tommy_rot(a, 25); \
 63 | 		c ^= b; c -= tommy_rot(b, 16); \
 64 | 		a ^= c; a -= tommy_rot(c, 4);  \
 65 | 		b ^= a; b -= tommy_rot(a, 14); \
 66 | 		c ^= b; c -= tommy_rot(b, 24); \
 67 | 	} while (0)
 68 | 
 69 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len)
 70 | {
 71 | 	const unsigned char* key = tommy_cast(const unsigned char*, void_key);
 72 | 	tommy_uint32_t a, b, c;
 73 | 
 74 | 	a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val;
 75 | 
 76 | 	while (key_len > 12) {
 77 | 		a += tommy_le_uint32_read(key + 0);
 78 | 		b += tommy_le_uint32_read(key + 4);
 79 | 		c += tommy_le_uint32_read(key + 8);
 80 | 
 81 | 		tommy_mix(a, b, c);
 82 | 
 83 | 		key_len -= 12;
 84 | 		key += 12;
 85 | 	}
 86 | 
 87 | 	switch (key_len) {
 88 | 	case 0 :
 89 | 		return c; /* used only when called with a zero length */
 90 | 	case 12 :
 91 | 		c += tommy_le_uint32_read(key + 8);
 92 | 		b += tommy_le_uint32_read(key + 4);
 93 | 		a += tommy_le_uint32_read(key + 0);
 94 | 		break;
 95 | 	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
 96 | 	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
 97 | 	case 9 : c += key[8];
 98 | 	case 8 :
 99 | 		b += tommy_le_uint32_read(key + 4);
100 | 		a += tommy_le_uint32_read(key + 0);
101 | 		break;
102 | 	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
103 | 	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
104 | 	case 5 : b += key[4];
105 | 	case 4 :
106 | 		a += tommy_le_uint32_read(key + 0);
107 | 		break;
108 | 	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
109 | 	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
110 | 	case 1 : a += key[0];
111 | 	}
112 | 
113 | 	tommy_final(a, b, c);
114 | 
115 | 	return c;
116 | }
117 | 
118 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len)
119 | {
120 | 	const unsigned char* key = tommy_cast(const unsigned char*, void_key);
121 | 	tommy_uint32_t a, b, c;
122 | 
123 | 	a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff);
124 | 	c += init_val >> 32;
125 | 
126 | 	while (key_len > 12) {
127 | 		a += tommy_le_uint32_read(key + 0);
128 | 		b += tommy_le_uint32_read(key + 4);
129 | 		c += tommy_le_uint32_read(key + 8);
130 | 
131 | 		tommy_mix(a, b, c);
132 | 
133 | 		key_len -= 12;
134 | 		key += 12;
135 | 	}
136 | 
137 | 	switch (key_len) {
138 | 	case 0 :
139 | 		return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */
140 | 	case 12 :
141 | 		c += tommy_le_uint32_read(key + 8);
142 | 		b += tommy_le_uint32_read(key + 4);
143 | 		a += tommy_le_uint32_read(key + 0);
144 | 		break;
145 | 	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
146 | 	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
147 | 	case 9 : c += key[8];
148 | 	case 8 :
149 | 		b += tommy_le_uint32_read(key + 4);
150 | 		a += tommy_le_uint32_read(key + 0);
151 | 		break;
152 | 	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
153 | 	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
154 | 	case 5 : b += key[4];
155 | 	case 4 :
156 | 		a += tommy_le_uint32_read(key + 0);
157 | 		break;
158 | 	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
159 | 	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
160 | 	case 1 : a += key[0];
161 | 	}
162 | 
163 | 	tommy_final(a, b, c);
164 | 
165 | 	return c + ((tommy_uint64_t)b << 32);
166 | }
167 | 
168 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/tommyds/tommyhash.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved.
  3 |  *
  4 |  * Redistribution and use in source and binary forms, with or without
  5 |  * modification, are permitted provided that the following conditions
  6 |  * are met:
  7 |  *
  8 |  * 1. Redistributions of source code must retain the above copyright
  9 |  *    notice, this list of conditions and the following disclaimer.
 10 |  *
 11 |  * 2. Redistributions in binary form must reproduce the above copyright
 12 |  *    notice, this list of conditions and the following disclaimer in the
 13 |  *    documentation and/or other materials provided with the distribution.
 14 |  *
 15 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 16 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 17 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 18 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 19 |  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 20 |  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 21 |  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 22 |  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 23 |  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 24 |  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 25 |  * POSSIBILITY OF SUCH DAMAGE.
 26 |  */
 27 | 
 28 | /** \file
 29 |  * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin.
 30 |  */
 31 | 
 32 | #ifndef __TOMMYHASH_H
 33 | #define __TOMMYHASH_H
 34 | 
 35 | #include "tommytypes.h"
 36 | 
 37 | /******************************************************************************/
 38 | /* hash */
 39 | 
 40 | /**
 41 |  * Hash type used in hashtables.
 42 |  */
 43 | typedef tommy_key_t tommy_hash_t;
 44 | 
 45 | /**
 46 |  * Hash function with a 32 bits result.
 47 |  * Implementation of the Robert Jenkins "lookup3" hash 32 bits version,
 48 |  * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle().
 49 |  * \param init_val Initialization value.
 50 |  * Using a different initialization value, you can generate a completely different set of hash values.
 51 |  * Use 0 if not relevant.
 52 |  * \param void_key Pointer at the data to hash.
 53 |  * \param key_len Size of the data to hash.
 54 |  * \note
 55 |  * This function is endianess independent.
 56 |  * \return The hash value of 32 bits.
 57 |  */
 58 | tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len);
 59 | 
 60 | /**
 61 |  * Hash function with a 64 bits result.
 62 |  * Implementation of the Robert Jenkins "lookup3" hash 64 bits versions,
 63 |  * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2().
 64 |  * \param init_val Initialization value.
 65 |  * Using a different initialization value, you can generate a completely different set of hash values.
 66 |  * Use 0 if not relevant.
 67 |  * \param void_key Pointer at the data to hash.
 68 |  * \param key_len Size of the data to hash.
 69 |  * \note
 70 |  * This function is endianess independent.
 71 |  * \return The hash value of 64 bits.
 72 |  */
 73 | tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len);
 74 | 
 75 | /**
 76 |  * Integer hash of 32 bits.
 77 |  * Implementation of the Robert Jenkins "4-byte Integer Hashing",
 78 |  * from http://burtleburtle.net/bob/hash/integer.html
 79 |  */
 80 | tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key)
 81 | {
 82 | 	key -= key << 6;
 83 | 	key ^= key >> 17;
 84 | 	key -= key << 9;
 85 | 	key ^= key << 4;
 86 | 	key -= key << 3;
 87 | 	key ^= key << 10;
 88 | 	key ^= key >> 15;
 89 | 
 90 | 	return key;
 91 | }
 92 | 
 93 | /**
 94 |  * Integer hash of 64 bits.
 95 |  * Implementation of the Thomas Wang "Integer Hash Function",
 96 |  * from http://www.cris.com/~Ttwang/tech/inthash.htm
 97 |  */
 98 | tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key)
 99 | {
100 | 	key = ~key + (key << 21);
101 | 	key = key ^ (key >> 24);
102 | 	key = key + (key << 3) + (key << 8);
103 | 	key = key ^ (key >> 14);
104 | 	key = key + (key << 2) + (key << 4);
105 | 	key = key ^ (key >> 28);
106 | 	key = key + (key << 31);
107 | 
108 | 	return key;
109 | }
110 | 
111 | #endif
112 | 
113 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/tommyds/tommylist.c:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2010, Andrea Mazzoleni. All rights reserved.
 3 |  *
 4 |  * Redistribution and use in source and binary forms, with or without
 5 |  * modification, are permitted provided that the following conditions
 6 |  * are met:
 7 |  *
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  *
11 |  * 2. Redistributions in binary form must reproduce the above copyright
12 |  *    notice, this list of conditions and the following disclaimer in the
13 |  *    documentation and/or other materials provided with the distribution.
14 |  *
15 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 |  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 |  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 |  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 |  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 |  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 |  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 |  * POSSIBILITY OF SUCH DAMAGE.
26 |  */
27 | 
28 | #include "tommylist.h"
29 | #include "tommychain.h"
30 | 
31 | void tommy_list_concat(tommy_list* first, tommy_list* second)
32 | {
33 | 	tommy_node* first_head;
34 | 	tommy_node* first_tail;
35 | 	tommy_node* second_head;
36 | 
37 | 	if (tommy_list_empty(second))
38 | 		return;
39 | 
40 | 	if (tommy_list_empty(first)) {
41 | 		*first = *second;
42 | 		return;
43 | 	}
44 | 
45 | 	first_head = tommy_list_head(first);
46 | 	second_head = tommy_list_head(second);
47 | 	first_tail = tommy_list_tail(first);
48 | 
49 | 	/* set the "circular" prev list */
50 | 	first_head->prev = second_head->prev;
51 | 	second_head->prev = first_tail;
52 | 
53 | 	/* set the "0 terminated" next list */
54 | 	first_tail->next = second_head;
55 | }
56 | 
57 | /** \internal
58 |  * Setup a list.
59 |  */
60 | tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail)
61 | {
62 | 	head->prev = tail;
63 | 	tail->next = 0;
64 | 	*list = head;
65 | }
66 | 
67 | void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp)
68 | {
69 | 	tommy_chain chain;
70 | 	tommy_node* head;
71 | 
72 | 	if (tommy_list_empty(list))
73 | 		return;
74 | 
75 | 	head = tommy_list_head(list);
76 | 
77 | 	/* create a chain from the list */
78 | 	chain.head = head;
79 | 	chain.tail = head->prev;
80 | 
81 | 	tommy_chain_mergesort(&chain, cmp);
82 | 
83 | 	/* restore the list */
84 | 	tommy_list_set(list, chain.head, chain.tail);
85 | }
86 | 
87 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/uuid_database.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #include "uuid_database.h"
22 | 
23 | #include <assert.h>
24 | #include <string.h>
25 | 
26 | static const uint64_t hashtable_seed = 0;
27 | #define hash_str(key) tommy_hash_u32(hashtable_seed, key, strlen(key))
28 | 
29 | /** Asserts that we're using a valid uuid_entry */
30 | static void uuid_entry_assert(const uuid_entry_t *uuid_entry) {
31 |         (void)uuid_entry;
32 | 	assert(UUID_ENTRY_MAGIC == uuid_entry->magic);
33 | }
34 | 
35 | /** Check if an uuid_entry has an uuid
36 |   @param arg uuid in const char * format
37 |   @param obj uuid_entry
38 |   @return 0 if equal, 1 ioc
39 |   */
40 | static int uuid_entry_cmp(const void* arg, const void* obj) {
41 | 	const char *uuid = arg;
42 | 	const uuid_entry_t *uuid_entry = obj;
43 | 	uuid_entry_assert(uuid_entry);
44 | 
45 | 	return strcmp(uuid,uuid_entry->uuid);
46 | }
47 | 
48 | /** Inserts an uuid element in uuid hashtable
49 |   @param h hashtable
50 |   @param n uuid_entry to insert
51 |   */
52 | void uuid_db_insert(uuid_db_t *db, uuid_entry_t *entry) {
53 | 	tommy_hashdyn_insert(db, &entry->node, entry, hash_str(entry->uuid));
54 | }
55 | 
56 | /** Get an entry from uuid database.
57 |   @note Obtained entry need to be freed with uuid_entry_decref
58 |   @param db database
59 |   @param uuid uuid
60 |   @returns uuid entry
61 |   */
62 | uuid_entry_t *uuid_db_search(uuid_db_t *db, const char *uuid) {
63 | 	return tommy_hashdyn_search(db, uuid_entry_cmp, uuid,
64 | 							hash_str(uuid));
65 | }
66 | 


--------------------------------------------------------------------------------
/src/decoder/rb_http2k/uuid_database.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "tommyds/tommyhashdyn.h"
24 | 
25 | #define uuid_db_entry tommy_hashdyn_node
26 | 
27 | /// Sensors database entry
28 | typedef struct uuid_entry_s {
29 | 
30 | #ifndef NDEBUG
31 | 	/* Private data - do not access directly */
32 | 
33 | /// Magic to assert coherency.
34 | #define UUID_ENTRY_MAGIC 0x001D3A1C001D3A1CL
35 | 	/// Magic to assert coherency.
36 | 	uint64_t magic;
37 | #endif
38 | 
39 | 	/// Entry uuid
40 | 	const char *uuid;
41 | 
42 | 	/// hashtable node
43 | 	uuid_db_entry node;
44 | 
45 | 	/// Actual data.
46 | 	void *data;
47 | } uuid_entry_t;
48 | 
49 | /// UUID db
50 | #define uuid_db_t tommy_hashdyn
51 | 
52 | #ifdef UUID_ENTRY_MAGIC
53 | #define uuid_entry_init(e) ((e)->magic = UUID_ENTRY_MAGIC)
54 | #else
55 | #define uuid_entry_init(e)
56 | #endif
57 | 
58 | /// Init an uuid database
59 | #define uuid_db_init tommy_hashdyn_init
60 | #define uuid_db_count tommy_hashdyn_count
61 | #define uuid_db_done tommy_hashdyn_done
62 | #define uuid_db_foreach tommy_hashdyn_foreach
63 | #define uuid_db_foreach_arg tommy_hashdyn_foreach_arg
64 | 
65 | /** Inserts an uuid element in uuid database
66 |   @param db database
67 |   @param entry uuid_entry to insert
68 |   */
69 | void uuid_db_insert(uuid_db_t *db, uuid_entry_t *entry);
70 | 
71 | /** Remove an existing entry from database
72 |   @param db database
73 |   @param entry Entry to remove
74 |   */
75 | #define uuid_db_remove(uuid_db, entry) \
76 | 	tommy_hashdyn_remove_existing(uuid_db, &(entry)->node)
77 | 
78 | 
79 | 
80 | /** Get an entry from uuid database.
81 |   @note Obtained entry need to be freed with uuid_entry_decref
82 |   @param db database
83 |   @param uuid uuid
84 |   @returns uuid entry
85 |   */
86 | uuid_entry_t *uuid_db_search(uuid_db_t *db, const char *uuid);


--------------------------------------------------------------------------------
/src/engine/Makefile.mk:
--------------------------------------------------------------------------------
 1 | THIS_SRCS := \
 2 | 	engine.c \
 3 | 	global_config.c \
 4 | 	n2kafka.c \
 5 | 	rb_addr.c \
 6 | 
 7 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS))
 8 | 
 9 | THIS_SRCS :=
10 | 


--------------------------------------------------------------------------------
/src/engine/engine.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #include "config.h"
20 | #include "util/util.h"
21 | 
22 | #include "engine.h"
23 | #include "parse.h"
24 | #include "util/kafka.h"
25 | #include "global_config.h"
26 | 
27 | #ifdef HAVE_LIBMICROHTTPD
28 | #include "listener/http.h"
29 | #endif
30 | 
31 | int do_shutdown = 0;
32 | 
33 | 


--------------------------------------------------------------------------------
/src/engine/engine.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include <pthread.h>
22 | #include <stdint.h>
23 | 
24 | extern int do_shutdown;
25 | 
26 | void main_loop();
27 | 


--------------------------------------------------------------------------------
/src/engine/global_config.h:
--------------------------------------------------------------------------------
  1 | /*
  2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
  3 | ** Author: Eugenio Perez <eupm90@gmail.com>
  4 | **
  5 | ** This program is free software; you can redistribute it and/or modify
  6 | ** it under the terms of the GNU Affero General Public License as
  7 | ** published by the Free Software Foundation, either version 3 of the
  8 | ** License, or (at your option) any later version.
  9 | **
 10 | ** This program is distributed in the hope that it will be useful,
 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | ** GNU Affero General Public License for more details.
 14 | **
 15 | ** You should have received a copy of the GNU General Public License
 16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17 | */
 18 | 
 19 | #pragma once
 20 | 
 21 | #include "config.h"
 22 | 
 23 | #include "decoder/mse/rb_mse.h"
 24 | #include "decoder/meraki/rb_meraki.h"
 25 | #include "decoder/rb_http2k/rb_http2k_decoder.h"
 26 | #include "util/kafka.h"
 27 | #include "util/in_addr_list.h"
 28 | #include "util/pair.h"
 29 | #include "util/rb_timer.h"
 30 | 
 31 | #include <stdint.h>
 32 | #include <stdbool.h>
 33 | #include <sys/queue.h>
 34 | #include <librdkafka/rdkafka.h>
 35 | 
 36 | struct json_t;
 37 | struct listener;
 38 | typedef void (*decoder_callback)(char *buffer,size_t buf_size,
 39 |     const keyval_list_t *props,void *listener_callback_opaque,
 40 |     void **sessionp);
 41 | typedef struct listener* (*listener_creator)(struct json_t *config,
 42 |                         decoder_callback cb,int cb_flags,void *cb_opaque);
 43 | typedef void (*listener_join)(void *listener_private);
 44 | typedef int (*decoder_listener_opaque_creator)(struct json_t *config,void **opaque);
 45 | typedef int (*decoder_listener_opaque_reload)(struct json_t *config,void *opaque);
 46 | typedef void (*decoder_listener_opaque_destructor)(void *opaque);
 47 | // @TODO we need this callback to split data acquiring || data processing
 48 | // typedef void (*data_process)(void *data_process_private,const char *buffer,size_t bsize);
 49 | typedef void (*listener_reload)(struct json_t *new_config,
 50 |     decoder_listener_opaque_reload opaque_reload,
 51 |     void *cb_opaque,void *listener_private);
 52 | struct listener{
 53 |     uint16_t port; // as listener ID
 54 |     void *private;
 55 | 
 56 |     struct{
 57 |         void *cb_opaque;
 58 |         decoder_callback callback;
 59 |         decoder_listener_opaque_destructor cb_opaque_destructor;
 60 |         decoder_listener_opaque_reload cb_opaque_reload;
 61 |         int flags;
 62 |     }cb;
 63 |     listener_creator create;
 64 |     listener_join join;
 65 |     listener_reload reload;
 66 |     LIST_ENTRY(listener) entry;
 67 | };
 68 | 
 69 | enum {
 70 | /**
 71 |     Tells if the listener support straming API, instead of wait to all data
 72 |     to be available
 73 | */
 74 |     DECODER_F_SUPPORT_STREAMING = 0x01,
 75 | };
 76 | 
 77 | typedef LIST_HEAD(,listener) listener_list;
 78 | 
 79 | struct n2kafka_config{
 80 | #ifdef HAVE_LIBMICROHTTPD
 81 | #define N2KAFKA_HTTP 3
 82 | #endif
 83 | 
 84 |     /// Path to reload
 85 |     char *config_path;
 86 | 
 87 |     char *format;
 88 | 
 89 |     char *topic;
 90 |     char *brokers;
 91 |     char *n2kafka_id;
 92 | 
 93 |     rd_kafka_conf_t *kafka_conf;
 94 |     rd_kafka_topic_conf_t *kafka_topic_conf;rd_kafka_t *rk;
 95 | 
 96 |     in_addr_list_t *blacklist;
 97 | 
 98 |     /// List of global timers
 99 |     rb_timers_list_t timers;
100 | 
101 |     /// @TODO this should belong to decoders, not here.
102 |     struct mse_config mse;
103 |     struct meraki_config meraki;
104 |     struct rb_config rb;
105 | 
106 |     char *response;
107 |     int response_len;
108 | 
109 |     listener_list listeners;
110 | 
111 |     struct json_t *stream_enrichment;
112 | 
113 |     bool debug;
114 | };
115 | 
116 | extern struct n2kafka_config global_config;
117 | 
118 | static inline bool only_stdout_output(){
119 | 	return global_config.debug && !global_config.brokers;
120 | }
121 | 
122 | void init_global_config();
123 | 
124 | void parse_config(const char *config_file_path);
125 | 
126 | void reload_config(struct n2kafka_config *config);
127 | 
128 | /** Register a new timer to call from this decoder
129 |   @param interval Interval to call callback
130 |   @param cb Callback
131 |   @param ctx Context to call callback
132 |   @note Callback will be called from another thread than decoder.
133 |   */
134 | rb_timer_t *decoder_register_timer(const struct itimerspec *interval,
135 |     void (*cb)(void *), void *ctx);
136 | 
137 | /** Unregister a registered timer
138 |   @param timer timer to unregister
139 |   */
140 | void decoder_deregister_timer(rb_timer_t *timer);
141 | 
142 | /** Change a timer interval
143 |   @param timer timer to change interval
144 |   @param ts New timer interval
145 |   @return 0 if success. !0 in other case.
146 |   */
147 | int decoder_timer_set_interval0(struct rb_timer *timer, int flags,
148 | 						const struct itimerspec *ts);
149 | 
150 | #define decoder_timer_set_interval(timer, timerspec) \
151 | 	decoder_timer_set_interval0(timer, 0, timerspec)
152 | 
153 | /// @TODO use SIGEV_THREAD and delete this function!!
154 | void execute_global_timers();
155 | 
156 | void free_global_config();
157 | 


--------------------------------------------------------------------------------
/src/engine/n2kafka.c:
--------------------------------------------------------------------------------
  1 | /*
  2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
  3 | ** Author: Eugenio Perez <eupm90@gmail.com>
  4 | **
  5 | ** This program is free software; you can redistribute it and/or modify
  6 | ** it under the terms of the GNU Affero General Public License as
  7 | ** published by the Free Software Foundation, either version 3 of the
  8 | ** License, or (at your option) any later version.
  9 | **
 10 | ** This program is distributed in the hope that it will be useful,
 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | ** GNU Affero General Public License for more details.
 14 | **
 15 | ** You should have received a copy of the GNU General Public License
 16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17 | */
 18 | 
 19 | #include "version.h"
 20 | #include "engine.h"
 21 | #include "util/kafka.h"
 22 | #include "global_config.h"
 23 | 
 24 | #include <signal.h>
 25 | #include <stdio.h>
 26 | #include <string.h>
 27 | #include <unistd.h>
 28 | #include <jansson.h>
 29 | 
 30 | #define DEFAULT_PORT 2057
 31 | 
 32 | static int do_reload = 0;
 33 | 
 34 | static void shutdown_process(){
 35 | 	printf("Exiting\n");
 36 | 	do_shutdown=1;
 37 | }
 38 | 
 39 | static void show_usage(const char *progname){
 40 | 	fprintf(stdout,"n2kafka version %s-%s\n",n2kafka_version,n2kafka_revision);
 41 | 	fprintf(stdout,"Usage: %s <config_file>\n",progname);
 42 | 	fprintf(stdout,"\n");
 43 | 	fprintf(stdout,
 44 | 	        "Where <config_file> is a json file that can contains the \n");
 45 | 	fprintf(stdout,"the next configurations:\n");
 46 | 	
 47 | 	fprintf(stdout,"{\n");
 48 | 	fprintf(stdout,"\t\"listeners:\":[\n");
 49 | 	fprintf(stdout,
 50 | 	        "\t\t{\"proto\":\"http\",\"port\":2057,\"mode\":\"(1)\","
 51 | 	        "\"threads\":20}\n");
 52 | 	fprintf(stdout,
 53 | 	        "\t\t{\"proto\":\"tcp\",\"port\":2056,"
 54 | 	        "\"tcp_leepalive\":true,\"mode\"},\n");
 55 | 	fprintf(stdout,"\t\t{\"proto\":\"udp\",\"port\":2058,\"threads\":20}\n");
 56 | 	fprintf(stdout,"\t],\n");
 57 | 	fprintf(stdout,"\t\"brokers\":\"kafka brokers\",\n");
 58 | 	fprintf(stdout,"\t\"topic\":\"kafka topic\",\n");
 59 | 	fprintf(stdout,"\t\"rdkafka.socket.max.fails\":\"3\",\n");
 60 | 	fprintf(stdout,"\t\"rdkafka.socket.keepalive.enable\":\"true\",\n");
 61 | 	fprintf(stdout,"\t\"blacklist\":[\"192.168.101.3\"]\n");
 62 | 	fprintf(stdout,"}\n\n");
 63 | 	fprintf(stdout,"(1) Modes can be:\n");
 64 | 	fprintf(stdout,
 65 | 	        "\tthread_per_connection: Creates a thread for each connection.\n");
 66 | 	fprintf(stdout,"\t\tThread argument will be ignored in this mode\n");
 67 | 	fprintf(stdout,
 68 | 	        "\tselect,poll,epoll: Fixed number of threads (with threads "
 69 | 	        "parameter) manages all connections\n");
 70 | }
 71 | 
 72 | static int is_asking_help(const char *param){
 73 | 	return 0==strcmp(param,"-h") || 0==strcmp(param,"--help");
 74 | }
 75 | 
 76 | static void sighup_proc(int signum __attribute__((unused))){
 77 | 	do_reload = 1;
 78 | }
 79 | 
 80 | int main(int argc,char *argv[]){
 81 | 	if(argc != 2 || is_asking_help(argv[1])){
 82 | 		show_usage(argv[0]);
 83 | 		exit(1);
 84 | 	}
 85 | 
 86 | 	init_global_config();
 87 | 	parse_config(argv[1]);
 88 | 
 89 | 	signal(SIGINT,shutdown_process);
 90 | 	signal(SIGHUP,sighup_proc);
 91 | 
 92 | 	while(!do_shutdown){
 93 | 		kafka_poll(1000 /* ms */);
 94 | 		if(do_reload){
 95 | 			reload_config(&global_config);
 96 | 			do_reload = 0;
 97 | 		}
 98 | 		// @TODO this should be not neccesary with SIGEV_THREAD
 99 | 		execute_global_timers();
100 | 	}
101 | 
102 | 	free_global_config();
103 | 
104 | 	return 0;
105 | }
106 | 


--------------------------------------------------------------------------------
/src/engine/parse.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | 


--------------------------------------------------------------------------------
/src/engine/rb_addr.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #include "rb_addr.h"
20 | 
21 | #include <stdio.h>
22 | #include <errno.h>
23 | #include <librd/rdlog.h>
24 | 
25 | const char *sockaddr2str(char *buf, size_t buf_size, struct sockaddr *sockaddr) {
26 | 	char errbuf[BUFSIZ];
27 | 
28 | 	const void *addr_buf = NULL;
29 | 	const char *ret = NULL;
30 | 
31 | 	switch(sockaddr->sa_family) {
32 | 	case AF_INET:
33 | 		addr_buf = &((struct sockaddr_in *)sockaddr)->sin_addr;
34 | 		break;
35 | 	case AF_INET6:
36 | 		addr_buf = &((struct sockaddr_in6 *)sockaddr)->sin6_addr;
37 | 		break;
38 | 	default:
39 | 		break;
40 | 	}
41 | 
42 | 	if(NULL == addr_buf) {
43 | 		errno = EAFNOSUPPORT;
44 | 	} else {
45 | 		ret = inet_ntop(sockaddr->sa_family, addr_buf,buf,buf_size);
46 | 	}
47 | 
48 | 	if(NULL == ret) {
49 | 		strerror_r(errno,errbuf,sizeof(errbuf));
50 | 		rdlog(LOG_ERR,"Can't print client address: %s",errbuf);
51 | 	}
52 | 
53 | 	return ret;
54 | }
55 | 


--------------------------------------------------------------------------------
/src/engine/rb_addr.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include <string.h>
22 | #include <netinet/in.h>
23 | #include <arpa/inet.h>
24 | 
25 | const char *sockaddr2str(char *buf, size_t buf_size, struct sockaddr *sockaddr);
26 | 


--------------------------------------------------------------------------------
/src/listener/Makefile.mk:
--------------------------------------------------------------------------------
1 | THIS_SRCS := \
2 | 	http.c \
3 | 	socket.c \
4 | 
5 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS))
6 | 
7 | THIS_SRCS :=


--------------------------------------------------------------------------------
/src/listener/http.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #ifdef HAVE_LIBMICROHTTPD
22 | #include "engine/global_config.h"
23 | struct http_handler;
24 | struct json_t;
25 | struct listener *create_http_listener(struct json_t *config,
26 |             decoder_callback cb,int cb_flags,void *cb_opaque);
27 | #endif
28 | 


--------------------------------------------------------------------------------
/src/listener/socket.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include "engine/global_config.h"
22 | struct http_handler;
23 | struct json_t;
24 | struct listener *create_socket_listener(struct json_t *config,
25 | 	decoder_callback callback,int callback_flags,void *callback_opaque);
26 | 
27 | #define create_tcp_listener create_socket_listener
28 | #define create_udp_listener create_socket_listener
29 | 


--------------------------------------------------------------------------------
/src/util/Makefile.mk:
--------------------------------------------------------------------------------
 1 | THIS_SRCS := \
 2 | 	in_addr_list.c \
 3 | 	kafka.c \
 4 | 	kafka_message_list.c \
 5 | 	pair.c \
 6 | 	rb_json.c \
 7 | 	rb_mac.c \
 8 | 	topic_database.c \
 9 | 	rb_timer.c \
10 | 
11 | SRCS += $(addprefix $(CURRENT_N2KAFKA_DIR), $(THIS_SRCS))
12 | 
13 | THIS_SRCS :=
14 | 


--------------------------------------------------------------------------------
/src/util/in_addr_list.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #include "in_addr_list.h"
20 | 
21 | #include <sys/queue.h>
22 | #include <netinet/in.h>
23 | #include <sys/socket.h>
24 | #include <stdlib.h>
25 | #include <string.h>
26 | 
27 | 
28 | typedef struct in_addr_list_node_s{
29 | 	struct in_addr addr;
30 | 	LIST_ENTRY(in_addr_list_node_s) entry;
31 | }in_addr_list_node_t;
32 | 
33 | typedef LIST_HEAD(,in_addr_list_node_s) in_addr_list_head_t;
34 | 
35 | struct in_addr_list_s{
36 | 	in_addr_list_head_t head;
37 | };
38 | 
39 | /// Return the syslist from sockadd_in_list_t
40 | #define syslist(list) (&(list)->head)
41 | 
42 | /// Init a sockaddr_in list.
43 | in_addr_list_t *in_addr_list_new(){
44 | 	in_addr_list_t *list = calloc(1,sizeof(*list));
45 | 	LIST_INIT(syslist(list));
46 | 	return list;
47 | }
48 | 
49 | /// Add an address to list.
50 | void in_addr_list_add(in_addr_list_t *list,const struct in_addr *addr){
51 | 	in_addr_list_node_t *node = calloc(1,sizeof(*node));
52 | 
53 | 	memcpy(&node->addr,addr,sizeof(*addr));
54 | 
55 | 	LIST_INSERT_HEAD(syslist(list),node,entry);
56 | }
57 | 
58 | /// Check if an addr is in list.
59 | int in_addr_list_contains(const in_addr_list_t *list,const struct in_addr *addr){
60 | 	in_addr_list_node_t *n=NULL;
61 | 	for(n=LIST_FIRST(syslist(list));n!=NULL;n=LIST_NEXT(n,entry)){
62 | 		if(0==memcmp(addr,&n->addr,sizeof(n->addr)))
63 | 			return 1;
64 | 	}
65 | 	return 0;
66 | }
67 | 
68 | /// Deallocate a list.
69 | void in_addr_list_done(in_addr_list_t *list){
70 | 	in_addr_list_node_t *n=NULL;
71 | 	while((n = LIST_FIRST(syslist(list)))){
72 | 		LIST_REMOVE(n,entry);
73 | 		free(n);
74 | 	}
75 | 	free(list);
76 | }
77 | 


--------------------------------------------------------------------------------
/src/util/in_addr_list.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include <sys/queue.h>
22 | #include <netinet/in.h>
23 | 
24 | typedef struct in_addr_list_s in_addr_list_t; /* FW DECLARATION */
25 | 
26 | /// Init a sockaddr_in list.
27 | in_addr_list_t *in_addr_list_new();
28 | 
29 | /// Add an address to list.
30 | void in_addr_list_add(in_addr_list_t *list,const struct in_addr *addr);
31 | 
32 | /// Check if an addr is in list.
33 | int in_addr_list_contains(const in_addr_list_t *list,const struct in_addr *addr);
34 | 
35 | /// Deallocate a list.
36 | void in_addr_list_done(in_addr_list_t *list);
37 | 


--------------------------------------------------------------------------------
/src/util/kafka.h:
--------------------------------------------------------------------------------
  1 | /*
  2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
  3 | ** Author: Eugenio Perez <eupm90@gmail.com>
  4 | **
  5 | ** This program is free software; you can redistribute it and/or modify
  6 | ** it under the terms of the GNU Affero General Public License as
  7 | ** published by the Free Software Foundation, either version 3 of the
  8 | ** License, or (at your option) any later version.
  9 | **
 10 | ** This program is distributed in the hope that it will be useful,
 11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | ** GNU Affero General Public License for more details.
 14 | **
 15 | ** You should have received a copy of the GNU General Public License
 16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17 | */
 18 | 
 19 | #pragma once
 20 | #include "engine/parse.h"
 21 | #include "util/pair.h"
 22 | #include <librdkafka/rdkafka.h>
 23 | 
 24 | #include <string.h>
 25 | 
 26 | /* Private data */
 27 | struct rd_kafka_message_s;
 28 | 
 29 | struct kafka_message_array{
 30 | 	size_t count; /* Number of used elements in msgs */
 31 | 	size_t size;  /* Number of elements in msgs */
 32 | 	struct rd_kafka_message_s *msgs; /* Real msgs */
 33 | };
 34 | 
 35 | /// Array of topics
 36 | struct rkt_array {
 37 | 	/// Topics
 38 | 	rd_kafka_topic_t **rkt;
 39 | 	/// Number of topics
 40 | 	size_t count;
 41 | };
 42 | 
 43 | /** Swap two topics array */
 44 | void rkt_array_swap(struct rkt_array *a, struct rkt_array *b);
 45 | /** deallocate topics array resources */
 46 | void rkt_array_done(struct rkt_array *rkt_array);
 47 | 
 48 | void init_rdkafka();
 49 | void send_to_kafka(rd_kafka_topic_t *rkt,char *buffer,const size_t bufsize,
 50 | 	int flags,void *opaque);
 51 | void dumb_decoder(char *buffer,size_t buf_size,const keyval_list_t *keyval,
 52 |     void *listener_callback_opaque,void **sessionp);
 53 | 
 54 | /// @TODO join with rb_http2k_decoder mac partitioner
 55 | int32_t rb_client_mac_partitioner (const rd_kafka_topic_t *_rkt,
 56 | 					const void *key,size_t keylen,int32_t partition_cnt,
 57 | 					void *rkt_opaque,void *msg_opaque);
 58 | 
 59 | struct kafka_message_array *new_kafka_message_array(size_t size);
 60 | int save_kafka_msg_key_partition_in_array(struct kafka_message_array *array,
 61 | 				char *key, size_t key_size,
 62 | 				char *buffer, size_t buf_size,int partition,
 63 | 				void *opaque);
 64 | #define save_kafka_msg_key_in_array(array, key, key_size, buffer, buf_size, \
 65 | 								opaque) \
 66 | 	save_kafka_msg_key_partition_in_array(array, key, key_size, buffer, \
 67 | 				buf_size, RD_KAFKA_PARTITION_UA, opaque)
 68 | #define save_kafka_msg_in_array(array, buffer, buf_size, opaque) \
 69 | 	save_kafka_msg_key_in_array(array, NULL, 0, buffer, buf_size, opaque);
 70 | 
 71 | /** Send an array of messages to a given topic. Messages that couldn't be
 72 |   sent will be freed, and error message will be shown
 73 |   @param rkt Topic to send messages
 74 |   @param msgs Messages to be sent
 75 |   @return Number of messages sent
 76 |   */
 77 | int send_array_to_kafka(rd_kafka_topic_t *rkt,
 78 | 					struct kafka_message_array *msgs);
 79 | 
 80 | /** Send an array of messages to many kafka topics
 81 |   @rkt_array Topics to send messages
 82 |   @msgs Messages to send
 83 |   @return Messages sent. It should be rkt_size*msgs->count.
 84 |   @note msgs[i]->opaque will be lost if you use it!
 85 |   @note You CAN'T use msgs[i].msg anymore, regardless of what this function
 86 |   return
 87 |   */
 88 | int send_array_to_kafka_topics(struct rkt_array *rkt_array,
 89 | 	struct kafka_message_array *msgs);
 90 | 
 91 | void kafka_poll();
 92 | 
 93 | typedef int32_t (*rb_rd_kafka_partitioner_t) (
 94 | 						const rd_kafka_topic_t *rkt,
 95 | 						const void *keydata,
 96 | 						size_t keylen,
 97 | 						int32_t partition_cnt,
 98 | 						void *rkt_opaque,
 99 | 						void *msg_opaque);
100 | 
101 | /** Creates a new topic handler using global configuration
102 |     @param topic_name Topic name
103 |     @param partitioner Partitioner function
104 |     @return New topic handler */
105 | rd_kafka_topic_t *new_rkt_global_config(const char *topic_name,
106 |     rb_rd_kafka_partitioner_t partitioner,char *err,size_t errsiz);
107 | 
108 | /** Default kafka topic name (if any)
109 | 	@return Default kafka topic name (if any)
110 | 	*/
111 | const char *default_topic_name();
112 | 
113 | /** Get metadata topic
114 |   @param rkt Topic to obtain information
115 |   @param metadata pointer to hold metadata result. The \p *metadatap pointer
116 | 	must be released with rd_kafka_metadata_destroy().
117 |   @param maximum response time before failing. If we get this timeout, this
118 | 	function will retry forever.
119 |   @return RD_KAFKA_RESP_ERR_X code
120 | 	*/
121 | int kafka_get_topic_metadata(rd_kafka_topic_t *rkt,
122 | 		const struct rd_kafka_metadata **metadata, int timeout_ms);
123 | 
124 | void flush_kafka();
125 | void stop_rdkafka();
126 | 


--------------------------------------------------------------------------------
/src/util/kafka_message_list.c:
--------------------------------------------------------------------------------
 1 | 
 2 | #include "kafka_message_list.h"
 3 | 
 4 | #include <stdlib.h>
 5 | #include <string.h>
 6 | 
 7 | struct rd_kafka_message_queue_elm_s {
 8 | 	/** Kafka message */
 9 | 	rd_kafka_message_t msg;
10 | 	/** List entry */
11 | 	TAILQ_ENTRY(rd_kafka_message_queue_elm_s) list_entry;
12 | };
13 | 
14 | void rd_kafka_msg_q_init(rd_kafka_message_queue_t *q) {
15 | 	q->count = 0;
16 | 	TAILQ_INIT(&q->list);
17 | }
18 | 
19 | int rd_kafka_msg_q_add(rd_kafka_message_queue_t *q,
20 |                     const rd_kafka_message_t *msg) {
21 | 	rd_kafka_message_queue_elm_t *elm = malloc(sizeof(*elm));
22 | 	if(elm) {
23 | 		++q->count;
24 | 		memcpy(&elm->msg,msg,sizeof(msg[0]));
25 | 		TAILQ_INSERT_TAIL(&q->list,elm,list_entry);
26 | 	}
27 | 
28 | 	return elm != NULL;
29 | }
30 | 
31 | static void rd_kafka_msg_q_dump0(rd_kafka_message_queue_t *q,
32 | 	rd_kafka_message_t *msgs) {
33 | 
34 | 	rd_kafka_message_queue_elm_t *elm = NULL;
35 | 	size_t i = 0;
36 | 
37 | 	while((elm = TAILQ_FIRST(&q->list))) {
38 | 		TAILQ_REMOVE(&q->list,elm,list_entry);
39 | 		if(msgs) {
40 | 			memcpy(&msgs[i++],&elm->msg,sizeof(msgs[0]));
41 | 		}
42 | 		free(elm);
43 | 	}
44 | 
45 | 	q->count = 0;
46 | }
47 | 
48 | void rd_kafka_msg_q_dump(rd_kafka_message_queue_t *q,
49 | 	rd_kafka_message_t *msgs) {
50 | 
51 | 	rd_kafka_msg_q_dump0(q,msgs);
52 | }
53 | 
54 | void rd_kafka_msg_q_clean(rd_kafka_message_queue_t *q) {
55 | 	rd_kafka_msg_q_dump0(q,NULL);
56 | }


--------------------------------------------------------------------------------
/src/util/kafka_message_list.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <librdkafka/rdkafka.h>
 4 | #include <sys/queue.h>
 5 | 
 6 | struct rd_kafka_message_queue_elm_s;
 7 | 
 8 | /** Queue element with one rdkafka message */
 9 | typedef struct rd_kafka_message_queue_elm_s rd_kafka_message_queue_elm_t;
10 | 
11 | /** Kafka message queue */
12 | typedef struct rd_kafka_message_queue_s {
13 | 	/** Number of elements */
14 | 	size_t count;
15 | 	/** Actual list */
16 | 	TAILQ_HEAD(,rd_kafka_message_queue_elm_s) list;
17 | } rd_kafka_message_queue_t;
18 | 
19 | #define rd_kafka_msg_q_size(q) ((q)->count)
20 | 
21 | /** Init a message queue
22 | 	@param q Queue */
23 | void rd_kafka_msg_q_init(rd_kafka_message_queue_t *q);
24 | 
25 | /** Add a message to the message queue
26 | 	@param q Queue
27 | 	@param msg Message
28 | 	@note The message will be copied
29 | 	@return 1 if ok, 0 if no memory avalable */
30 | int rd_kafka_msg_q_add(rd_kafka_message_queue_t *q,
31 | 	const rd_kafka_message_t *msg);
32 | 
33 | /** Dump messages to an allocated messages array. It is
34 | 	suppose to be able to hold as many messages as
35 | 	rd_kafka_msg_q_size(q)
36 | 	@param q Queue
37 | 	@param msgs Messages to dump to
38 | 	@note after this call, queue will be empty */
39 | void rd_kafka_msg_q_dump(rd_kafka_message_queue_t *q,
40 | 	rd_kafka_message_t *msgs);
41 | 
42 | /** Discards all messages in queue
43 | 	@param q Queue
44 | 	*/
45 | void rd_kafka_msg_q_clean(rd_kafka_message_queue_t *q);


--------------------------------------------------------------------------------
/src/util/pair.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #include "pair.h"
20 | 
21 | #include <string.h>
22 | 
23 | void add_key_value_pair(keyval_list_t *list,struct pair *pair) {
24 | 	TAILQ_INSERT_TAIL(list,pair,entry);
25 | }
26 | 
27 | const char *valueof(const keyval_list_t *list,const char *key) {
28 | 	struct pair *pair = NULL;
29 | 	TAILQ_FOREACH(pair, list, entry) {
30 | 		if(0 == strcmp(key,pair->key)) {
31 | 			return pair->value;
32 | 		}
33 | 	}
34 | 
35 | 	return NULL;
36 | }


--------------------------------------------------------------------------------
/src/util/pair.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include <sys/queue.h>
22 | 
23 | struct pair {
24 | 	const char *key;
25 | 	const char *value;
26 | 	TAILQ_ENTRY(pair) entry;
27 | };
28 | 
29 | typedef TAILQ_HEAD(,pair) keyval_list_t;
30 | #define keyval_list_initializer TAILQ_HEAD_INITIALIZER
31 | #define keyval_list_init TAILQ_INIT
32 | 
33 | void add_key_value_pair(keyval_list_t *list,struct pair *pair);
34 | const char *valueof(const keyval_list_t *list,const char *key);
35 | 


--------------------------------------------------------------------------------
/src/util/rb_json.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #include "rb_json.h"
22 | 
23 | #include <string.h>
24 | 
25 | int json_object_update_missing_copy(json_t *dst, const json_t *src) {
26 | 	const char *key=NULL;
27 | 	const json_t *value = NULL;
28 | 	/* We promise that we will not modify this */
29 | 	json_t *my_src = NULL;
30 | 
31 | 	if(!json_is_object(src) || !json_is_object(dst))
32 | 		return -1;
33 | 
34 | 	memcpy(&my_src,&src,sizeof(my_src));
35 | 
36 | 	json_object_foreach(my_src,key,value) {
37 | 		if(NULL == json_object_get(dst,key)){
38 | 			json_t *new_json = json_deep_copy(value);
39 | 			json_object_set_new_nocheck(dst,key,new_json);
40 | 		}
41 | 	}
42 | 
43 | 	return 0;
44 | }


--------------------------------------------------------------------------------
/src/util/rb_json.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include <jansson.h>
24 | 
25 | int json_object_update_missing_copy(json_t *dst, const json_t *src);
26 | 


--------------------------------------------------------------------------------
/src/util/rb_mac.c:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #include "rb_mac.h"
22 | #include <string.h>
23 | 
24 | static int ishexchar(const char x) {
25 | 	return ('0' <= x && x <= '9') || ('a' <= x && x <= 'f') || ('A' <= x && x <= 'F');
26 | }
27 | 
28 | static uint64_t hexchar(const unsigned x) {
29 | 	return ('0' <= x && x <= '9') ? x - '0' : 
30 | 	       ('a' <= x && x <= 'f') ? x - 'a' + 10 :
31 | 	       x - 'A' +10;
32 | }
33 | 
34 | static int validmac(const char *a) {
35 | 	return 
36 | 		ishexchar(a[ 0]) && ishexchar(a[ 1]) &&
37 | 		ishexchar(a[ 3]) && ishexchar(a[ 4]) &&
38 | 		ishexchar(a[ 6]) && ishexchar(a[ 7]) &&
39 | 		ishexchar(a[ 9]) && ishexchar(a[10]) &&
40 | 		ishexchar(a[12]) && ishexchar(a[13]) &&
41 | 		ishexchar(a[15]) && ishexchar(a[16]) &&
42 | 		a[2]  == ':' && a[5] == ':' && a[8] == ':' &&
43 | 		a[11] == ':' && a[14] == ':';
44 | }
45 | 
46 | uint64_t parse_mac(const char *_mac) {
47 | 	if(strlen(_mac) != strlen("00:00:00:00:00:00"))
48 | 		return 0xFFFFFFFFFFFFFFFFL;
49 | 
50 | 	if(!validmac(_mac))
51 | 		return 0xFFFFFFFFFFFFFFFFL;
52 | 
53 | 	const unsigned char *mac = (const unsigned char *)_mac;
54 | 
55 | 	return 0L + 
56 | 		(hexchar(mac[16])<<0)+
57 | 		(hexchar(mac[15])<<4)+
58 | 		(hexchar(mac[13])<<8)+
59 | 		(hexchar(mac[12])<<12)+
60 | 		(hexchar(mac[10])<<16)+
61 | 		(hexchar(mac[ 9])<<20)+
62 | 		(hexchar(mac[ 7])<<24)+
63 | 		(hexchar(mac[ 6])<<28)+
64 | 		(hexchar(mac[ 4])<<32)+
65 | 		(hexchar(mac[ 3])<<36)+
66 | 		(hexchar(mac[ 1])<<40)+
67 | 		(hexchar(mac[ 0])<<44);
68 | }
69 | 


--------------------------------------------------------------------------------
/src/util/rb_mac.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | **
 3 | ** Copyright (c) 2014, Eneo Tecnologia
 4 | ** Author: Eugenio Perez <eupm90@gmail.com>
 5 | ** All rights reserved.
 6 | **
 7 | ** This program is free software; you can redistribute it and/or modify
 8 | ** it under the terms of the GNU Affero General Public License as
 9 | ** published by the Free Software Foundation, either version 3 of the
10 | ** License, or (at your option) any later version.
11 | **
12 | ** This program is distributed in the hope that it will be useful,
13 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 | ** GNU Affero General Public License for more details.
16 | **
17 | ** You should have received a copy of the GNU General Public License
18 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | #include <stdint.h>
23 | 
24 | /// MAX mac can return parse_mac
25 | #define MAX_MAC 0xffffffffffffL
26 | 
27 | #define valid_mac(mac) (mac <= MAX_MAC)
28 | #define INVALID_MAC (MAX_MAC+1)
29 | 
30 | // error if return 0xFFFFFFFFFFFFFFFFL
31 | uint64_t parse_mac(const char *mac);
32 | 


--------------------------------------------------------------------------------
/src/util/rb_time.h:
--------------------------------------------------------------------------------
 1 |  /*
 2 | ** Copyright (C) 2016 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #include <time.h>
20 | 
21 | /** Checks a timespec struct
22 |   @param @tv Timespec
23 |   @returns 0 if timespec is different than zero. !0 ioc
24 |   */
25 | static int zero_timespec(const struct timespec *ts) __attribute__((unused));
26 | static int zero_timespec(const struct timespec *ts) {
27 | 	return 0 == ts->tv_sec && 0 == ts->tv_nsec;
28 | }
29 | 
30 | /** Compare two timespecs
31 |   @param ts1 First timespec
32 |   @param ts2 second timespec
33 |   @return 0 if equal, >0 if second timespec if bigger than first, and
34 |   <0 in other case
35 |   */
36 | static int cmp_timespec(const struct timespec *ts1,
37 | 			const struct timespec *ts2) __attribute__((unused));
38 | static int cmp_timespec(const struct timespec *ts1,
39 | 			const struct timespec *ts2) {
40 | 	if(ts1->tv_sec != ts2->tv_sec) {
41 | 		return ts2->tv_sec - ts1->tv_sec;
42 | 	} else {
43 | 		return ts2->tv_nsec - ts1->tv_nsec;
44 | 	}
45 | }
46 | 
47 | static int zero_itimerspec(const struct itimerspec *its)
48 | 						__attribute__((unused));
49 | static int zero_itimerspec(const struct itimerspec *its) {
50 | 	return zero_timespec(&its->it_value)
51 | 					&& zero_timespec(&its->it_interval);
52 | }


--------------------------------------------------------------------------------
/src/util/rb_timer.h:
--------------------------------------------------------------------------------
 1 |  /*
 2 | ** Copyright (C) 2016 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include <signal.h>
22 | #include <time.h>
23 | #include <sys/queue.h>
24 | 
25 | /// Internal - single timer
26 | struct rb_timer;
27 | typedef struct rb_timer rb_timer_t;
28 | 
29 | /// List of timers
30 | typedef struct rb_timers_list_s {
31 | 	/* Internal */
32 | 	pthread_mutex_t mutex;
33 | 
34 | 	/// List of registered timers
35 | 	LIST_HEAD(, rb_timer) list;
36 | } rb_timers_list_t;
37 | 
38 | /** Sets a new interval to a timer
39 |   @param timer timer.
40 |   @param tv new interval
41 |   @param err Error string (if any)
42 |   @param errsize Size of err
43 |   @return 0 is success, !0 and errno if error
44 |   */
45 | int rb_timer_set_interval0(rb_timer_t *timer, int flags,
46 | 		const struct itimerspec *ts, char *err, size_t errsize);
47 | 
48 | /** Convenience macro */
49 | #define rb_timer_set_interval(timer, timerspec, err, errsize) \
50 | 	rb_timer_set_interval0(timer, 0, timerspec, err, errsize)
51 | 
52 | 
53 | /** Gets timer interval
54 |   @param timer timer to get the interval
55 |   @param ts Timespec to save the interval
56 |   @return 0 if success. !0 and errno if error
57 |   */
58 | int rb_timer_get_interval(const rb_timer_t *timer, struct itimerspec *ts);
59 | 
60 | /** Start a timers list
61 |   @param list of timers
62 |   @return 0 in success, 1 ioc.
63 |   */
64 | void rb_timers_list_init(rb_timers_list_t *list);
65 | 
66 | /** Destroy a list of timers and all timers that it contains
67 |   @param list list of timers
68 |   */
69 | void rb_timers_list_done(rb_timers_list_t *list);
70 | 
71 | /** Destroy a timer
72 |   @param list List of timers
73 |   @param timer Timer to destroy
74 |   */
75 | void rb_timer_done(rb_timers_list_t *list, struct rb_timer *timer);
76 | 
77 | /**
78 |   Runs all pending timers
79 |   @TODO this function is pointless if we can use SIGEV_THREAD.
80 |   */
81 | void rb_timers_run(rb_timers_list_t *list);
82 | 
83 | /** Create a timer and append it to a list of timers
84 |   @param tlist timer list
85 |   @param interval Time interval you want the timer to signal you.
86 |   @param cb Callback to call
87 |   @param cb_ctx Context to send to callback
88 |   @param err Error string
89 |   @param errsize Error string size
90 |   @return new timer if successful
91 |   */
92 | rb_timer_t *rb_timer_create(rb_timers_list_t *tlist,
93 | 	const struct itimerspec *interval, void (*cb)(void *), void *cb_ctx,
94 | 	char *err, size_t errsize);
95 | 
96 | /// Need to call this function if a registered timer calls SIGALRM
97 | void rb_timer_sigaction(int signum, siginfo_t *siginfo, void *ucontext);
98 | 


--------------------------------------------------------------------------------
/src/util/topic_database.c:
--------------------------------------------------------------------------------
  1 | 
  2 | #include "config.h"
  3 | #include "topic_database.h"
  4 | 
  5 | #include <librd/rdavl.h>
  6 | #include <librd/rdmem.h>
  7 | 
  8 | typedef TAILQ_HEAD(,topic_s) topics_list;
  9 | 
 10 | #ifndef NDEBUG
 11 | #define TOPIC_S_MAGIC 0x01CA1C01CA1C01CL
 12 | #endif
 13 | 
 14 | #define topic_list_init(l) TAILQ_INIT(l)
 15 | #define topic_list_push(l,e) TAILQ_INSERT_TAIL(l,e,list_node);
 16 | 
 17 | struct topic_s {
 18 | #ifdef TOPIC_S_MAGIC
 19 |        uint64_t magic;
 20 | #endif
 21 |        rd_kafka_topic_t *rkt;
 22 |        const char *topic_name;
 23 |        uint64_t refcnt;
 24 | 
 25 |        rd_avl_node_t avl_node;
 26 |        TAILQ_ENTRY(topic_s) list_node;
 27 | 
 28 |        const char *partition_key;
 29 | };
 30 | 
 31 | rd_kafka_topic_t *topics_db_get_rdkafka_topic(struct topic_s *topic) {
 32 | 	return topic->rkt;
 33 | }
 34 | 
 35 | void topic_decref(struct topic_s *topic) {
 36 | 	if(0==ATOMIC_OP(sub,fetch,&topic->refcnt,1)) {
 37 | 		rd_kafka_topic_destroy(topic->rkt);
 38 | 		free(topic);
 39 | 	}
 40 | }
 41 | 
 42 | static int topics_cmp(const struct topic_s *t1, const struct topic_s *t2) {
 43 | 	assert(t1);
 44 | 	assert(t1->topic_name);
 45 | 	assert(t2);
 46 | 	assert(t2->topic_name);
 47 | 
 48 | 	return strcmp(t1->topic_name,t2->topic_name);
 49 | }
 50 | 
 51 | static int void_topics_cmp(const void *_t1, const void *_t2) {
 52 | 	assert(_t1);
 53 | 	assert(_t2);
 54 | 	const struct topic_s *t1 = _t1;
 55 | 	const struct topic_s *t2 = _t2;
 56 | 
 57 | #ifdef TOPIC_S_MAGIC
 58 | 	assert(TOPIC_S_MAGIC == t1->magic);
 59 | 	assert(TOPIC_S_MAGIC == t2->magic);
 60 | #endif
 61 | 
 62 | 	return topics_cmp(t1,t2);
 63 | }
 64 | 
 65 | struct topics_db {
 66 | 	rd_avl_t topics;
 67 | 	/// @TODO change by a memctx
 68 | 	topics_list list;
 69 | };
 70 | 
 71 | const char *topics_db_partition_key(struct topic_s *topic) {
 72 | 	return topic->partition_key;
 73 | }
 74 | 
 75 | struct topics_db *topics_db_new() {
 76 | 	struct topics_db *ret = calloc(1,sizeof(*ret));
 77 | 
 78 | 	if(ret) {
 79 | 		rd_avl_init(&ret->topics, void_topics_cmp, 0);
 80 | 		topic_list_init(&ret->list);
 81 | 	}
 82 | 
 83 | 	return ret;
 84 | }
 85 | 
 86 | static void free_topics(topics_list *list) {
 87 | 	struct topic_s *elm = NULL, *tmpelm = NULL;
 88 | 	TAILQ_FOREACH_SAFE(elm, tmpelm, list, list_node) {
 89 | #ifdef TOPIC_S_MAGIC
 90 | 		assert(TOPIC_S_MAGIC == elm->magic);
 91 | #endif
 92 | 		topic_decref(elm);
 93 | 	}
 94 | }
 95 | 
 96 | static struct topic_s *get_topic_borrow(struct topics_db *db, const char *topic) {
 97 | 	char buf[strlen(topic) + 1];
 98 | 	strcpy(buf, topic);
 99 | 
100 | 	struct topic_s dumb_topic = {
101 | #ifdef TOPIC_S_MAGIC
102 | 		.magic = TOPIC_S_MAGIC,
103 | #endif
104 | 		.topic_name = buf
105 | 	};
106 | 
107 | 	return RD_AVL_FIND_NODE_NL(&db->topics, &dumb_topic);
108 | }
109 | 
110 | struct topic_s *topics_db_get_topic(struct topics_db *db, const char *topic) {
111 | 	struct topic_s *ret = get_topic_borrow(db,topic);
112 | 	if(ret) {
113 | 		ATOMIC_OP(add,fetch,&ret->refcnt,1);
114 | 	}
115 | 	return ret;
116 | }
117 | 
118 | int topics_db_topic_exists(struct topics_db *db, const char *topic) {
119 | 	return NULL != get_topic_borrow(db, topic);
120 | }
121 | 
122 | int topics_db_add(struct topics_db *db,rd_kafka_topic_t *rkt,
123 |         const char *partition_key, size_t partition_key_len) {
124 | 	/* Quick hack to finalize partition_key with '\0' */
125 | 	char zero='\0',*aux_zero=NULL;
126 | 
127 | 	struct topic_s *topic_s = NULL;
128 | 
129 | 	const char *topic_name = rd_kafka_topic_name(rkt);
130 | 
131 | 	rd_calloc_struct(&topic_s,sizeof(*topic_s),
132 | 			-1,topic_name,&topic_s->topic_name,
133 | 			partition_key_len,partition_key,&topic_s->partition_key,
134 | 			1,&zero,aux_zero,
135 | 			RD_MEM_END_TOKEN);
136 | 
137 | 	if(topic_s) {
138 | #ifdef TOPIC_S_MAGIC
139 | 		topic_s->magic = TOPIC_S_MAGIC;
140 | #endif
141 | 
142 | 		if (0 == partition_key_len) {
143 | 			topic_s->partition_key = NULL;
144 | 		}
145 | 
146 | 		topic_s->rkt = rkt;
147 | 		topic_s->refcnt = 1;
148 | 
149 | 		RD_AVL_INSERT(&db->topics,topic_s,avl_node);
150 | 		topic_list_push(&db->list,topic_s);
151 | 	}
152 | 
153 | 	return topic_s != NULL;
154 | }
155 | 
156 | void topics_db_done(struct topics_db *db) {
157 |    free_topics(&db->list);
158 |    free(db);
159 | }


--------------------------------------------------------------------------------
/src/util/topic_database.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <librdkafka/rdkafka.h>
 4 | 
 5 | struct topic_s;
 6 | 
 7 | void topic_decref(struct topic_s *topic);
 8 | 
 9 | /** Topics database.
10 | 	@warning The only function thread safe is topics_db_get_topic */
11 | struct topics_db;
12 | 
13 | /** It creates a new database */
14 | struct topics_db *topics_db_new();
15 | 
16 | /** Add a topic into a database 
17 | 	@param topics_db Topics database
18 | 	@param topic Topic to produce
19 | 	@param partition_key Field to extract partition key
20 | 	@param partition_key_len Length of partition key
21 | 	@return 1 if OK, 0 ioc.
22 | 	*/
23 | int topics_db_add(struct topics_db *topics_db,rd_kafka_topic_t *rkt,
24 | 	const char *partition_key,size_t partition_key_len);
25 | 
26 | /** Returns JSON key for what message has to be partitioned
27 | 	@param topic Topic.
28 | 	@return JSON key */
29 | const char *topics_db_partition_key(struct topic_s *topic);
30 | 
31 | /** Get a topic from database.
32 | 	@param db Database
33 | 	@param topic Topic name to search
34 | 	@return Associated topic. Need to decref returned value when finish. */
35 | struct topic_s *topics_db_get_topic(struct topics_db *db, const char *topic);
36 | 
37 | /** Check if a topic exists in database
38 | 	@param db Database to search in.
39 | 	@param topic Topic to search for.
40 | 	@return 1 if exists, 0 if not */
41 | int topics_db_topic_exists(struct topics_db *db, const char *topic);
42 | 
43 | /** Extract rdkafka topic from topic handler
44 | 	@param topic topic handler
45 | 	@return rdkafka usable topic */
46 | rd_kafka_topic_t *topics_db_get_rdkafka_topic(struct topic_s *topic);
47 | 
48 | /** Get topic handler's topic name
49 | 	@param topic Topic handler
50 | 	@return Name of topic
51 | 	*/
52 | const char *topics_db_get_topic_name(const struct topic_s *topic);
53 | 
54 | /** Destroy a topics db.
55 | 	You can keep using topics extracted from it, but you can't search for more topics
56 | 	*/
57 | void topics_db_done(struct topics_db *topics_db);
58 | 


--------------------------------------------------------------------------------
/src/util/util.h:
--------------------------------------------------------------------------------
 1 | /*
 2 | ** Copyright (C) 2015 Eneo Tecnologia S.L.
 3 | ** Author: Eugenio Perez <eupm90@gmail.com>
 4 | **
 5 | ** This program is free software; you can redistribute it and/or modify
 6 | ** it under the terms of the GNU Affero General Public License as
 7 | ** published by the Free Software Foundation, either version 3 of the
 8 | ** License, or (at your option) any later version.
 9 | **
10 | ** This program is distributed in the hope that it will be useful,
11 | ** but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | ** GNU Affero General Public License for more details.
14 | **
15 | ** You should have received a copy of the GNU General Public License
16 | ** along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 | */
18 | 
19 | #pragma once
20 | 
21 | #include "librd/rdlog.h"
22 | 
23 | #include <string.h>
24 | 
25 | #define RB_UNUSED __attribute__((unused))
26 | 
27 | #ifdef likely
28 | #undef likely
29 | #endif
30 | #define likely(x)       __builtin_expect(!!(x), 1)
31 | 
32 | #ifdef unlikely
33 | #undef unlikely
34 | #endif
35 | #define unlikely(x)     __builtin_expect(!!(x), 0)
36 | 
37 | #define rblog(x...) rdlog(x)
38 | 
39 | #define fatal(msg...) do{rblog(LOG_ERR,msg);exit(1);}while(0)
40 | 
41 | #define swap_ptrs(p1,p2) do{void *aux = p1;p1 = p2;p2 = aux;}while(0)
42 | 
43 | static inline char *mystrerror(int _errno,char *buffer,size_t buffer_size){
44 | 	strerror_r(_errno,buffer,buffer_size);
45 | 	return buffer;
46 | }
47 | 
48 | 


--------------------------------------------------------------------------------
/tests/0000-MSE8.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0001-MSE10.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0002-meraki.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0003-MSE8_listener_enrich.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0004-MSE10_listener_enrich.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/decoder/rb_http2k/rb_http2k_decoder.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0005-rb_decoder.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0006-meraki-null-fields.c:
--------------------------------------------------------------------------------
  1 | #include "rb_meraki_tests.h"
  2 | 
  3 | #include <setjmp.h>
  4 | #include <cmocka.h>
  5 | 
  6 | // these message have some fields null, meraki decoder should throw them
  7 | // away
  8 | static const char MERAKI_MSG[] =
  9 | 	"{"
 10 | 	    "\"secret\": \"redborder\","
 11 | 	    "\"version\": \"2.0\","
 12 | 	    "\"type\": \"DevicesSeen\","
 13 | 	    "\"data\": {"
 14 | 	        "\"apMac\": \"00:a0:f9:37:a0:d1\","
 15 | 	        "\"apFloors\": [],"
 16 | 	        "\"observations\": [{"
 17 | 	            "\"clientMac\": \"00:00:5f:f1:cd:8d\","
 18 | 	            "\"ipv4\": null,"
 19 | 	            "\"ipv6\": null,"
 20 | 	            "\"seenTime\": \"2016-03-01T14:57:49Z\","
 21 | 	            "\"seenEpoch\": 1456844269,"
 22 | 	            "\"ssid\": null,"
 23 | 	            "\"rssi\": 34,"
 24 | 	            "\"manufacturer\": \"SamsungE\","
 25 | 	            "\"os\": null,"
 26 | 	            "\"location\": null"
 27 | 	        "}, {"
 28 | 	            "\"clientMac\": \"00:00:c5:46:53:10\","
 29 | 	            "\"ipv4\": null,"
 30 | 	            "\"ipv6\": null,"
 31 | 	            "\"seenTime\": \"2016-03-01T14:57:51Z\","
 32 | 	            "\"seenEpoch\": 1456844271,"
 33 | 	            "\"ssid\": null,"
 34 | 	            "\"rssi\": 14,"
 35 | 	            "\"manufacturer\": \"HonHaiPr\","
 36 | 	            "\"os\": null,"
 37 | 	            "\"location\": null"
 38 | 	        "}, {"
 39 | 	            "\"clientMac\": \"00:00:51:1f:53:b0\","
 40 | 	            "\"ipv4\": null,"
 41 | 	            "\"ipv6\": null,"
 42 | 	            "\"seenTime\": \"2016-03-01T14:57:52Z\","
 43 | 	            "\"seenEpoch\": 1456844272,"
 44 | 	            "\"ssid\": null,"
 45 | 	            "\"rssi\": 14,"
 46 | 	            "\"manufacturer\": \"LiteonTe\","
 47 | 	            "\"os\": null,"
 48 | 	            "\"location\": null"
 49 | 	        "}]"
 50 | 	    "}"
 51 | 	"}";
 52 |   // *INDENT-ON*
 53 | 
 54 | /** Location is not NULL or valid object, so output message should not contains
 55 | 	location
 56 | 	*/
 57 | static const char MERAKI_MSG_BAD_FORMAT_LOCATION[] =
 58 | 	"{"
 59 | 	    "\"secret\": \"redborder\","
 60 | 	    "\"version\": \"2.0\","
 61 | 	    "\"type\": \"DevicesSeen\","
 62 | 	    "\"data\": {"
 63 | 	        "\"apMac\": \"00:a0:f9:37:a0:d1\","
 64 | 	        "\"apFloors\": [],"
 65 | 	        "\"observations\": [{"
 66 | 	            "\"clientMac\": \"00:00:5f:f1:cd:8d\","
 67 | 	            "\"ipv4\": null,"
 68 | 	            "\"ipv6\": null,"
 69 | 	            "\"seenTime\": \"2016-03-01T14:57:49Z\","
 70 | 	            "\"seenEpoch\": 1456844269,"
 71 | 	            "\"ssid\": null,"
 72 | 	            "\"rssi\": 34,"
 73 | 	            "\"manufacturer\": \"SamsungE\","
 74 | 	            "\"os\": null,"
 75 | 	            "\"location\": \"abc\""
 76 | 	        "}, {"
 77 | 	            "\"clientMac\": \"00:00:c5:46:53:10\","
 78 | 	            "\"ipv4\": null,"
 79 | 	            "\"ipv6\": null,"
 80 | 	            "\"seenTime\": \"2016-03-01T14:57:51Z\","
 81 | 	            "\"seenEpoch\": 1456844271,"
 82 | 	            "\"ssid\": null,"
 83 | 	            "\"rssi\": 14,"
 84 | 	            "\"manufacturer\": \"HonHaiPr\","
 85 | 	            "\"os\": null,"
 86 | 	            "\"location\": 6"
 87 | 	        "}, {"
 88 | 	            "\"clientMac\": \"00:00:51:1f:53:b0\","
 89 | 	            "\"ipv4\": null,"
 90 | 	            "\"ipv6\": null,"
 91 | 	            "\"seenTime\": \"2016-03-01T14:57:52Z\","
 92 | 	            "\"seenEpoch\": 1456844272,"
 93 | 	            "\"ssid\": null,"
 94 | 	            "\"rssi\": 14,"
 95 | 	            "\"manufacturer\": \"LiteonTe\","
 96 | 	            "\"os\": null,"
 97 | 	            "\"location\": true"
 98 | 	        "}]"
 99 | 	    "}"
100 | 	"}";
101 |   // *INDENT-ON*
102 | 
103 | static const char MERAKI_SECRETS[] = \
104 |   // *INDENT-OFF*
105 | 	"{"
106 | 		/* "\"meraki-secrets\": {" */
107 | 	        "\"redborder\": { "
108 | 	          "\"sensor_name\": \"meraki1\" "
109 | 	          ", \"sensor_id\": 2"
110 | 	        "},"
111 | 	        "\"redborder2\": { "
112 | 	          "\"sensor_name\": \"meraki2\" "
113 | 	          ", \"sensor_id\": 3"
114 | 	        "}"
115 | 	    /* "}" */
116 | 	"}";
117 |   // *INDENT-ON*
118 | 
119 | CHECKDATA(check1,
120 |     {.key = "type", .value = "meraki"},
121 |     {.key = "client_mac", .value="00:00:5f:f1:cd:8d"},
122 |     {.key = "client_mac_vendor", .value = "SamsungE"},
123 |     {.key = "src", .value = NULL},
124 |     {.key = "wireless_id", .value = NULL},
125 |     {.key = "client_rssi_num", .value ="-61"},
126 |     {.key = "os", .value = NULL},
127 |     {.key = "location", .value = NULL},
128 | );
129 | 
130 | CHECKDATA(check2,
131 |     {.key = "type", .value = "meraki"},
132 |     {.key = "client_mac", .value="00:00:c5:46:53:10"},
133 |     {.key = "client_mac_vendor", .value = "HonHaiPr"},
134 |     {.key = "src", .value = NULL},
135 |     {.key = "wireless_id", .value = NULL},
136 |     {.key = "client_rssi_num", .value ="-81"},
137 |     {.key = "os", .value = NULL},
138 |     {.key = "location", .value = NULL},
139 | );
140 | 
141 | CHECKDATA(check3,
142 |     {.key = "type", .value = "meraki"},
143 |     {.key = "client_mac", .value="00:00:51:1f:53:b0"},
144 |     {.key = "client_mac_vendor", .value = "LiteonTe"},
145 |     {.key = "src", .value = NULL},
146 |     {.key = "wireless_id", .value = NULL},
147 |     {.key = "client_rssi_num", .value ="-81"},
148 |     {.key = "os", .value = NULL},
149 |     {.key = "location", .value = NULL},
150 | );
151 | 
152 | static void MerakiDecoder_nulls() {
153 | 	static const struct checkdata *checkdata_array[] = {
154 | 		&check1, &check2, &check3
155 | 	};
156 | 
157 | 	static const struct checkdata_array checkdata = {
158 | 		.checks = checkdata_array,
159 | 		.size = sizeof(checkdata_array) / sizeof(checkdata_array[0]),
160 | 	};
161 | 
162 | 	MerakiDecoder_test_base(NULL, MERAKI_SECRETS,
163 | 		MERAKI_MSG, &checkdata);
164 | }
165 | 
166 | static void MerakiDecoder_bad_location() {
167 | 	static const struct checkdata *checkdata_array[] = {
168 | 		&check1, &check2, &check3
169 | 	};
170 | 
171 | 	static const struct checkdata_array checkdata = {
172 | 		.checks = checkdata_array,
173 | 		.size = sizeof(checkdata_array) / sizeof(checkdata_array[0]),
174 | 	};
175 | 
176 | 	MerakiDecoder_test_base(NULL, MERAKI_SECRETS,
177 | 		MERAKI_MSG_BAD_FORMAT_LOCATION, &checkdata);
178 | }
179 | 
180 | int main() {
181 | 	const struct CMUnitTest tests[] = {
182 | 		cmocka_unit_test(MerakiDecoder_nulls),
183 | 		cmocka_unit_test(MerakiDecoder_bad_location),
184 | 	};
185 | 
186 | 	return cmocka_run_group_tests(tests, NULL, NULL);
187 | 
188 | 	return 0;
189 | }
190 | 


--------------------------------------------------------------------------------
/tests/0006-meraki-null-fields.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o  src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o
2 | 


--------------------------------------------------------------------------------
/tests/0007-mac-partitioner.c:
--------------------------------------------------------------------------------
  1 | #include "rb_json_tests.c"
  2 | #include <librdkafka/rdkafka.h>
  3 | 
  4 | #include "../src/decoder/rb_http2k/rb_http2k_decoder.c"
  5 | #include "../src/listener/http.c"
  6 | 
  7 | #include <assert.h>
  8 | #include <setjmp.h>
  9 | #include <cmocka.h>
 10 | 
 11 | static const int32_t default_partition_cnt = 4;
 12 | 
 13 | struct test_mac {
 14 | 	const char mac[sizeof("00:00:00:00:00:00")];
 15 | 	int32_t idx;
 16 | };
 17 | 
 18 | static const struct test_mac test_mac[] = {
 19 | 	{.mac = "d2:65:2f:83:68:cd", .idx = 1},
 20 | 	{.mac = "97:0f:d4:6e:92:05", .idx = 1},
 21 | 	{.mac = "f8:43:aa:3b:e0:ce", .idx = 2},
 22 | 	{.mac = "de:fb:64:43:65:ae", .idx = 2},
 23 | 	{.mac = "ab:e4:94:bc:61:78", .idx = 0},
 24 | 	{.mac = "4a:7e:b9:9c:3b:89", .idx = 1},
 25 | 	{.mac = "00:1c:bf:45:7a:68", .idx = 0},
 26 | 	{.mac = "c2:cb:7c:3a:9b:a9", .idx = 1},
 27 | 	{.mac = "2c:77:58:60:dd:ce", .idx = 2},
 28 | 	{.mac = "05:2e:4c:41:84:05", .idx = 1},
 29 | 	{.mac = "6a:76:92:dc:84:c6", .idx = 2},
 30 | 	{.mac = "fe:12:91:75:19:b8", .idx = 0},
 31 | 	{.mac = "05:85:19:42:15:cb", .idx = 3},
 32 | 	{.mac = "67:d0:9a:59:9f:5e", .idx = 2},
 33 | 	{.mac = "97:70:ba:00:23:a6", .idx = 2},
 34 | 	{.mac = "5c:4d:ee:b5:23:6e", .idx = 2},
 35 | 	{.mac = "46:c1:04:97:f5:ba", .idx = 2},
 36 | 	{.mac = "f7:1f:cd:1b:a9:82", .idx = 2},
 37 | 	{.mac = "cd:0b:b6:b8:92:a0", .idx = 0},
 38 | 	{.mac = "48:a3:c3:25:da:aa", .idx = 2},
 39 | 	{.mac = "90:a3:b5:ef:e9:d1", .idx = 1},
 40 | 	{.mac = "d9:43:06:a8:f2:43", .idx = 3},
 41 | 	{.mac = "2c:3f:e6:35:bf:b0", .idx = 0},
 42 | 	{.mac = "32:dc:d4:4d:65:da", .idx = 2},
 43 | 	{.mac = "3a:15:85:35:4c:78", .idx = 0},
 44 | 	{.mac = "49:80:d6:34:e5:da", .idx = 2},
 45 | 	{.mac = "89:0b:35:43:a2:53", .idx = 3},
 46 | 	{.mac = "78:ec:60:96:d4:e2", .idx = 2},
 47 | 	{.mac = "c3:ce:a8:7d:89:3c", .idx = 0},
 48 | 	{.mac = "83:e1:bc:5d:cd:73", .idx = 3},
 49 | };
 50 | 
 51 | static int32_t mac_partitioner (const rd_kafka_topic_t *rkt,
 52 |                                 const void *keydata, size_t keylen,
 53 |                                 int32_t partition_cnt,
 54 |                                 void *rkt_opaque, void *msg_opaque);
 55 | 
 56 | /// @note HOOK to know that mac_partitioner could not find a valid MAC
 57 | int32_t rd_kafka_msg_partitioner_random(const rd_kafka_topic_t *rkt,
 58 |                                 const void *keydata, size_t keylen,
 59 |                                 int32_t partition_cnt,
 60 |                                 void *rkt_opaque, void *msg_opaque) {
 61 | 	(void)rkt;
 62 | 	(void)keydata;
 63 | 	(void)keylen;
 64 | 	(void)partition_cnt;
 65 | 	(void)rkt_opaque;
 66 | 	(void)msg_opaque;
 67 | 	return 100;
 68 | }
 69 | 
 70 | static void samples_test() {
 71 | 	const rd_kafka_topic_t *rkt = NULL;
 72 | 	void *rkt_opaque = NULL, *msg_opaque = NULL;
 73 | 
 74 | 	size_t i;
 75 | 	for(i=0; i<sizeof(test_mac)/sizeof(test_mac[0]); ++i) {
 76 | 		int32_t result = mac_partitioner(rkt,
 77 | 		                test_mac[i].mac, strlen(test_mac[i].mac),
 78 | 		                default_partition_cnt,
 79 | 		                rkt_opaque, msg_opaque);
 80 | 
 81 | 		assert(result == test_mac[i].idx);
 82 | 	}
 83 | }
 84 | 
 85 | /// Tests all mac lengths between 0 and mac_length+1
 86 | static void invalid_mac_len_test() {
 87 | 	size_t i;
 88 | 	const rd_kafka_topic_t *rkt = NULL;
 89 | 	void *rkt_opaque = NULL, *msg_opaque = NULL;
 90 | 	char aux_mac[sizeof(test_mac[0].mac) + 1];
 91 | 	const size_t mac_len = strlen(test_mac[0].mac);
 92 | 
 93 | 	/// Test all sizes under right size
 94 | 	for (i = 0; i <= mac_len + 1; ++i) {
 95 | 		if (i == mac_len) {
 96 | 			continue;
 97 | 		}
 98 | 
 99 | 		aux_mac[0] = '\0';
100 | 		strncat(aux_mac,test_mac[0].mac,i);
101 | 
102 | 		if (i > mac_len) {
103 | 			aux_mac[mac_len] = '0';
104 | 			aux_mac[i] = '\0';
105 | 		}
106 | 
107 | 		const int32_t result = mac_partitioner(rkt,
108 | 		                aux_mac, strlen(aux_mac),
109 | 		                default_partition_cnt,
110 | 		                rkt_opaque, msg_opaque);
111 | 
112 | 		assert(100 == result);
113 | 	}
114 | }
115 | 
116 | /// Test introducing weird characters in all position of the mac
117 | static void invalid_mac_character() {
118 | 	size_t i,c;
119 | 	const rd_kafka_topic_t *rkt = NULL;
120 | 	void *rkt_opaque = NULL, *msg_opaque = NULL;
121 | 	char aux_mac[sizeof(test_mac[0].mac)];
122 | 	aux_mac[0] = '\0';
123 | 	strncat(aux_mac,test_mac[0].mac,sizeof(aux_mac));
124 | 
125 | 	const char buggy_chars[] = {'g', '?', '\0', ':'};
126 | 
127 | 	for (c = 0; c < sizeof(buggy_chars)/sizeof(buggy_chars[0]); ++c) {
128 | 		for (i=0; i<strlen(aux_mac); ++i) {
129 | 			if (aux_mac[i] == buggy_chars[c]) {
130 | 				/* Already the same, this will not raise error*/
131 | 				continue;
132 | 			}
133 | 			char aux = aux_mac[i];
134 | 			aux_mac[i] = buggy_chars[c];
135 | 
136 | 			const int32_t result = mac_partitioner(rkt,
137 | 			                aux_mac, strlen(aux_mac),
138 | 			                default_partition_cnt,
139 | 			                rkt_opaque, msg_opaque);
140 | 
141 | 			aux_mac[i] = aux;
142 | 
143 | 			assert(100 == result);
144 | 		}
145 | 	}
146 | }
147 | 
148 | int main() {
149 | 	const struct CMUnitTest tests[] = {
150 | 		cmocka_unit_test(samples_test),
151 | 		cmocka_unit_test(invalid_mac_len_test),
152 | 		cmocka_unit_test(invalid_mac_character),
153 | 	};
154 | 
155 | 	return cmocka_run_group_tests(tests, NULL, NULL);
156 | }
157 | 


--------------------------------------------------------------------------------
/tests/0007-mac-partitioner.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0008-http2k-malformed-messages.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0009-http2k-url.c:
--------------------------------------------------------------------------------
  1 | #include "rb_json_tests.c"
  2 | #include "rb_http2k_tests.c"
  3 | 
  4 | #include <setjmp.h>
  5 | #include <cmocka.h>
  6 | #include <assert.h>
  7 | 
  8 | static const char TEMP_TEMPLATE[] = "n2ktXXXXXX";
  9 | 
 10 | static const char CONFIG_TEST[] =
 11 |     "{"
 12 |         "\"brokers\": \"localhost\","
 13 |         "\"rb_http2k_config\": {"
 14 |             "\"sensors_uuids\" : {"
 15 |                     "\"abc\" : {"
 16 |                         "\"enrichment\": {"
 17 |                             "\"sensor_uuid\":\"abc\","
 18 |                             "\"a\":1,"
 19 |                             "\"b\":\"c\","
 20 |                             "\"d\":true,"
 21 |                             "\"e\":null"
 22 |                         "}"
 23 |                     "},"
 24 |                     "\"def\" : {"
 25 |                         "\"enrichment\": {"
 26 |                             "\"sensor_uuid\":\"def\","
 27 |                             "\"f\":1,"
 28 |                             "\"g\":\"w\","
 29 |                             "\"h\":false,"
 30 |                             "\"i\":null,"
 31 |                             "\"j\":2.5"
 32 |                         "}"
 33 |                     "},"
 34 |                     "\"ghi\" : {"
 35 |                         "\"enrichment\": {"
 36 |                             "\"o\": {"
 37 |                                 "\"a\":90"
 38 |                             "}"
 39 |                         "}"
 40 |                     "},"
 41 |                     "\"jkl\" : {"
 42 |                         "\"enrichment\": {"
 43 |                             "\"v\":[1,2,3,4,5]"
 44 |                         "}"
 45 |                     "}"
 46 |             "},"
 47 |             "\"topics\" : {"
 48 |                     "\"rb_flow\": {"
 49 |                             "\"partition_key\":\"client_mac\","
 50 |                             "\"partition_algo\":\"mac\""
 51 |                     "},"
 52 |                     "\"rb_event\": {"
 53 |                     "}"
 54 |             "}"
 55 |         "}"
 56 |     "}";
 57 | 
 58 | static void validate_test(void (*f)(struct rb_config *rb)) {
 59 | 	init_global_config();
 60 | 	char temp_filename[sizeof(TEMP_TEMPLATE)];
 61 | 	strcpy(temp_filename,TEMP_TEMPLATE);
 62 | 	int temp_fd = mkstemp(temp_filename);
 63 | 	assert(temp_fd >= 0);
 64 | 	write(temp_fd, CONFIG_TEST, strlen(CONFIG_TEST));
 65 | 
 66 | 	parse_config(temp_filename);
 67 | 	unlink(temp_filename);
 68 | 
 69 | 	f(&global_config.rb);
 70 | 
 71 | 	free_global_config();
 72 | 
 73 | 	close(temp_fd);
 74 | }
 75 | 
 76 | static void validate_topic_test0(struct rb_config *rb) {
 77 | 	size_t i;
 78 | 	struct {
 79 | 		const char *topic;
 80 | 		int expected;
 81 | 	} validations[] = {
 82 | 		{.topic = "rb_flow", .expected = 1},
 83 | 		{.topic = "rb_event", .expected = 1},
 84 | 		{.topic = "rb_unknown", .expected = 0},
 85 | 		{.topic = "rb_anotherone", .expected = 0},
 86 | 	};
 87 | 
 88 | 	for (i=0; i<sizeof(validations)/sizeof(validations[0]); ++i) {
 89 | 		const int result = rb_http2k_validate_topic(
 90 | 			&rb->database,validations[i].topic);
 91 | 		assert(validations[i].expected == result);
 92 | 	}
 93 | }
 94 | 
 95 | static void validate_topic_test() {
 96 | 	validate_test(validate_topic_test0);
 97 | }
 98 | 
 99 | static void validate_uuid_test0(struct rb_config *rb) {
100 | 	size_t i;
101 | 	struct {
102 | 		const char *uuid;
103 | 		int expected;
104 | 	} validations[] = {
105 | 		{.uuid = "abc", .expected = 1},
106 | 		{.uuid = "def", .expected = 1},
107 | 		{.uuid = "ghi", .expected = 1},
108 | 		{.uuid = "jkl", .expected = 1},
109 | 		{.uuid = "mno", .expected = 0},
110 | 	};
111 | 
112 | 	for (i=0; i<sizeof(validations)/sizeof(validations[0]); ++i) {
113 | 		const int result = rb_http2k_validate_uuid(
114 | 			&rb->database,validations[i].uuid);
115 | 		assert(validations[i].expected == result);
116 | 	}
117 | }
118 | 
119 | static void validate_uuid_test() {
120 | 	validate_test(validate_uuid_test0);
121 | }
122 | 
123 | #if 0
124 | int rb_http2k_validate_topic(struct rb_database *db, const char *topic) {
125 | 	pthread_rwlock_rdlock(&db->rwlock);
126 | 	const int ret = topics_db_topic_exists(db->topics_db,topic);
127 | 	pthread_rwlock_unlock(&db->rwlock);
128 | }
129 | #endif
130 | 
131 | 
132 | int main() {
133 | 	const struct CMUnitTest tests[] = {
134 | 		cmocka_unit_test(validate_uuid_test),
135 | 		cmocka_unit_test(validate_topic_test),
136 | 	};
137 | 
138 | 	return cmocka_run_group_tests(tests, NULL, NULL);
139 | 
140 | 
141 | 	return 0;
142 | }
143 | 


--------------------------------------------------------------------------------
/tests/0009-http2k-url.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0010-http2k-organization_enrichment.c:
--------------------------------------------------------------------------------
  1 | #include "rb_json_tests.c"
  2 | #include "rb_http2k_tests.c"
  3 | 
  4 | #include "../src/listener/http.c"
  5 | 
  6 | #include <setjmp.h>
  7 | #include <cmocka.h>
  8 | #include <assert.h>
  9 | 
 10 | static const char CONFIG_TEST[] =
 11 |     "{"
 12 |         "\"brokers\": \"localhost\","
 13 |         "\"rb_http2k_config\": {"
 14 |             "\"sensors_uuids\" : {"
 15 |                 "\"abc\" : {"
 16 |                     "\"enrichment\": {"
 17 |                         "\"sensor_uuid\":\"abc\","
 18 |                         "\"a\":1,"
 19 |                         "\"b\":\"c\","
 20 |                         "\"d\":true,"
 21 |                         "\"e\":null"
 22 |                     "},"
 23 |                     "\"organization_uuid\":\"abc_org\""
 24 |                 "},"
 25 |                 "\"def\" : {"
 26 |                     "\"enrichment\": {"
 27 |                         "\"sensor_uuid\":\"def\","
 28 |                         "\"a\":2,"
 29 |                         "\"g\":\"w\","
 30 |                         "\"h\":false,"
 31 |                         "\"i\":null,"
 32 |                         "\"j\":2.5"
 33 |                     "},"
 34 |                     "\"organization_uuid\":\"def_org\""
 35 |                 "}"
 36 |             "},"
 37 |             "\"organizations_uuids\":{"
 38 |                 "\"abc_org\" : {"
 39 |                   "\"enrichment\":{"
 40 |                     "\"a\":3,"
 41 |                     "\"k\":4"
 42 |                   "}"
 43 |                 "},"
 44 |                 "\"def_org\" : {"
 45 |                   "\"enrichment\":{"
 46 |                     "\"a\":5,"
 47 |                     "\"k\":10"
 48 |                   "}"
 49 |                 "}"
 50 |             "},"
 51 |             "\"topics\" : {"
 52 |                     "\"rb_flow\": {"
 53 |                             "\"partition_key\":\"client_mac\","
 54 |                             "\"partition_algo\":\"mac\""
 55 |                     "},"
 56 |                     "\"rb_event\": {"
 57 |                     "}"
 58 |             "}"
 59 |         "}"
 60 |     "}";
 61 | 
 62 | static void prepare_args(
 63 |         const char *topic,const char *sensor_uuid,const char *client_ip,
 64 |         struct pair *mem,size_t memsiz,keyval_list_t *list) {
 65 | 	assert(3==memsiz);
 66 | 	memset(mem,0,sizeof(*mem)*3);
 67 | 
 68 | 	mem[0].key   = "topic";
 69 | 	mem[0].value = topic;
 70 | 	mem[1].key   = "sensor_uuid";
 71 | 	mem[1].value = sensor_uuid;
 72 | 	mem[2].key   = "client_ip";
 73 | 	mem[2].value = client_ip;
 74 | 
 75 | 	add_key_value_pair(list,&mem[0]);
 76 | 	add_key_value_pair(list,&mem[1]);
 77 | 	add_key_value_pair(list,&mem[2]);
 78 | }
 79 | 
 80 | static void check_organization_enrichment_abc(struct rb_session **sess,
 81 |                 void *unused __attribute__((unused))) {
 82 | 	rd_kafka_message_t rkm;
 83 | 	json_error_t jerr;
 84 | 	const char *client_mac,*application_name,*sensor_uuid,*b;
 85 | 	json_int_t a,k;
 86 | 	int d;
 87 | 
 88 | 	assert(1==rd_kafka_msg_q_size(&(*sess)->msg_queue));
 89 | 	rd_kafka_msg_q_dump(&(*sess)->msg_queue,&rkm);
 90 | 	assert(0==rd_kafka_msg_q_size(&(*sess)->msg_queue));
 91 | 
 92 | 	json_t *root = json_loadb(rkm.payload, rkm.len, 0, &jerr);
 93 | 	if(NULL == root) {
 94 | 		rdlog(LOG_ERR,"Couldn load file: %s",jerr.text);
 95 | 		assert(0);
 96 | 	}
 97 | 
 98 | 	const int rc = json_unpack_ex(root, &jerr, JSON_STRICT,
 99 | 		"{s:s,s:s,s:s,s:I,s:s,s:n,s:b,s:I}",
100 | 		"client_mac",&client_mac,"application_name",&application_name,
101 | 		"sensor_uuid",&sensor_uuid,"a",&a,"b",&b,"e","d",&d,"k",&k);
102 | 
103 | 	if (rc != 0) {
104 | 		rdlog(LOG_ERR,"Couldn't unpack values: %s",jerr.text);
105 | 		assert(0);
106 | 	}
107 | 
108 | 	assert(0==strcmp(client_mac,"54:26:96:db:88:01"));
109 | 	assert(0==strcmp(application_name,"wwww"));
110 | 	assert(0==strcmp(sensor_uuid,"abc"));
111 | 	assert(a == 1); /* Enrichment! original message had 5 here,
112 |                            and organization has 3 */
113 | 	assert(0==strcmp(b,"c"));
114 | 	assert(d == 1);
115 |         assert(k == 4); /* organization enrichment */
116 | 
117 | 	json_decref(root);
118 | 	free(rkm.payload);
119 | }
120 | 
121 | static void test_organization_enrichment() {
122 |         struct pair mem[3];
123 |         keyval_list_t args;
124 |         keyval_list_init(&args);
125 |         prepare_args("rb_flow","abc","127.0.0.1",mem,RD_ARRAYSIZE(mem),&args);
126 | 
127 | #define MESSAGES                                                              \
128 |         X("{\"client_mac\":\"54:26:96:db:88:01\","                            \
129 |                 "\"application_name\":\"wwww\",\"sensor_uuid\":\"abc\","      \
130 |                 "\"a\":5}",                                                   \
131 |                 check_organization_enrichment_abc)                            \
132 |         /* Free & Check that session has been freed */                        \
133 |         X(NULL,check_null_session)
134 | 
135 |         struct message_in msgs[] = {
136 | #define X(a,fn) {a,sizeof(a)-1},
137 |                 MESSAGES
138 | #undef X
139 |         };
140 | 
141 |         check_callback_fn callbacks_functions[] = {
142 | #define X(a,fn) fn,
143 |                 MESSAGES
144 | #undef X
145 |         };
146 | 
147 |         test_rb_decoder0(CONFIG_TEST, &args, msgs, callbacks_functions,
148 |                 RD_ARRAYSIZE(msgs), NULL);
149 | 
150 | #undef MESSAGES
151 | }
152 | 
153 | int main() {
154 | 	const struct CMUnitTest tests[] = {
155 | 		cmocka_unit_test(test_organization_enrichment),
156 | 	};
157 | 
158 | 	return cmocka_run_group_tests(tests, NULL, NULL);
159 | }
160 | 


--------------------------------------------------------------------------------
/tests/0010-http2k-organization_enrichment.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0011-rb_http2k-limits.c:
--------------------------------------------------------------------------------
  1 | #include "rb_json_tests.c"
  2 | #include "rb_http2k_tests.c"
  3 | 
  4 | #include "../src/listener/http.c"
  5 | 
  6 | #include <setjmp.h>
  7 | #include <cmocka.h>
  8 | #include <assert.h>
  9 | 
 10 | static const char TEMP_TEMPLATE[] = "n2ktXXXXXX";
 11 | 
 12 | static const char CONFIG_TEST[] =
 13 |     "{"
 14 |         "\"brokers\": \"localhost\","
 15 |         "\"rb_http2k_config\": {"
 16 |             "\"sensors_uuids\" : {"
 17 |                     "\"abc\" : {"
 18 |                         "\"enrichment\":{},"
 19 |                         "\"organization_uuid\":\"abc_org\""
 20 |                     "},"
 21 |                     "\"def\" : {"
 22 |                         "\"enrichment\":{},"
 23 |                         "\"organization_uuid\":\"def_org\""
 24 |                     "}"
 25 |             "},"
 26 |             "\"organizations_uuids\":{"
 27 |                     "\"abc_org\" : {"
 28 |                         "\"limits\":{"
 29 |                             "\"bytes\":1024"
 30 |                         "}"
 31 |                     "},"
 32 |                     "\"def_org\" : {"
 33 |                         "\"limits\":{"
 34 |                             "\"bytes\":0"
 35 |                         "}"
 36 |                     "}"
 37 |             "},"
 38 |             "\"topics\" : {"
 39 |                     "\"rb_flow\": {"
 40 |                             "\"partition_key\":\"client_mac\","
 41 |                             "\"partition_algo\":\"mac\""
 42 |                     "},"
 43 |                     "\"rb_event\": {"
 44 |                     "}"
 45 |             "}"
 46 |         "}"
 47 |     "}";
 48 | 
 49 | /** 500 bytes message + given sensor uuid */
 50 | #define MSG500(sensor_uuid) \
 51 | "{\"client_mac\":\"54:26:96:db:88:02\"," \
 52 | "\"application_name\":\"wwww\","          \
 53 | "\"sensor_uuid\":\"" sensor_uuid "\","    \
 54 | "\"padding\":\"aaaaaaaaaaaaaaaaaaaaaa"     \
 55 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 56 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 57 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 58 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 59 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 60 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 61 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 62 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 63 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 64 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"     \
 65 | "aaaaaaaaaaaaaaaaaaaaaaaaaaaa\""           \
 66 | "}"
 67 | 
 68 | static void prepare_args(
 69 |         const char *topic,const char *sensor_uuid,const char *client_ip,
 70 |         struct pair *mem,size_t memsiz,keyval_list_t *list) {
 71 | 	assert(3==memsiz);
 72 | 	memset(mem,0,sizeof(*mem)*3);
 73 | 
 74 | 	mem[0].key   = "topic";
 75 | 	mem[0].value = topic;
 76 | 	mem[1].key   = "sensor_uuid";
 77 | 	mem[1].value = sensor_uuid;
 78 | 	mem[2].key   = "client_ip";
 79 | 	mem[2].value = client_ip;
 80 | 
 81 | 	add_key_value_pair(list,&mem[0]);
 82 | 	add_key_value_pair(list,&mem[1]);
 83 | 	add_key_value_pair(list,&mem[2]);
 84 | }
 85 | 
 86 | static void check_rb_decoder_in_quota(struct rb_session **sess,
 87 |                 void *unused __attribute__((unused))) {
 88 | 	rd_kafka_message_t rkm;
 89 | 	json_error_t jerr;
 90 | 	const char *client_mac;
 91 | 	static const char expected_mac[] = "54:26:96:db:88:02";
 92 | 
 93 | 	assert(1==rd_kafka_msg_q_size(&(*sess)->msg_queue));
 94 | 	rd_kafka_msg_q_dump(&(*sess)->msg_queue,&rkm);
 95 | 	assert(0==rd_kafka_msg_q_size(&(*sess)->msg_queue));
 96 | 
 97 | 	json_t *root = json_loadb(rkm.payload, rkm.len, 0, &jerr);
 98 | 	if(NULL == root) {
 99 | 		rdlog(LOG_ERR,"Couldn load file: %s",jerr.text);
100 | 		assert(0);
101 | 	}
102 | 
103 | 	const int rc = json_unpack_ex(root, &jerr, 0,
104 | 		"{s:s}", "client_mac",&client_mac);
105 | 
106 | 	if(rc != 0) {
107 | 		rdlog(LOG_ERR,"Couldn't unpack values: %s",jerr.text);
108 | 		assert(0);
109 | 	}
110 | 
111 | 	assert(0==strcmp(client_mac,expected_mac));
112 | 	assert(0==strncmp(rkm.key,expected_mac,strlen(expected_mac)));
113 | 
114 | 	json_decref(root);
115 | 	free(rkm.payload);
116 | }
117 | 
118 | static void check_rb_decoder_out_quota(struct rb_session **sess,
119 | 								void *unused) {
120 | 	check_zero_messages(sess,unused);
121 | 	organization_db_entry_t *org = (*sess)->sensor->organization;
122 | 	assert(4*strlen(MSG500("abc")) == organization_consumed_bytes(org));
123 | }
124 | 
125 | /** This test send 2000 bytes to a client that have 1024 bytes of quota. */
126 | static void test_limited_client() {
127 | 	struct pair mem[3];
128 | 	keyval_list_t args;
129 | 	keyval_list_init(&args);
130 | 	prepare_args("rb_flow","abc","127.0.0.1",mem,RD_ARRAYSIZE(mem),&args);
131 | 
132 | #define MESSAGES                                                             \
133 | 	X(MSG500("abc"),check_rb_decoder_in_quota)                           \
134 | 	X(MSG500("abc"),check_rb_decoder_in_quota)                           \
135 | 	X(MSG500("abc"),check_zero_messages)                                 \
136 | 	X(MSG500("abc"),check_rb_decoder_out_quota)                          \
137 | 	/* Free & Check that session has been freed */                       \
138 | 	X(NULL,check_null_session)
139 | 
140 | 	struct message_in msgs[] = {
141 | #define X(a,fn) {a,sizeof(a)-1},
142 | 		MESSAGES
143 | #undef X
144 | 	};
145 | 
146 | 	check_callback_fn callbacks_functions[] = {
147 | #define X(a,fn) fn,
148 | 		MESSAGES
149 | #undef X
150 | 	};
151 | 
152 | 	test_rb_decoder0(CONFIG_TEST, &args, msgs, callbacks_functions,
153 | 		RD_ARRAYSIZE(msgs), NULL);
154 | 
155 | #undef MESSAGES
156 | }
157 | 
158 | int main() {
159 | 	const struct CMUnitTest tests[] = {
160 | 		cmocka_unit_test(test_limited_client),
161 | 	};
162 | 
163 | 	return cmocka_run_group_tests(tests, NULL, NULL);
164 | }
165 | 


--------------------------------------------------------------------------------
/tests/0011-rb_http2k-limits.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0012-http2k-reload.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/rb_http2k_sync_thread.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0013-http2k-clients-reports.objdeps:
--------------------------------------------------------------------------------
1 | src/engine/engine.o src/engine/global_config.o src/util/kafka.o src/util/in_addr_list.o src/listener/http.o src/listener/socket.o version.o src/util/rb_mac.o src/decoder/mse/rb_mse.o src/decoder/meraki/rb_meraki.o src/decoder/rb_http2k/rb_database.o src/decoder/rb_http2k/rb_http2k_decoder.o src/decoder/rb_http2k/rb_http2k_sensors_database.o src/decoder/rb_http2k/rb_http2k_organizations_database.o src/decoder/rb_http2k/rb_http2k_curl_handler.o src/decoder/rb_http2k/uuid_database.o src/decoder/rb_http2k/tommyds/tommyhash.o src/decoder/rb_http2k/tommyds/tommyhashdyn.o src/decoder/rb_http2k/tommyds/tommylist.o src/decoder/rb_http2k/rb_http2k_parser.o src/util/rb_json.o src/engine/rb_addr.o src/util/pair.o src/util/topic_database.o src/util/kafka_message_list.o src/util/rb_timer.o 
2 | 


--------------------------------------------------------------------------------
/tests/0014-http-posts.objdeps:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/redBorder/n2kafka/0bdc5ddeb019d08a28fabf4aa48d912657edbab0/tests/0014-http-posts.objdeps


--------------------------------------------------------------------------------
/tests/0015-kafka-producer.objdeps:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/redBorder/n2kafka/0bdc5ddeb019d08a28fabf4aa48d912657edbab0/tests/0015-kafka-producer.objdeps


--------------------------------------------------------------------------------
/tests/Makefile.tests:
--------------------------------------------------------------------------------
 1 | include ../mklove/Makefile.base
 2 | 
 3 | CC ?= cc
 4 | TEST_PROGS := ${TEST_SRCS:%.c=%.test}
 5 | XML_FILES  := ${TEST_SRCS:%.c=%.test.xml} ${TEST_SRCS:%.c=%.test.mem.xml} \
 6 | 	${TEST_SRCS:%.c=%.drd.xml} ${TEST_SRCS:%.c=%.test.helgrind.xml}
 7 | COV_FILES  := ${TEST_SRCS:%.c=%.gcda} ${TEST_SRCS:%.c=%.gcno}
 8 | CFLAGS += -g
 9 | CFLAGS += -fstack-protector -DFORTIFY_SOURCE=2 --param=ssp-buffer-size=4 -Wformat
10 | CFLAGS += -Wall -Wfloat-equal -Wpointer-arith -O0 \
11 | 	-I$(TOPDIR) -I$(TOPDIR)/src
12 | 
13 | CPPFLAGS := $(filter-out -I. -I./src,$(CPPFLAGS))
14 | 
15 | LIBS += -lcmocka
16 | 
17 | VALGRIND_SUP = --suppressions=valgrind.suppressions
18 | 
19 | COVERAGE_INFO = coverage.info
20 | COVERAGE_OUTPUT_DIRECTORY = out
21 | 
22 | .PHONY = coverage
23 | .PRECIOUS = $(TEST_PROGS)
24 | 
25 | # Profiling
26 | #CFLAGS += -O0 -pg
27 | #LDFLAGS += -pg
28 | 
29 | all: test
30 | 
31 | %.test: %.c %.objdeps
32 | 	@printf "\033[1m\033[33mBuilding $@\033[1m\033[0m\n"
33 | 	@$(CC) $(CFLAGS) $(CPPFLAGS) $< `cat $(subst .c,.objdeps,$<)` -o $@ $(LDFLAGS) $(LIBS)
34 | 
35 | coverage: test
36 | 	@lcov --gcov-tool=gcov -q \
37 | 		--rc lcov_branch_coverage=1 --capture \
38 | 		--directory ../ --output-file ${COVERAGE_INFO}
39 | 	@genhtml --branch-coverage ${COVERAGE_INFO} --output-directory ${COVERAGE_OUTPUT_DIRECTORY} > coverage.out
40 | 	@./display_coverage.sh
41 | 
42 | test: $(TEST_PROGS)
43 | 	@./run_tests.sh
44 | 
45 | clean:
46 | 	rm -f $(TEST_PROGS) $(XML_FILES) $(COV_FILES) $(COVERAGE_INFO)
47 | 	rm -rf $(COVERAGE_OUTPUT_DIRECTORY)
48 | 


--------------------------------------------------------------------------------
/tests/assertion_handler.c:
--------------------------------------------------------------------------------
  1 | #include "assertion_handler.h"
  2 | 
  3 | #include <assert.h>
  4 | #include <jansson.h>
  5 | 
  6 | struct assertion_handler_s {
  7 |   TAILQ_HEAD(, assertion_e) assertion_q;
  8 |   TAILQ_HEAD(, value_e) value_q;
  9 |   pthread_mutex_t assertion_mutex;
 10 |   pthread_mutex_t value_mutex;
 11 | };
 12 | 
 13 | /**
 14 |  * Creates a new assertion_handler
 15 |  * @return assertion_handler The new assertion handler
 16 |  */
 17 | struct assertion_handler_s *assertion_handler_new() {
 18 |   struct assertion_handler_s *assertion_handler =
 19 |       (struct assertion_handler_s *)malloc(sizeof(struct assertion_handler_s));
 20 |   TAILQ_INIT(&assertion_handler->assertion_q);
 21 |   TAILQ_INIT(&assertion_handler->value_q);
 22 | 
 23 |   if ((pthread_mutex_init(&assertion_handler->assertion_mutex, NULL) ||
 24 |        pthread_mutex_init(&assertion_handler->value_mutex, NULL)) != 0) {
 25 |     printf("\n mutex init failed\n");
 26 |   }
 27 | 
 28 |   return assertion_handler;
 29 | }
 30 | 
 31 | /**
 32 |  * Inserts an assertion at the end of the assertion queue
 33 |  * @param assertion_handler assertion_handler_s to add the assertion
 34 |  * @param assertion         New assertion to add
 35 |  */
 36 | void assertion_handler_push_assertion(
 37 |     struct assertion_handler_s *assertion_handler,
 38 |     struct assertion_e *assertion) {
 39 | 
 40 |   pthread_mutex_lock(&assertion_handler->assertion_mutex);
 41 |   TAILQ_INSERT_TAIL(&assertion_handler->assertion_q, assertion, tailq);
 42 |   pthread_mutex_unlock(&assertion_handler->assertion_mutex);
 43 | }
 44 | 
 45 | /**
 46 |  * Inserts a new value at the end of the value queue
 47 |  * @param assertion_handler assertion_handler_s to add the assertion
 48 |  * @param value             New value to add
 49 |  */
 50 | void assertion_handler_push_value(struct assertion_handler_s *assertion_handler,
 51 |                                   struct value_e *value) {
 52 | 
 53 |   pthread_mutex_lock(&assertion_handler->value_mutex);
 54 |   TAILQ_INSERT_TAIL(&assertion_handler->value_q, value, tailq);
 55 |   pthread_mutex_unlock(&assertion_handler->value_mutex);
 56 | }
 57 | 
 58 | /**
 59 |  * Iterates the assertions and check for matchs between assertions and values
 60 |  * @param assertion_handler assertion_handler_s to use
 61 |  */
 62 | int assertion_handler_assert(struct assertion_handler_s *assertion_handler) {
 63 |   uint found = 0;
 64 |   int err = 0;
 65 | 
 66 |   pthread_mutex_lock(&assertion_handler->value_mutex);
 67 |   pthread_mutex_lock(&assertion_handler->assertion_mutex);
 68 | 
 69 |   struct assertion_e *assertion;
 70 |   TAILQ_FOREACH(assertion, &assertion_handler->assertion_q, tailq) {
 71 |     struct value_e *value;
 72 | 
 73 |     json_error_t assertion_jerr;
 74 |     char *assertion_message = NULL;
 75 | 
 76 |     json_t *assertion_json = json_loads(assertion->str, 0, &assertion_jerr);
 77 |     json_unpack_ex(assertion_json, &assertion_jerr, JSON_STRICT, "{s:s}",
 78 |                    "message", &assertion_message);
 79 | 
 80 |     found = 0;
 81 | 
 82 |     TAILQ_FOREACH(value, &assertion_handler->value_q, tailq) {
 83 |       json_error_t value_jerr;
 84 |       char *value_message = NULL;
 85 | 
 86 |       json_t *value_json = json_loads(value->str, 0, &value_jerr);
 87 |       json_unpack_ex(value_json, &value_jerr, JSON_STRICT, "{s:s}",
 88 |                               "message", &value_message);
 89 | 
 90 |       if (assertion_message == NULL || value_message == NULL) {
 91 |         return 1;
 92 |       }
 93 | 
 94 |       // printf("%s:%s -> %d\n", assertion_message, value_message,
 95 |       //        strcmp(assertion_message, value_message) == 0);
 96 | 
 97 |       if (strcmp(assertion_message, value_message) == 0) {
 98 |         found++;
 99 |         json_decref(value_json);
100 |         break;
101 |       }
102 | 
103 |       json_decref(value_json);
104 |     }
105 | 
106 |     json_decref(assertion_json);
107 | 
108 |     if (!found) {
109 |       err++;
110 |       break;
111 |     }
112 |   }
113 | 
114 |   pthread_mutex_unlock(&assertion_handler->assertion_mutex);
115 |   pthread_mutex_unlock(&assertion_handler->value_mutex);
116 | 
117 |   return err;
118 | }
119 | 
120 | void assertion_handler_destroy(struct assertion_handler_s *assertion_handler) {
121 |   pthread_mutex_lock(&assertion_handler->value_mutex);
122 |   pthread_mutex_lock(&assertion_handler->assertion_mutex);
123 | 
124 |   struct assertion_e *assertion = NULL;
125 |   while (!TAILQ_EMPTY(&assertion_handler->assertion_q)) {
126 |     assertion = TAILQ_FIRST(&assertion_handler->assertion_q);
127 |     TAILQ_REMOVE(&assertion_handler->assertion_q, assertion, tailq);
128 |     free(assertion->str);
129 |     free(assertion);
130 |   }
131 | 
132 |   struct value_e *value;
133 |   while (!TAILQ_EMPTY(&assertion_handler->value_q)) {
134 |     value = TAILQ_FIRST(&assertion_handler->value_q);
135 |     TAILQ_REMOVE(&assertion_handler->value_q, value, tailq);
136 |     free(value->str);
137 |     free(value);
138 |   }
139 | 
140 |   pthread_mutex_unlock(&assertion_handler->assertion_mutex);
141 |   pthread_mutex_unlock(&assertion_handler->value_mutex);
142 | 
143 |   pthread_mutex_destroy(&assertion_handler->value_mutex);
144 |   pthread_mutex_destroy(&assertion_handler->assertion_mutex);
145 | 
146 |   free(assertion_handler);
147 | }
148 | 


--------------------------------------------------------------------------------
/tests/assertion_handler.h:
--------------------------------------------------------------------------------
 1 | #include "stdio.h"
 2 | #include "stdlib.h"
 3 | 
 4 | #include <string.h>
 5 | #include <pthread.h>
 6 | #include <sys/queue.h>
 7 | 
 8 | // Public structs
 9 | struct assertion_e {
10 |   TAILQ_ENTRY(assertion_e) tailq;
11 |   char *str;
12 | };
13 | 
14 | struct value_e {
15 |   TAILQ_ENTRY(value_e) tailq;
16 |   char *str;
17 |   size_t len;
18 | };
19 | 
20 | // Private structs
21 | struct assertion_handler_s;
22 | 
23 | // Functions
24 | struct assertion_handler_s *assertion_handler_new();
25 | void assertion_handler_push_assertion(
26 |     struct assertion_handler_s *assertion_handler,
27 |     struct assertion_e *assertion);
28 | void assertion_handler_push_value(struct assertion_handler_s *assertion_handler,
29 |                                   struct value_e *value);
30 | int assertion_handler_assert(struct assertion_handler_s *assertion_handler);
31 | void assertion_handler_destroy(struct assertion_handler_s *assertion_handler);
32 | 


--------------------------------------------------------------------------------
/tests/display_coverage.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | COLUMNS=89
 4 | INPUT="coverage.out"
 5 | 
 6 | # echo "$INPUT"
 7 | BRANCH_COVERAGE=$(grep branches coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1)
 8 | LINES_COVERAGE=$(grep lines coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1)
 9 | FUNCTIONS_COVERAGE=$(grep functions coverage.out | cut -d ' ' -f 4 | cut -d '%' -f 1)
10 | 
11 | COVERAGE=$(echo "$BRANCH_COVERAGE" | cut -d '.' -f 1)
12 | if [ $COVERAGE -gt 89 ]; then
13 |   COLOR="\e[1m\e[32m"
14 | elif [ $COVERAGE -gt 69 ]; then
15 |   COLOR="\e[1m\e[33m"
16 | else
17 |   COLOR="\e[1m\e[31m"
18 | fi
19 | 
20 | TITLE="CODE COVERAGE"
21 | CONTENT=$(printf "\e[1mLINES: %s%% | FUNCTIONS: %s%% | BRANCH: %s%%\e[0m\n" "$LINES_COVERAGE"  "$FUNCTIONS_COVERAGE" "$BRANCH_COVERAGE")
22 | 
23 | printf "$COLOR=====================================================================================================\e[0m\n"
24 | printf "$COLOR%*s\e[0m\n" $(((${#TITLE}+$COLUMNS)/2)) "$TITLE"
25 | printf "$COLOR\n"
26 | printf "$COLOR%*s\e[0m\n" $(((${#CONTENT}+$COLUMNS)/2)) "$CONTENT"
27 | printf "$COLOR=====================================================================================================\e[0m\n"
28 | 


--------------------------------------------------------------------------------
/tests/rb_http2k_tests.c:
--------------------------------------------------------------------------------
 1 | #include "decoder/rb_http2k/rb_http2k_decoder.c"
 2 | #include "util/kafka.h"
 3 | 
 4 | #include <setjmp.h>
 5 | #include <cmocka.h>
 6 | 
 7 | struct message_in {
 8 | 	const char *msg;
 9 | 	size_t size;
10 | };
11 | 
12 | typedef void (*check_callback_fn)(struct rb_session **,void *opaque);
13 | 
14 | static void test_rb_decoder_setup(const char *config_txt) {
15 | 	static const char TEMP_TEMPLATE[] = "n2ktXXXXXX";
16 | 	init_global_config();
17 | 	char temp_filename[sizeof(TEMP_TEMPLATE)];
18 | 	strcpy(temp_filename,TEMP_TEMPLATE);
19 | 	int temp_fd = mkstemp(temp_filename);
20 | 	assert_true(temp_fd >= 0);
21 | 	write(temp_fd, config_txt, strlen(config_txt));
22 | 	parse_config(temp_filename);
23 | 	unlink(temp_filename);
24 | 	close(temp_fd);
25 | }
26 | 
27 | static void test_rb_decoder_teardown() {
28 | 	free_global_config();
29 | }
30 | 
31 | /** Template for rb_decoder test
32 | 	@param args Arguments like client_ip, topic, etc
33 | 	@param msgs Input messages
34 | 	@param msgs_len Length of msgs
35 | 	@param check_callback Array of functions that will be called with each
36 | 	session status. It is suppose to be the same length as msgs array.
37 | 	@param check_callback_opaque Opaque used in the second parameter of
38 | 	check_callback[iteration] call
39 | 	*/
40 | static void test_rb_decoder0(const char *config_str, keyval_list_t *args,
41 | 		struct message_in *msgs, check_callback_fn *check_callback,
42 | 		size_t msgs_len, void *check_callback_opaque) {
43 | 	size_t i;
44 | 
45 | 	test_rb_decoder_setup(config_str);
46 | 
47 | 	struct rb_opaque rb_opaque = {
48 | #ifdef RB_OPAQUE_MAGIC
49 | 		.magic = RB_OPAQUE_MAGIC,
50 | #endif
51 | 		.rb_config = &global_config.rb,
52 | 	};
53 | 
54 | 	struct rb_session *my_session = NULL;
55 | 
56 | 	for(i=0;i<msgs_len;++i) {
57 | 		process_rb_buffer(msgs[i].msg, msgs[i].msg ? msgs[i].size : 0, args,
58 | 			&rb_opaque, &my_session);
59 | 		check_callback[i](&my_session,check_callback_opaque);
60 | 	}
61 | 
62 | 	test_rb_decoder_teardown();
63 | }
64 | 
65 | /** Function that check that session has no messages
66 | 	@param sess Session pointer
67 | 	@param unused context information
68 | */
69 | static void check_zero_messages(struct rb_session **sess,
70 |             void *unused __attribute__((unused))) __attribute__((unused));
71 | static void check_zero_messages(struct rb_session **sess,
72 |                     void *unused __attribute__((unused))) {
73 | 
74 | 	assert_true(NULL != sess);
75 | 	assert_true(NULL != *sess);
76 | 	assert_true(0==rd_kafka_msg_q_size(&(*sess)->msg_queue));
77 | }
78 | 
79 | /** This function just checks that session is NULL */
80 | static void check_null_session(struct rb_session **sess,
81 | 		void *unused __attribute__((unused)))
82 | 		__attribute__((unused));
83 | static void check_null_session(struct rb_session **sess,
84 |                     void *unused __attribute__((unused))) {
85 | 
86 | 	assert_true(NULL != sess);
87 | 	assert_true(NULL == *sess);
88 | }
89 | 


--------------------------------------------------------------------------------
/tests/rb_json_tests.c:
--------------------------------------------------------------------------------
  1 | // rb_json_tests.c
  2 | #include <stdarg.h>
  3 | #include <stddef.h>
  4 | #include <setjmp.h>
  5 | #include <cmocka.h>
  6 | #include <jansson.h>
  7 | #include <string.h>
  8 | #include <librdkafka/rdkafka.h>
  9 | 
 10 | typedef json_error_t rb_json_err_t;
 11 | 
 12 | /// you have to json_decref(return) when done
 13 | static void *rb_json_assert_unpack(const char *json, size_t flags,
 14 |                                    const char *fmt, ...) __attribute((unused));
 15 | static void *rb_json_assert_unpack(const char *json, size_t flags,
 16 |                                    const char *fmt, ...) {
 17 | 	json_error_t error;
 18 | 	json_t *root = json_loads(json, 0, &error);
 19 | 	if (root == NULL) {
 20 | 		fprintf(stderr, "[EROR PARSING JSON][%s][%s]\n", error.text, error.source);
 21 | 		assert_true(0);
 22 | 	}
 23 | 
 24 | 	va_list args;
 25 | 	va_start (args, fmt);
 26 | 
 27 | 	const int unpack_rc = json_vunpack_ex(root, &error, flags, fmt, args);
 28 | 
 29 | 	if (unpack_rc != 0 /* Failure */) {
 30 | 		fprintf(stderr, "[ERROR UNPACKING][%s][%s]\n", error.text, error.source);
 31 | 		assert_true(0);
 32 | 	}
 33 | 
 34 | 	va_end(args);
 35 | 
 36 | 	return root;
 37 | }
 38 | 
 39 | static void free_json_unpacked(void *mem) __attribute__((unused));
 40 | static void free_json_unpacked(void *mem) {
 41 | 	json_decref(mem);
 42 | }
 43 | 
 44 | static int str_equal(const char *str1,
 45 |                      const char *str2) __attribute__((unused));
 46 | static int str_equal(const char *str1, const char *str2) {
 47 | 	if ((str1 != NULL && str2 == NULL) || (str1 == NULL && str2 != NULL))
 48 | 		return 0;
 49 | 	if (str1 == NULL && str2 == NULL)
 50 | 		return 1;
 51 | 	return 0 == strcmp(str1, str2);
 52 | }
 53 | 
 54 | struct checkdata_value {
 55 | 	const char *key;
 56 | 	json_type type;
 57 | 	const char *value;
 58 | };
 59 | 
 60 | struct checkdata {
 61 | 	size_t size;
 62 | 	const struct checkdata_value *checks;
 63 | };
 64 | 
 65 | #define CHECKDATA(name,...) \
 66 | 	static const struct checkdata_value name##checks[] = { \
 67 | 		__VA_ARGS__ \
 68 | 	}; \
 69 | 	static const struct checkdata name = { \
 70 | 		.size = sizeof(name##checks) / sizeof(name##checks[0]), \
 71 | 		.checks = name##checks \
 72 | 	}
 73 | 
 74 | struct checkdata_array {
 75 | 	size_t size;
 76 | 	const struct checkdata **checks;
 77 | };
 78 | 
 79 | #define CHECKDATA_ARRAY(name,...) \
 80 | 	static const struct checkdata *name##elms[] = { \
 81 | 		__VA_ARGS__ \
 82 | 	}; \
 83 | 	static const struct checkdata_array name = { \
 84 | 		.checks = name##elms, \
 85 | 		.size = sizeof(name##elms) / sizeof(name##elms[0]), \
 86 | 	}
 87 | 
 88 | static void assert_trueEqual(const int64_t a, const int64_t b, const char *key,
 89 |                         const char *src) __attribute__((unused));
 90 | static void assert_trueEqual(const int64_t a, const int64_t b, const char *key,
 91 |                         const char *src) {
 92 | 	if (a != b) {
 93 | 		fprintf(stderr,
 94 | 		        "[%s integer value mismatch] Actual: %ld, Expected: %ld in %s\n",
 95 | 		        key, a, b, src);
 96 | 		assert_true(a == b);
 97 | 	}
 98 | }
 99 | 
100 | static void rb_assert_json_value(const struct checkdata_value *chk_value,
101 |                                  const json_t *json_value, const char *src)__attribute__((unused));
102 | static void rb_assert_json_value(const struct checkdata_value *chk_value,
103 |                                  const json_t *json_value, const char *src) {
104 | 	//assert_true(chk_value->type == json_typeof(json_value));
105 | 	if (chk_value->value == NULL && json_value == NULL) {
106 | 		return; // All ok
107 | 	}
108 | 
109 | 	if (chk_value->value == NULL && json_value != NULL) {
110 | 		fprintf(stderr, "Json key %s with value %s, should not exists in (%s)\n",
111 | 		        chk_value->key, json_string_value(json_value), src);
112 | 		assert_true(!json_value);
113 | 	}
114 | 
115 | 	if (NULL == json_value) {
116 | 		fprintf(stderr, "Json value %s does not exists in %s\n", chk_value->key, src);
117 | 		assert_true(json_value);
118 | 	}
119 | 	switch (json_typeof(json_value)) {
120 | 	case JSON_INTEGER: {
121 | 		const json_int_t json_int_value = json_integer_value(json_value);
122 | 		const long chk_int_value = atol(chk_value->value);
123 | 		assert_trueEqual(json_int_value, chk_int_value, chk_value->key, src);
124 | 	}
125 | 	break;
126 | 	case JSON_STRING: {
127 | 		const char *json_str_value = json_string_value(json_value);
128 | 		assert_true(0 == strcmp(json_str_value, chk_value->value));
129 | 	}
130 | 	break;
131 | 	default:
132 | 		assert_true(!"You should not be here");
133 | 	}
134 | }
135 | 
136 | static json_t *rb_assert_json_loadb(const char *buf, size_t buflen) {
137 | 	json_error_t error;
138 | 	json_t *root = json_loadb(buf, buflen, 0, &error);
139 | 
140 | 	if (root == NULL) {
141 | 		fprintf(stderr, "[EROR PARSING JSON][%s][%s]\n", error.text, error.source);
142 | 		assert_true(0);
143 | 	}
144 | 
145 | 	return root;
146 | }
147 | 
148 | static void rb_assert_json_n(const char *str, size_t sz,
149 |                              const struct checkdata *checkdata) __attribute__((unused));
150 | static void rb_assert_json_n(const char *str, size_t sz,
151 |                              const struct checkdata *checkdata) {
152 | 	size_t i = 0;
153 | 	json_t *root = rb_assert_json_loadb(str, sz);
154 | 
155 | 	for (i = 0; i < checkdata->size; ++i) {
156 | 		const json_t *json_value = json_object_get(root, checkdata->checks[i].key);
157 | 		rb_assert_json_value(&checkdata->checks[i], json_value, str);
158 | 	}
159 | 
160 | 	json_decref(root);
161 | }
162 | 
163 | static void rb_assert_json(const char *str,
164 |                            const struct checkdata *checkdata) __attribute__((unused));
165 | static void rb_assert_json(const char *str, const struct checkdata *checkdata) {
166 | 	rb_assert_json_n(str, strlen(str), checkdata);
167 | }
168 | 
169 | 
170 | static void rb_assert_json_array(const rd_kafka_message_t *msgs,
171 |                                  size_t msgs_size,
172 |                                  const struct checkdata_array *checkdata_array) __attribute__((unused));
173 | static void rb_assert_json_array(const rd_kafka_message_t *msgs,
174 |                                  size_t msgs_size,
175 |                                  const struct checkdata_array *checkdata_array) {
176 | 
177 | 	size_t i;
178 | 
179 | 	assert_true(msgs_size == checkdata_array->size);
180 | 	for (i = 0; i < checkdata_array->size; ++i) {
181 | 		size_t payload_size = msgs[i].len;
182 | 		rb_assert_json_n(msgs[i].payload, payload_size, checkdata_array->checks[i]);
183 | 	}
184 | }
185 | 


--------------------------------------------------------------------------------
/tests/rb_meraki_tests.h:
--------------------------------------------------------------------------------
 1 | #include "../src/decoder/meraki/rb_meraki.c"
 2 | #include "rb_json_tests.c"
 3 | 
 4 | #include <stdarg.h>
 5 | #include <stddef.h>
 6 | #include <setjmp.h>
 7 | #include <cmocka.h>
 8 | 
 9 | #define RB_UNUSED __attribute__((unused))
10 | 
11 | static void MerakiDecoder_test_base(const char *config_str, const char *secrets,
12 | 		const char *msg, const struct checkdata_array *checkdata) RB_UNUSED;
13 | static void MerakiDecoder_test_base(const char *config_str, const char *secrets,
14 | 		const char *msg, const struct checkdata_array *checkdata) {
15 | 	size_t i;
16 | 	const char *topic_name = NULL;
17 | 	json_error_t jerr;
18 | 	struct meraki_config meraki_config;
19 | 	struct meraki_decoder_info decoder_info;
20 | 	json_t *config = NULL;
21 | 
22 | 	memset(&meraki_config, 0, sizeof(meraki_config));
23 | 	init_meraki_database(&meraki_config.database);
24 | 
25 | 	meraki_decoder_info_create(&decoder_info);
26 | 
27 | 	if (config_str) {
28 | 		config = json_loads(config_str, 0, NULL);
29 | 		assert_true(config);
30 | 		parse_meraki_decoder_info(&decoder_info, &topic_name, config);
31 | 		assert_true(decoder_info.per_listener_enrichment);
32 | 	}
33 | 
34 | 	// Workaround
35 | 	decoder_info.meraki_config = &meraki_config;
36 | 
37 | 	json_t *meraki_secrets_array = json_loadb(secrets, strlen(secrets), 0,
38 | 									&jerr);
39 | 	assert_true(meraki_secrets_array);
40 | 
41 | 	const int parse_rc = parse_meraki_secrets(&meraki_config.database,
42 | 	                     meraki_secrets_array);
43 | 
44 | 	assert_true(parse_rc == 0);
45 | 	json_decref(meraki_secrets_array);
46 | 
47 | 	char *aux = strdup(msg);
48 | 	struct kafka_message_array *notifications_array = process_meraki_buffer(
49 | 		aux, strlen(msg), "127.0.0.1", &decoder_info);
50 | 	free(aux);
51 | 
52 | 	if (checkdata) {
53 | 		rb_assert_json_array(notifications_array->msgs,
54 | 		                     notifications_array->count, checkdata);
55 | 
56 | 		for (i = 0; i < notifications_array->count; ++i)
57 | 			free(notifications_array->msgs[i].payload);
58 | 		free(notifications_array);
59 | 	} else {
60 | 		assert_true(0==notifications_array);
61 | 	}
62 | 
63 | 	meraki_decoder_info_destructor(&decoder_info);
64 | 	if (config) {
65 | 		json_decref(config);
66 | 	}
67 | 	meraki_database_done(&meraki_config.database);
68 | }
69 | 


--------------------------------------------------------------------------------
/tests/rb_mse_tests.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "../src/decoder/mse/rb_mse.c"
 4 | 
 5 | #include <stdarg.h>
 6 | #include <stddef.h>
 7 | #include <setjmp.h>
 8 | #include <cmocka.h>
 9 | 
10 | /// @TODO keep in sync with testMSE10Decoder
11 | 
12 | static void testMSE10Decoder(const char *mse_array_str,
13 |                              const char *_listener_config,
14 |                              const char *mse_input,
15 |                              const time_t now,
16 |                              void (*check_result)(struct mse_array *))
17 | __attribute__((unused));
18 | static void testMSE10Decoder(const char *mse_array_str,
19 |                              const char *_listener_config,
20 |                              const char *mse_input,
21 |                              const time_t now,
22 |                              void (*check_result)(struct mse_array *)) {
23 | 	json_error_t jerr;
24 | 	size_t i;
25 | 	const char *topic_name;
26 | 	struct mse_config mse_config;
27 | 	struct mse_decoder_info decoder_info;
28 | 
29 | 	memset(&mse_config, 0, sizeof(mse_config));
30 | 	mse_decoder_info_create(&decoder_info);
31 | 
32 | 	json_t *listener_config = json_loads(_listener_config, 0, &jerr);
33 | 	const int opaque_creator_rc = parse_decoder_info(&decoder_info,
34 | 		listener_config,&topic_name);
35 | 
36 | 	assert_true(0 == opaque_creator_rc);
37 | 	json_decref(listener_config);
38 | 
39 | 	/* Currently, uses global_config */
40 | 	decoder_info.mse_config = &mse_config;
41 | 
42 | 	json_t *mse_array = json_loads(mse_array_str, 0, &jerr);
43 | 	assert_true(mse_array);
44 | 	const int parse_rc = parse_mse_array(&decoder_info.mse_config->database,
45 | 		mse_array);
46 | 	assert_true(parse_rc == 0);
47 | 
48 | 	char *aux = strdup(mse_input);
49 | 	struct mse_array *notifications_array = process_mse_buffer(
50 | 	        aux,
51 | 	        strlen(mse_input),
52 | 	        "127.0.0.1",
53 | 	        &decoder_info,
54 | 	        now);
55 | 
56 | 	check_result(notifications_array);
57 | 
58 | 	free(aux);
59 | 	for (i = 0; notifications_array && i < notifications_array->size; ++i)
60 | 		free(notifications_array->data[i].string);
61 | 
62 | 	free(notifications_array);
63 | 	mse_decoder_info_destroy(&decoder_info);
64 | 	free_valid_mse_database(&mse_config.database);
65 | 	json_decref(mse_array);
66 | }
67 | 
68 | #define testMSE8Decoder testMSE10Decoder // same function
69 | 


--------------------------------------------------------------------------------
/version.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | const char *n2kafka_revision;
4 | const char *n2kafka_version;
5 | 


--------------------------------------------------------------------------------