├── INSTALL.windows.md
├── LICENSE
├── README.md
├── patch
└── nxlog-ce-2.8.1248-kafka-output-module.patch
└── src
└── kafka
├── Makefile.am
├── Makefile.in
├── om_kafka.c
└── om_kafka.h
/INSTALL.windows.md:
--------------------------------------------------------------------------------
1 | Still unable to compile in windows using minGW, waiting on nxlog guys to lend a hand.
2 |
3 | In any case,
4 |
5 | Install MinGW
6 | Download apr, pcre, openssl, zlib
7 | Install (configure + make + make install)
8 |
9 | Download nxlog-ce
10 | ./configure works but make fails (see https://meocloud.pt/link/a36b6b4e-5949-4d58-841b-c73032ca7b04/nxlogmakeerrors.txt/)
11 |
12 | Will try to get back to this as soon as possible but it might be a few weeks...
13 |
14 | Drop me a message on freenode, user "nowan" if you want to tackle this
15 |
16 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | No copyrights!
2 | Any contents of this repository are free for all to change and reditribute.
3 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | nxlog-kafka-output-module
2 | =========================
3 |
4 | An output module for nxlog to write to kafka brokers using librdkafka
5 |
6 | # Foreword
7 | This was implemented in a couple of hours and it requires proper structure and testing.
8 | It was coded by example inspired on the other output modules provided.
9 | The author unfortunately hasn't enough time to properly develop this much needed module.
10 | The author asks of the nxlog developers to add and maintain the module in their great product.
11 | Other developers are welcome to keep this going, will gladly add any challenger to the contributors of this project.
12 |
13 | In any case it's functional and you can use it at your own risk.
14 |
15 | # Usage
16 |
17 | ## Requirements
18 | * The GNU toolchain
19 | * GNU make
20 | * [https://github.com/edenhill/librdkafka](librdkafka)
21 | * [http://nxlog-ce.sourceforge.net/] nxlog-ce, patch included for version 2.8.1248
22 |
23 | ## Instructions
24 |
25 |
26 | ### Building and installing
27 | #### *NIXes
28 | Download and install librdkafka
29 |
30 | git clone https://github.com/edenhill/librdkafka.git
31 | cd librdkafka
32 | ./configure
33 | make
34 | sudo make install
35 | cd ..
36 |
37 | Download and uncompress [http://sourceforge.net/projects/nxlog-ce/files/](nxlog-ce source code) - nxlog-ce-2.8.1248.tar.gz
38 |
39 | tar zxvf nxlog-ce-2.8.1248.tar.gz
40 | cd nxlog-ce-2.8.1248
41 |
42 | Download or clone nxlog-kafka-output-module from git
43 |
44 | git clone https://github.com/filipealmeida/nxlog-kafka-output-module.git
45 | patch -p0 -f < nxlog-kafka-output-module/patch/nxlog-ce-2.8.1248-kafka-output-module.patch
46 |
47 | Export libraries
48 |
49 | export LIBS="-lrdkafka -lrt -lz"
50 |
51 | Build and install nxlog with kafka output module
52 |
53 | ./configure
54 | make
55 | sudo make install
56 |
57 | #### Microsoft Windows
58 | Keep calm (brace yourselves) and refer to the INSTALL.windows.md file
59 |
60 | ### Sample nxlog configuration
61 |
62 | ########################################
63 | # Global directives #
64 | ########################################
65 | User nxlog
66 | Group nxlog
67 | LogFile /var/log/nxlog/nxlog.log
68 | LogLevel INFO
69 | ########################################
70 | # Modules #
71 | ########################################
72 |
73 | Module im_file
74 | File "/home/user/logs/myevents.log"
75 | SavePos TRUE
76 | Recursive TRUE
77 |
78 |
86 | ########################################
87 | # Routes #
88 | ########################################
89 |
90 | Path inFile => outKafka
91 |
92 |
--------------------------------------------------------------------------------
/patch/nxlog-ce-2.8.1248-kafka-output-module.patch:
--------------------------------------------------------------------------------
1 | diff -arupN ./configure ../nxlog-ce-2.8.1248.tmp/configure
2 | --- ./configure 2014-07-19 14:52:12.000000000 +0100
3 | +++ ../nxlog-ce-2.8.1248.tmp/configure 2014-10-20 11:13:42.000000000 +0100
4 | @@ -13880,7 +13880,7 @@ MAINTAINERCLEANFILES=Makefile.in
5 |
6 |
7 |
8 | -ac_config_files="$ac_config_files Makefile doc/Makefile doc/reference-manual/Makefile doc/reference-manual/config-examples/Makefile doc/reference-manual/en/Makefile doc/reference-manual/hu/Makefile test/Makefile test/common/Makefile test/coresrc/Makefile src/Makefile src/common/Makefile src/core/Makefile src/modules/Makefile src/modules/input/Makefile src/modules/input/file/Makefile src/modules/input/uds/Makefile src/modules/input/kernel/Makefile src/modules/input/udp/Makefile src/modules/input/tcp/Makefile src/modules/input/ssl/Makefile src/modules/input/testgen/Makefile src/modules/input/null/Makefile src/modules/input/mark/Makefile src/modules/input/internal/Makefile src/modules/input/dbi/Makefile src/modules/input/exec/Makefile src/modules/input/mseventlog/Makefile src/modules/input/msvistalog/Makefile src/modules/processor/Makefile src/modules/processor/null/Makefile src/modules/processor/filter/Makefile src/modules/processor/transformer/Makefile src/modules/processor/norepeat/Makefile src/modules/processor/buffer/Makefile src/modules/processor/blocker/Makefile src/modules/processor/pattern/Makefile src/modules/processor/evcorr/Makefile src/modules/output/Makefile src/modules/output/file/Makefile src/modules/output/udp/Makefile src/modules/output/tcp/Makefile src/modules/output/ssl/Makefile src/modules/output/null/Makefile src/modules/output/dbi/Makefile src/modules/output/blocker/Makefile src/modules/output/uds/Makefile src/modules/output/exec/Makefile src/modules/output/http/Makefile src/modules/extension/Makefile src/modules/extension/syslog/Makefile src/modules/extension/exec/Makefile src/modules/extension/csv/Makefile src/modules/extension/kvp/Makefile src/modules/extension/charconv/Makefile src/modules/extension/json/Makefile src/modules/extension/json/yajl/Makefile src/modules/extension/xml/Makefile src/modules/extension/gelf/Makefile src/modules/extension/fileop/Makefile src/modules/extension/multiline/Makefile src/modules/extension/perl/Makefile src/modules/extension/wtmp/Makefile src/utils/Makefile packaging/redhat/nxlog.spec packaging/redhat/nxlog.spec.RHEL5 packaging/windows/nxlog.wxs packaging/windows/pkgmsi.sh"
9 | +ac_config_files="$ac_config_files Makefile doc/Makefile doc/reference-manual/Makefile doc/reference-manual/config-examples/Makefile doc/reference-manual/en/Makefile doc/reference-manual/hu/Makefile test/Makefile test/common/Makefile test/coresrc/Makefile src/Makefile src/common/Makefile src/core/Makefile src/modules/Makefile src/modules/input/Makefile src/modules/input/file/Makefile src/modules/input/uds/Makefile src/modules/input/kernel/Makefile src/modules/input/udp/Makefile src/modules/input/tcp/Makefile src/modules/input/ssl/Makefile src/modules/input/testgen/Makefile src/modules/input/null/Makefile src/modules/input/mark/Makefile src/modules/input/internal/Makefile src/modules/input/dbi/Makefile src/modules/input/exec/Makefile src/modules/input/mseventlog/Makefile src/modules/input/msvistalog/Makefile src/modules/processor/Makefile src/modules/processor/null/Makefile src/modules/processor/filter/Makefile src/modules/processor/transformer/Makefile src/modules/processor/norepeat/Makefile src/modules/processor/buffer/Makefile src/modules/processor/blocker/Makefile src/modules/processor/pattern/Makefile src/modules/processor/evcorr/Makefile src/modules/output/Makefile src/modules/output/file/Makefile src/modules/output/udp/Makefile src/modules/output/tcp/Makefile src/modules/output/ssl/Makefile src/modules/output/null/Makefile src/modules/output/dbi/Makefile src/modules/output/blocker/Makefile src/modules/output/uds/Makefile src/modules/output/exec/Makefile src/modules/output/http/Makefile src/modules/output/kafka/Makefile src/modules/extension/Makefile src/modules/extension/syslog/Makefile src/modules/extension/exec/Makefile src/modules/extension/csv/Makefile src/modules/extension/kvp/Makefile src/modules/extension/charconv/Makefile src/modules/extension/json/Makefile src/modules/extension/json/yajl/Makefile src/modules/extension/xml/Makefile src/modules/extension/gelf/Makefile src/modules/extension/fileop/Makefile src/modules/extension/multiline/Makefile src/modules/extension/perl/Makefile src/modules/extension/wtmp/Makefile src/utils/Makefile packaging/redhat/nxlog.spec packaging/redhat/nxlog.spec.RHEL5 packaging/windows/nxlog.wxs packaging/windows/pkgmsi.sh"
10 |
11 | cat >confcache <<\_ACEOF
12 | # This file is a shell script that caches the results of configure
13 | diff -arupN ./src/modules/output/Makefile.am ../nxlog-ce-2.8.1248.tmp/src/modules/output/Makefile.am
14 | --- ./src/modules/output/Makefile.am 2014-07-19 14:52:07.000000000 +0100
15 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/Makefile.am 2014-10-20 10:44:36.000000000 +0100
16 | @@ -1 +1 @@
17 | -SUBDIRS = file udp tcp ssl null dbi blocker uds exec http
18 | +SUBDIRS = file udp tcp ssl null dbi blocker uds exec http kafka
19 | diff -arupN ./src/modules/output/Makefile.in ../nxlog-ce-2.8.1248.tmp/src/modules/output/Makefile.in
20 | --- ./src/modules/output/Makefile.in 2014-07-19 14:52:14.000000000 +0100
21 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/Makefile.in 2014-10-20 11:06:59.000000000 +0100
22 | @@ -237,7 +237,7 @@ target_vendor = @target_vendor@
23 | top_build_prefix = @top_build_prefix@
24 | top_builddir = @top_builddir@
25 | top_srcdir = @top_srcdir@
26 | -SUBDIRS = file udp tcp ssl null dbi blocker uds exec http
27 | +SUBDIRS = file udp tcp ssl null dbi blocker uds exec http kafka
28 | all: all-recursive
29 |
30 | .SUFFIXES:
31 | diff -arupN ./src/modules/output/kafka/Makefile.am ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/Makefile.am
32 | --- ./src/modules/output/kafka/Makefile.am 1970-01-01 01:00:00.000000000 +0100
33 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/Makefile.am 2014-10-20 10:43:41.000000000 +0100
34 | @@ -0,0 +1,4 @@
35 | +om_kafka_LTLIBRARIES = om_kafka.la
36 | +om_kafka_la_SOURCES = om_kafka.c om_kafka.h
37 | +om_kafka_la_LDFLAGS = -module -no-undefined -avoid-version
38 | +om_kafkadir = $(NX_MODULEDIR)/output
39 | diff -arupN ./src/modules/output/kafka/Makefile.in ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/Makefile.in
40 | --- ./src/modules/output/kafka/Makefile.in 1970-01-01 01:00:00.000000000 +0100
41 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/Makefile.in 2014-10-20 10:53:29.000000000 +0100
42 | @@ -0,0 +1,571 @@
43 | +# Makefile.in generated by automake 1.11.3 from Makefile.am.
44 | +# @configure_input@
45 | +
46 | +# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
47 | +# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
48 | +# Foundation, Inc.
49 | +# This Makefile.in is free software; the Free Software Foundation
50 | +# gives unlimited permission to copy and/or distribute it,
51 | +# with or without modifications, as long as this notice is preserved.
52 | +
53 | +# This program is distributed in the hope that it will be useful,
54 | +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
55 | +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
56 | +# PARTICULAR PURPOSE.
57 | +
58 | +@SET_MAKE@
59 | +
60 | +VPATH = @srcdir@
61 | +pkgdatadir = $(datadir)/@PACKAGE@
62 | +pkgincludedir = $(includedir)/@PACKAGE@
63 | +pkglibdir = $(libdir)/@PACKAGE@
64 | +pkglibexecdir = $(libexecdir)/@PACKAGE@
65 | +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
66 | +install_sh_DATA = $(install_sh) -c -m 644
67 | +install_sh_PROGRAM = $(install_sh) -c
68 | +install_sh_SCRIPT = $(install_sh) -c
69 | +INSTALL_HEADER = $(INSTALL_DATA)
70 | +transform = $(program_transform_name)
71 | +NORMAL_INSTALL = :
72 | +PRE_INSTALL = :
73 | +POST_INSTALL = :
74 | +NORMAL_UNINSTALL = :
75 | +PRE_UNINSTALL = :
76 | +POST_UNINSTALL = :
77 | +build_triplet = @build@
78 | +host_triplet = @host@
79 | +target_triplet = @target@
80 | +subdir = src/modules/output/kafka
81 | +DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
82 | +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
83 | +am__aclocal_m4_deps = $(top_srcdir)/configure.in
84 | +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
85 | + $(ACLOCAL_M4)
86 | +mkinstalldirs = $(install_sh) -d
87 | +CONFIG_HEADER = $(top_builddir)/src/common/config.h
88 | +CONFIG_CLEAN_FILES =
89 | +CONFIG_CLEAN_VPATH_FILES =
90 | +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
91 | +am__vpath_adj = case $$p in \
92 | + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
93 | + *) f=$$p;; \
94 | + esac;
95 | +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
96 | +am__install_max = 40
97 | +am__nobase_strip_setup = \
98 | + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
99 | +am__nobase_strip = \
100 | + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
101 | +am__nobase_list = $(am__nobase_strip_setup); \
102 | + for p in $$list; do echo "$$p $$p"; done | \
103 | + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
104 | + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
105 | + if (++n[$$2] == $(am__install_max)) \
106 | + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
107 | + END { for (dir in files) print dir, files[dir] }'
108 | +am__base_list = \
109 | + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
110 | + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
111 | +am__uninstall_files_from_dir = { \
112 | + test -z "$$files" \
113 | + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
114 | + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
115 | + $(am__cd) "$$dir" && rm -f $$files; }; \
116 | + }
117 | +am__installdirs = "$(DESTDIR)$(om_kafkadir)"
118 | +LTLIBRARIES = $(om_kafka_LTLIBRARIES)
119 | +am__DEPENDENCIES_1 =
120 | +om_kafka_la_DEPENDENCIES = $(am__DEPENDENCIES_1)
121 | +am_om_kafka_la_OBJECTS = om_kafka.lo
122 | +om_kafka_la_OBJECTS = $(am_om_kafka_la_OBJECTS)
123 | +om_kafka_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \
124 | + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
125 | + $(om_kafka_la_LDFLAGS) $(LDFLAGS) -o $@
126 | +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/src/common
127 | +depcomp = $(SHELL) $(top_srcdir)/depcomp
128 | +am__depfiles_maybe = depfiles
129 | +am__mv = mv -f
130 | +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
131 | + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
132 | +LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
133 | + --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
134 | + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
135 | +CCLD = $(CC)
136 | +LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
137 | + --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
138 | + $(LDFLAGS) -o $@
139 | +SOURCES = $(om_kafka_la_SOURCES)
140 | +DIST_SOURCES = $(om_kafka_la_SOURCES)
141 | +ETAGS = etags
142 | +CTAGS = ctags
143 | +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
144 | +ACLOCAL = @ACLOCAL@
145 | +AMTAR = @AMTAR@
146 | +APRCONFIG = @APRCONFIG@
147 | +APR_CFLAGS = @APR_CFLAGS@
148 | +APR_LIBS = @APR_LIBS@
149 | +AR = @AR@
150 | +AS = @AS@
151 | +AUTOCONF = @AUTOCONF@
152 | +AUTOHEADER = @AUTOHEADER@
153 | +AUTOMAKE = @AUTOMAKE@
154 | +AWK = @AWK@
155 | +CC = @CC@
156 | +CCDEPMODE = @CCDEPMODE@
157 | +CFLAGS = @CFLAGS@
158 | +CLEANFILES = @CLEANFILES@
159 | +CPP = @CPP@
160 | +CPPFLAGS = @CPPFLAGS@
161 | +CYGPATH_W = @CYGPATH_W@
162 | +DEFS = @DEFS@
163 | +DEPDIR = @DEPDIR@
164 | +DLLTOOL = @DLLTOOL@
165 | +DSYMUTIL = @DSYMUTIL@
166 | +DUMPBIN = @DUMPBIN@
167 | +ECHO_C = @ECHO_C@
168 | +ECHO_N = @ECHO_N@
169 | +ECHO_T = @ECHO_T@
170 | +EGREP = @EGREP@
171 | +EXEEXT = @EXEEXT@
172 | +FGREP = @FGREP@
173 | +GREP = @GREP@
174 | +INCLUDES = @INCLUDES@
175 | +INSTALL = @INSTALL@
176 | +INSTALL_DATA = @INSTALL_DATA@
177 | +INSTALL_PROGRAM = @INSTALL_PROGRAM@
178 | +INSTALL_SCRIPT = @INSTALL_SCRIPT@
179 | +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
180 | +LD = @LD@
181 | +LDFLAGS = @LDFLAGS@
182 | +LEX = @LEX@
183 | +LEXLIB = @LEXLIB@
184 | +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
185 | +LIBCAP = @LIBCAP@
186 | +LIBDBI = @LIBDBI@
187 | +LIBEXPAT = @LIBEXPAT@
188 | +LIBICONV = @LIBICONV@
189 | +LIBNX = @LIBNX@
190 | +LIBOBJS = @LIBOBJS@
191 | +LIBS = @LIBS@
192 | +LIBSSL = @LIBSSL@
193 | +LIBTOOL = @LIBTOOL@
194 | +LIBZLIB = @LIBZLIB@
195 | +LIPO = @LIPO@
196 | +LN_S = @LN_S@
197 | +LTLIBOBJS = @LTLIBOBJS@
198 | +MAINTAINERCLEANFILES = @MAINTAINERCLEANFILES@
199 | +MAKEINFO = @MAKEINFO@
200 | +MANIFEST_TOOL = @MANIFEST_TOOL@
201 | +MKDIR_P = @MKDIR_P@
202 | +NM = @NM@
203 | +NMEDIT = @NMEDIT@
204 | +NX_CACHEDIR = @NX_CACHEDIR@
205 | +NX_CONFIGFILE = @NX_CONFIGFILE@
206 | +NX_MODULEDIR = @NX_MODULEDIR@
207 | +NX_PIDFILE = @NX_PIDFILE@
208 | +OBJDUMP = @OBJDUMP@
209 | +OBJEXT = @OBJEXT@
210 | +OPENSSL = @OPENSSL@
211 | +OTOOL = @OTOOL@
212 | +OTOOL64 = @OTOOL64@
213 | +PACKAGE = @PACKAGE@
214 | +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
215 | +PACKAGE_NAME = @PACKAGE_NAME@
216 | +PACKAGE_STRING = @PACKAGE_STRING@
217 | +PACKAGE_TARNAME = @PACKAGE_TARNAME@
218 | +PACKAGE_URL = @PACKAGE_URL@
219 | +PACKAGE_VERSION = @PACKAGE_VERSION@
220 | +PATH_SEPARATOR = @PATH_SEPARATOR@
221 | +PCRECONFIG = @PCRECONFIG@
222 | +PCRE_CFLAGS = @PCRE_CFLAGS@
223 | +PCRE_LIBS = @PCRE_LIBS@
224 | +PERL = @PERL@
225 | +PERL_CFLAGS = @PERL_CFLAGS@
226 | +PERL_LIBS = @PERL_LIBS@
227 | +RANLIB = @RANLIB@
228 | +SED = @SED@
229 | +SET_MAKE = @SET_MAKE@
230 | +SHELL = @SHELL@
231 | +STRIP = @STRIP@
232 | +VERSION = @VERSION@
233 | +WIN32 = @WIN32@
234 | +YACC = @YACC@
235 | +YFLAGS = @YFLAGS@
236 | +abs_builddir = @abs_builddir@
237 | +abs_srcdir = @abs_srcdir@
238 | +abs_top_builddir = @abs_top_builddir@
239 | +abs_top_srcdir = @abs_top_srcdir@
240 | +ac_ct_AR = @ac_ct_AR@
241 | +ac_ct_CC = @ac_ct_CC@
242 | +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
243 | +am__include = @am__include@
244 | +am__leading_dot = @am__leading_dot@
245 | +am__quote = @am__quote@
246 | +am__tar = @am__tar@
247 | +am__untar = @am__untar@
248 | +apr_config = @apr_config@
249 | +bindir = @bindir@
250 | +build = @build@
251 | +build_alias = @build_alias@
252 | +build_cpu = @build_cpu@
253 | +build_os = @build_os@
254 | +build_vendor = @build_vendor@
255 | +builddir = @builddir@
256 | +datadir = @datadir@
257 | +datarootdir = @datarootdir@
258 | +docdir = @docdir@
259 | +dvidir = @dvidir@
260 | +exec_prefix = @exec_prefix@
261 | +host = @host@
262 | +host_alias = @host_alias@
263 | +host_cpu = @host_cpu@
264 | +host_os = @host_os@
265 | +host_vendor = @host_vendor@
266 | +htmldir = @htmldir@
267 | +includedir = @includedir@
268 | +infodir = @infodir@
269 | +install_sh = @install_sh@
270 | +libdir = @libdir@
271 | +libexecdir = @libexecdir@
272 | +localedir = @localedir@
273 | +localstatedir = @localstatedir@
274 | +mandir = @mandir@
275 | +mkdir_p = @mkdir_p@
276 | +oldincludedir = @oldincludedir@
277 | +pcre_config = @pcre_config@
278 | +pdfdir = @pdfdir@
279 | +prefix = @prefix@
280 | +program_transform_name = @program_transform_name@
281 | +psdir = @psdir@
282 | +sbindir = @sbindir@
283 | +sharedstatedir = @sharedstatedir@
284 | +srcdir = @srcdir@
285 | +sysconfdir = @sysconfdir@
286 | +target = @target@
287 | +target_alias = @target_alias@
288 | +target_cpu = @target_cpu@
289 | +target_os = @target_os@
290 | +target_vendor = @target_vendor@
291 | +top_build_prefix = @top_build_prefix@
292 | +top_builddir = @top_builddir@
293 | +top_srcdir = @top_srcdir@
294 | +om_kafka_LTLIBRARIES = om_kafka.la
295 | +om_kafka_la_SOURCES = om_kafka.c om_kafka.h
296 | +om_kafka_la_LDFLAGS = -module -no-undefined -avoid-version
297 | +om_kafka_la_LIBADD = $(LIBNX)
298 | +om_kafkadir = $(NX_MODULEDIR)/output
299 | +all: all-am
300 | +
301 | +.SUFFIXES:
302 | +.SUFFIXES: .c .lo .o .obj
303 | +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
304 | + @for dep in $?; do \
305 | + case '$(am__configure_deps)' in \
306 | + *$$dep*) \
307 | + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
308 | + && { if test -f $@; then exit 0; else break; fi; }; \
309 | + exit 1;; \
310 | + esac; \
311 | + done; \
312 | + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/modules/output/kafka/Makefile'; \
313 | + $(am__cd) $(top_srcdir) && \
314 | + $(AUTOMAKE) --foreign src/modules/output/kafka/Makefile
315 | +.PRECIOUS: Makefile
316 | +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
317 | + @case '$?' in \
318 | + *config.status*) \
319 | + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
320 | + *) \
321 | + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
322 | + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
323 | + esac;
324 | +
325 | +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
326 | + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
327 | +
328 | +$(top_srcdir)/configure: $(am__configure_deps)
329 | + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
330 | +$(ACLOCAL_M4): $(am__aclocal_m4_deps)
331 | + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
332 | +$(am__aclocal_m4_deps):
333 | +install-om_kafkaLTLIBRARIES: $(om_kafka_LTLIBRARIES)
334 | + @$(NORMAL_INSTALL)
335 | + test -z "$(om_kafkadir)" || $(MKDIR_P) "$(DESTDIR)$(om_kafkadir)"
336 | + @list='$(om_kafka_LTLIBRARIES)'; test -n "$(om_kafkadir)" || list=; \
337 | + list2=; for p in $$list; do \
338 | + if test -f $$p; then \
339 | + list2="$$list2 $$p"; \
340 | + else :; fi; \
341 | + done; \
342 | + test -z "$$list2" || { \
343 | + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(om_kafkadir)'"; \
344 | + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(om_kafkadir)"; \
345 | + }
346 | +
347 | +uninstall-om_kafkaLTLIBRARIES:
348 | + @$(NORMAL_UNINSTALL)
349 | + @list='$(om_kafka_LTLIBRARIES)'; test -n "$(om_kafkadir)" || list=; \
350 | + for p in $$list; do \
351 | + $(am__strip_dir) \
352 | + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(om_kafkadir)/$$f'"; \
353 | + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(om_kafkadir)/$$f"; \
354 | + done
355 | +
356 | +clean-om_kafkaLTLIBRARIES:
357 | + -test -z "$(om_kafka_LTLIBRARIES)" || rm -f $(om_kafka_LTLIBRARIES)
358 | + @list='$(om_kafka_LTLIBRARIES)'; for p in $$list; do \
359 | + dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
360 | + test "$$dir" != "$$p" || dir=.; \
361 | + echo "rm -f \"$${dir}/so_locations\""; \
362 | + rm -f "$${dir}/so_locations"; \
363 | + done
364 | +om_kafka.la: $(om_kafka_la_OBJECTS) $(om_kafka_la_DEPENDENCIES) $(EXTRA_om_kafka_la_DEPENDENCIES)
365 | + $(om_kafka_la_LINK) -rpath $(om_kafkadir) $(om_kafka_la_OBJECTS) $(om_kafka_la_LIBADD) $(LIBS)
366 | +
367 | +mostlyclean-compile:
368 | + -rm -f *.$(OBJEXT)
369 | +
370 | +distclean-compile:
371 | + -rm -f *.tab.c
372 | +
373 | +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/om_kafka.Plo@am__quote@
374 | +
375 | +.c.o:
376 | +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
377 | +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
378 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
379 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
380 | +@am__fastdepCC_FALSE@ $(COMPILE) -c $<
381 | +
382 | +.c.obj:
383 | +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
384 | +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
385 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
386 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
387 | +@am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'`
388 | +
389 | +.c.lo:
390 | +@am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
391 | +@am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
392 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
393 | +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
394 | +@am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $<
395 | +
396 | +mostlyclean-libtool:
397 | + -rm -f *.lo
398 | +
399 | +clean-libtool:
400 | + -rm -rf .libs _libs
401 | +
402 | +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
403 | + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
404 | + unique=`for i in $$list; do \
405 | + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
406 | + done | \
407 | + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
408 | + END { if (nonempty) { for (i in files) print i; }; }'`; \
409 | + mkid -fID $$unique
410 | +tags: TAGS
411 | +
412 | +TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
413 | + $(TAGS_FILES) $(LISP)
414 | + set x; \
415 | + here=`pwd`; \
416 | + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
417 | + unique=`for i in $$list; do \
418 | + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
419 | + done | \
420 | + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
421 | + END { if (nonempty) { for (i in files) print i; }; }'`; \
422 | + shift; \
423 | + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
424 | + test -n "$$unique" || unique=$$empty_fix; \
425 | + if test $$# -gt 0; then \
426 | + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
427 | + "$$@" $$unique; \
428 | + else \
429 | + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
430 | + $$unique; \
431 | + fi; \
432 | + fi
433 | +ctags: CTAGS
434 | +CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
435 | + $(TAGS_FILES) $(LISP)
436 | + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
437 | + unique=`for i in $$list; do \
438 | + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
439 | + done | \
440 | + $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
441 | + END { if (nonempty) { for (i in files) print i; }; }'`; \
442 | + test -z "$(CTAGS_ARGS)$$unique" \
443 | + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
444 | + $$unique
445 | +
446 | +GTAGS:
447 | + here=`$(am__cd) $(top_builddir) && pwd` \
448 | + && $(am__cd) $(top_srcdir) \
449 | + && gtags -i $(GTAGS_ARGS) "$$here"
450 | +
451 | +distclean-tags:
452 | + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
453 | +
454 | +distdir: $(DISTFILES)
455 | + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
456 | + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
457 | + list='$(DISTFILES)'; \
458 | + dist_files=`for file in $$list; do echo $$file; done | \
459 | + sed -e "s|^$$srcdirstrip/||;t" \
460 | + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
461 | + case $$dist_files in \
462 | + */*) $(MKDIR_P) `echo "$$dist_files" | \
463 | + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
464 | + sort -u` ;; \
465 | + esac; \
466 | + for file in $$dist_files; do \
467 | + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
468 | + if test -d $$d/$$file; then \
469 | + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
470 | + if test -d "$(distdir)/$$file"; then \
471 | + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
472 | + fi; \
473 | + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
474 | + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
475 | + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
476 | + fi; \
477 | + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
478 | + else \
479 | + test -f "$(distdir)/$$file" \
480 | + || cp -p $$d/$$file "$(distdir)/$$file" \
481 | + || exit 1; \
482 | + fi; \
483 | + done
484 | +check-am: all-am
485 | +check: check-am
486 | +all-am: Makefile $(LTLIBRARIES)
487 | +installdirs:
488 | + for dir in "$(DESTDIR)$(om_kafkadir)"; do \
489 | + test -z "$$dir" || $(MKDIR_P) "$$dir"; \
490 | + done
491 | +install: install-am
492 | +install-exec: install-exec-am
493 | +install-data: install-data-am
494 | +uninstall: uninstall-am
495 | +
496 | +install-am: all-am
497 | + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
498 | +
499 | +installcheck: installcheck-am
500 | +install-strip:
501 | + if test -z '$(STRIP)'; then \
502 | + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
503 | + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
504 | + install; \
505 | + else \
506 | + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
507 | + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
508 | + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
509 | + fi
510 | +mostlyclean-generic:
511 | +
512 | +clean-generic:
513 | + -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
514 | +
515 | +distclean-generic:
516 | + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
517 | + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
518 | +
519 | +maintainer-clean-generic:
520 | + @echo "This command is intended for maintainers to use"
521 | + @echo "it deletes files that may require special tools to rebuild."
522 | + -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES)
523 | +clean: clean-am
524 | +
525 | +clean-am: clean-generic clean-libtool clean-om_kafkaLTLIBRARIES \
526 | + mostlyclean-am
527 | +
528 | +distclean: distclean-am
529 | + -rm -rf ./$(DEPDIR)
530 | + -rm -f Makefile
531 | +distclean-am: clean-am distclean-compile distclean-generic \
532 | + distclean-tags
533 | +
534 | +dvi: dvi-am
535 | +
536 | +dvi-am:
537 | +
538 | +html: html-am
539 | +
540 | +html-am:
541 | +
542 | +info: info-am
543 | +
544 | +info-am:
545 | +
546 | +install-data-am: install-om_kafkaLTLIBRARIES
547 | +
548 | +install-dvi: install-dvi-am
549 | +
550 | +install-dvi-am:
551 | +
552 | +install-exec-am:
553 | +
554 | +install-html: install-html-am
555 | +
556 | +install-html-am:
557 | +
558 | +install-info: install-info-am
559 | +
560 | +install-info-am:
561 | +
562 | +install-man:
563 | +
564 | +install-pdf: install-pdf-am
565 | +
566 | +install-pdf-am:
567 | +
568 | +install-ps: install-ps-am
569 | +
570 | +install-ps-am:
571 | +
572 | +installcheck-am:
573 | +
574 | +maintainer-clean: maintainer-clean-am
575 | + -rm -rf ./$(DEPDIR)
576 | + -rm -f Makefile
577 | +maintainer-clean-am: distclean-am maintainer-clean-generic
578 | +
579 | +mostlyclean: mostlyclean-am
580 | +
581 | +mostlyclean-am: mostlyclean-compile mostlyclean-generic \
582 | + mostlyclean-libtool
583 | +
584 | +pdf: pdf-am
585 | +
586 | +pdf-am:
587 | +
588 | +ps: ps-am
589 | +
590 | +ps-am:
591 | +
592 | +uninstall-am: uninstall-om_kafkaLTLIBRARIES
593 | +
594 | +.MAKE: install-am install-strip
595 | +
596 | +.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
597 | + clean-libtool clean-om_kafkaLTLIBRARIES ctags distclean \
598 | + distclean-compile distclean-generic distclean-libtool \
599 | + distclean-tags distdir dvi dvi-am html html-am info info-am \
600 | + install install-am install-data install-data-am install-dvi \
601 | + install-dvi-am install-exec install-exec-am install-html \
602 | + install-html-am install-info install-info-am install-man \
603 | + install-om_kafkaLTLIBRARIES install-pdf install-pdf-am \
604 | + install-ps install-ps-am install-strip installcheck \
605 | + installcheck-am installdirs maintainer-clean \
606 | + maintainer-clean-generic mostlyclean mostlyclean-compile \
607 | + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
608 | + tags uninstall uninstall-am uninstall-om_kafkaLTLIBRARIES
609 | +
610 | +
611 | +# Tell versions [3.59,3.63) of GNU make to not export all variables.
612 | +# Otherwise a system limit (for SysV at least) may be exceeded.
613 | +.NOEXPORT:
614 | diff -arupN ./src/modules/output/kafka/om_kafka.c ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/om_kafka.c
615 | --- ./src/modules/output/kafka/om_kafka.c 1970-01-01 01:00:00.000000000 +0100
616 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/om_kafka.c 2014-10-20 12:02:45.000000000 +0100
617 | @@ -0,0 +1,202 @@
618 | +/*
619 | + * Free as in the oxygen you breathe, do whatever you like with this, at your own risk
620 | + * Author: Filipe Almeida
621 | + */
622 | +
623 | +#include "../../../common/module.h"
624 | +#include "../../../common/event.h"
625 | +#include "../../../common/error_debug.h"
626 | +#include "../../../common/alloc.h"
627 | +#include "om_kafka.h"
628 | +
629 | +#include
630 | +
631 | +#define NX_LOGMODULE NX_LOGMODULE_MODULE
632 | +
633 | +static void om_kafka_start(nx_module_t *module) {
634 | + log_debug("Kafka module start entrypoint");
635 | + //TODO: consolidate/merge with init
636 | +}
637 | +
638 | +static void om_kafka_stop(nx_module_t *module) {
639 | + log_debug("Kafka module stop entrypoint");
640 | + ASSERT(module != NULL);
641 | + //TODO: cleanup
642 | +}
643 | +
644 | +static void om_kafka_config(nx_module_t *module) {
645 | + const nx_directive_t* volatile curr;
646 | + nx_om_kafka_conf_t* volatile modconf;
647 | + nx_exception_t e;
648 | + curr = module->directives;
649 | + modconf = apr_pcalloc(module->pool, sizeof(nx_om_kafka_conf_t));
650 | + module->config = modconf;
651 | + modconf->partition = RD_KAFKA_PARTITION_UA;
652 | +
653 | + while (curr != NULL) {
654 | + if (nx_module_common_keyword(curr->directive) == TRUE) { //ignore common configuration keywords
655 | +
656 | + } else if (strcasecmp(curr->directive, "brokerlist") == 0) {
657 | + modconf->brokerlist = curr->args;
658 | + } else if (strcasecmp(curr->directive, "topic") == 0) {
659 | + modconf->topic = curr->args;
660 | + } else if (strcasecmp(curr->directive, "compression") == 0) {
661 | + modconf->compression = curr->args;
662 | + } else if (strcasecmp(curr->directive, "partition") == 0) {
663 | + modconf->partition = atoi(curr->args);
664 | + } else {
665 | + log_warn("Kafka output module ignored directive \"%s\" set to \"%s\"\n", curr->directive, curr->args);
666 | + }
667 | + curr = curr->next;
668 | + }
669 | +
670 | +}
671 | +
672 | +static void om_kafka_init(nx_module_t *module) {
673 | + log_debug("Kafka module init entrypoint");
674 | + char errstr[512];
675 | + nx_om_kafka_conf_t* modconf;
676 | + modconf = (nx_om_kafka_conf_t*) module->config;
677 | +
678 | + rd_kafka_conf_t *conf;
679 | + rd_kafka_topic_conf_t *topic_conf;
680 | + /* Kafka configuration */
681 | + conf = rd_kafka_conf_new();
682 | + /* Topic configuration */
683 | + topic_conf = rd_kafka_topic_conf_new();
684 | +
685 | + rd_kafka_conf_set_dr_cb(conf, msg_delivered);
686 | +
687 | + if (rd_kafka_conf_set(conf, "compression.codec", modconf->compression, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
688 | + log_error("Unable to set compression codec %s", modconf->compression);
689 | + } else {
690 | + log_info("Kafka compression set to %s", modconf->compression);
691 | + }
692 | +
693 | + if (!(modconf->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) {
694 | + log_error("Failed to create new producer: %s\n", errstr);
695 | + }
696 | +
697 | + if (rd_kafka_brokers_add(modconf->rk, modconf->brokerlist) == 0) {
698 | + log_error("No valid brokers specified (%s)", modconf->brokerlist);
699 | + } else {
700 | + log_info("Kafka brokers set to %s", modconf->brokerlist);
701 | + }
702 | +
703 | + modconf->rkt = rd_kafka_topic_new(modconf->rk, modconf->topic, topic_conf);
704 | +
705 | + modconf->kafka_conf = conf;
706 | + modconf->topic_conf = topic_conf;
707 | +}
708 | +
709 | +/**
710 | + * Message delivery report callback.
711 | + * Called once for each message.
712 | + * See rdkafka.h for more information.
713 | + */
714 | +static void msg_delivered (rd_kafka_t *rk, void *payload, size_t len, int error_code, void *opaque, void *msg_opaque) {
715 | + if (error_code) {
716 | + log_error("Message delivery failed: %s\n", rd_kafka_err2str(error_code));
717 | + } else {
718 | + //log_debug(stderr, "Message delivered (%zd bytes)\n", len);
719 | + log_debug("Message delivered (%zd bytes)\n", len);
720 | + }
721 | +}
722 | +
723 | +static void om_kafka_write(nx_module_t *module) {
724 | + nx_om_kafka_conf_t* modconf;
725 | + modconf = (nx_om_kafka_conf_t*) module->config;
726 | + nx_logdata_t *logdata;
727 | + if (nx_module_get_status(module) != NX_MODULE_STATUS_RUNNING) {
728 | + log_warn("Kafka module not running.");
729 | + return;
730 | + }
731 | +
732 | + if (module->output.buflen == 0) {
733 | + if ((logdata = nx_module_logqueue_peek(module)) != NULL) {
734 | + module->output.logdata = logdata;
735 | + if (rd_kafka_produce(modconf->rkt, modconf->partition, RD_KAFKA_MSG_F_COPY,
736 | + /* Payload and length */
737 | + logdata->raw_event->buf, (int) logdata->raw_event->len,
738 | + /* Optional key and its length */
739 | + NULL, 0,
740 | + /* Message opaque, provided in delivery report callback as msg_opaque. */
741 | + NULL) == -1) {
742 | + log_error("Unable to produce message");
743 | + rd_kafka_poll(modconf->rk, 0);
744 | + } else {
745 | + //TODO: report on message
746 | + log_debug("Message sent");
747 | + rd_kafka_poll(modconf->rk, 0);
748 | + nx_module_logqueue_pop(module, module->output.logdata);
749 | + nx_logdata_free(module->output.logdata);
750 | + module->output.logdata = NULL;
751 | + }
752 | + }
753 | + }
754 | +}
755 | +
756 | +static void om_kafka_event(nx_module_t *module, nx_event_t *event) {
757 | + log_debug("Kafka module event entrypoint");
758 | + nx_exception_t e;
759 | + ASSERT(module != NULL);
760 | + ASSERT(event != NULL);
761 | +
762 | + nx_om_kafka_conf_t* modconf;
763 | + modconf = (nx_om_kafka_conf_t*) module->config;
764 | + switch (event->type) {
765 | + case NX_EVENT_DATA_AVAILABLE:
766 | + log_debug("Output buflen: %d, bufstart: %d", (int) module->output.buflen, (int) module->output.bufstart);
767 | + try {
768 | + om_kafka_write(module);
769 | + } catch (e) {
770 | + io_err_handler(module, &e);
771 | + }
772 | + break;
773 | + case NX_EVENT_READ:
774 | + break;
775 | + case NX_EVENT_WRITE:
776 | + break;
777 | + case NX_EVENT_RECONNECT:
778 | + break;
779 | + case NX_EVENT_DISCONNECT:
780 | + break;
781 | + case NX_EVENT_TIMEOUT:
782 | + break;
783 | + case NX_EVENT_POLL:
784 | + if (nx_module_get_status(module) == NX_MODULE_STATUS_RUNNING) {
785 | + nx_module_pollset_poll(module, FALSE);
786 | + }
787 | + break;
788 | + default:
789 | + nx_panic("invalid event type: %d", event->type);
790 | + }
791 | +}
792 | +
793 | +static void io_err_handler(nx_module_t *module, nx_exception_t *e) NORETURN;
794 | +static void io_err_handler(nx_module_t *module, nx_exception_t *e)
795 | +{
796 | + ASSERT(e != NULL);
797 | + ASSERT(module != NULL);
798 | +
799 | + nx_module_stop_self(module);
800 | + om_kafka_stop(module);
801 | + rethrow(*e);
802 | +}
803 | +
804 | +NX_MODULE_DECLARATION nx_om_kafka_module =
805 | +{
806 | + NX_MODULE_API_VERSION,
807 | + NX_MODULE_TYPE_OUTPUT,
808 | + NULL, // capabilities
809 | + om_kafka_config, // config
810 | + om_kafka_start, // start
811 | + om_kafka_stop, // stop
812 | + NULL, // pause
813 | + NULL, // resume
814 | + om_kafka_init, // init
815 | + NULL, // shutdown
816 | + om_kafka_event, // event
817 | + NULL, // info
818 | + NULL, // exports
819 | +};
820 | diff -arupN ./src/modules/output/kafka/om_kafka.h ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/om_kafka.h
821 | --- ./src/modules/output/kafka/om_kafka.h 1970-01-01 01:00:00.000000000 +0100
822 | +++ ../nxlog-ce-2.8.1248.tmp/src/modules/output/kafka/om_kafka.h 2014-10-20 12:02:55.000000000 +0100
823 | @@ -0,0 +1,28 @@
824 | +/*
825 | + * Free as in the oxygen you breathe, do whatever you like with this, at your own risk
826 | + * Author: Filipe Almeida
827 | + */
828 | +
829 | +#ifndef __NX_OM_KAFKA_H
830 | +#define __NX_OM_KAFKA_H
831 | +
832 | +#include "../../../common/types.h"
833 | +#include
834 | +
835 | +static void msg_delivered (rd_kafka_t *rk, void *payload, size_t len, int error_code, void *opaque, void *msg_opaque);
836 | +static void io_err_handler(nx_module_t *module, nx_exception_t *e);
837 | +
838 | +typedef struct nx_om_kafka_conf_t {
839 | + char* brokerlist;
840 | + char* topic;
841 | + char* compression;
842 | + int partition;
843 | + rd_kafka_conf_t *kafka_conf;
844 | + rd_kafka_topic_conf_t *topic_conf;
845 | + rd_kafka_t *rk;
846 | + rd_kafka_topic_t *rkt;
847 | + nx_logdata_t *logdata;
848 | +} nx_om_kafka_conf_t;
849 | +
850 | +
851 | +#endif /* __NX_OM_KAFKA_H */
852 |
--------------------------------------------------------------------------------
/src/kafka/Makefile.am:
--------------------------------------------------------------------------------
1 | om_kafka_LTLIBRARIES = om_kafka.la
2 | om_kafka_la_SOURCES = om_kafka.c om_kafka.h
3 | om_kafka_la_LDFLAGS = -module -no-undefined -avoid-version
4 | om_kafkadir = $(NX_MODULEDIR)/output
5 |
--------------------------------------------------------------------------------
/src/kafka/Makefile.in:
--------------------------------------------------------------------------------
1 | # Makefile.in generated by automake 1.11.3 from Makefile.am.
2 | # @configure_input@
3 |
4 | # Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
5 | # 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
6 | # Foundation, Inc.
7 | # This Makefile.in is free software; the Free Software Foundation
8 | # gives unlimited permission to copy and/or distribute it,
9 | # with or without modifications, as long as this notice is preserved.
10 |
11 | # This program is distributed in the hope that it will be useful,
12 | # but WITHOUT ANY WARRANTY, to the extent permitted by law; without
13 | # even the implied warranty of MERCHANTABILITY or FITNESS FOR A
14 | # PARTICULAR PURPOSE.
15 |
16 | @SET_MAKE@
17 |
18 | VPATH = @srcdir@
19 | pkgdatadir = $(datadir)/@PACKAGE@
20 | pkgincludedir = $(includedir)/@PACKAGE@
21 | pkglibdir = $(libdir)/@PACKAGE@
22 | pkglibexecdir = $(libexecdir)/@PACKAGE@
23 | am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
24 | install_sh_DATA = $(install_sh) -c -m 644
25 | install_sh_PROGRAM = $(install_sh) -c
26 | install_sh_SCRIPT = $(install_sh) -c
27 | INSTALL_HEADER = $(INSTALL_DATA)
28 | transform = $(program_transform_name)
29 | NORMAL_INSTALL = :
30 | PRE_INSTALL = :
31 | POST_INSTALL = :
32 | NORMAL_UNINSTALL = :
33 | PRE_UNINSTALL = :
34 | POST_UNINSTALL = :
35 | build_triplet = @build@
36 | host_triplet = @host@
37 | target_triplet = @target@
38 | subdir = src/modules/output/kafka
39 | DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
40 | ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
41 | am__aclocal_m4_deps = $(top_srcdir)/configure.in
42 | am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
43 | $(ACLOCAL_M4)
44 | mkinstalldirs = $(install_sh) -d
45 | CONFIG_HEADER = $(top_builddir)/src/common/config.h
46 | CONFIG_CLEAN_FILES =
47 | CONFIG_CLEAN_VPATH_FILES =
48 | am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
49 | am__vpath_adj = case $$p in \
50 | $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
51 | *) f=$$p;; \
52 | esac;
53 | am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
54 | am__install_max = 40
55 | am__nobase_strip_setup = \
56 | srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
57 | am__nobase_strip = \
58 | for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
59 | am__nobase_list = $(am__nobase_strip_setup); \
60 | for p in $$list; do echo "$$p $$p"; done | \
61 | sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
62 | $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
63 | if (++n[$$2] == $(am__install_max)) \
64 | { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
65 | END { for (dir in files) print dir, files[dir] }'
66 | am__base_list = \
67 | sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
68 | sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
69 | am__uninstall_files_from_dir = { \
70 | test -z "$$files" \
71 | || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
72 | || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
73 | $(am__cd) "$$dir" && rm -f $$files; }; \
74 | }
75 | am__installdirs = "$(DESTDIR)$(om_kafkadir)"
76 | LTLIBRARIES = $(om_kafka_LTLIBRARIES)
77 | am__DEPENDENCIES_1 =
78 | om_kafka_la_DEPENDENCIES = $(am__DEPENDENCIES_1)
79 | am_om_kafka_la_OBJECTS = om_kafka.lo
80 | om_kafka_la_OBJECTS = $(am_om_kafka_la_OBJECTS)
81 | om_kafka_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \
82 | $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
83 | $(om_kafka_la_LDFLAGS) $(LDFLAGS) -o $@
84 | DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/src/common
85 | depcomp = $(SHELL) $(top_srcdir)/depcomp
86 | am__depfiles_maybe = depfiles
87 | am__mv = mv -f
88 | COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
89 | $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
90 | LTCOMPILE = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
91 | --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
92 | $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
93 | CCLD = $(CC)
94 | LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
95 | --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \
96 | $(LDFLAGS) -o $@
97 | SOURCES = $(om_kafka_la_SOURCES)
98 | DIST_SOURCES = $(om_kafka_la_SOURCES)
99 | ETAGS = etags
100 | CTAGS = ctags
101 | DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
102 | ACLOCAL = @ACLOCAL@
103 | AMTAR = @AMTAR@
104 | APRCONFIG = @APRCONFIG@
105 | APR_CFLAGS = @APR_CFLAGS@
106 | APR_LIBS = @APR_LIBS@
107 | AR = @AR@
108 | AS = @AS@
109 | AUTOCONF = @AUTOCONF@
110 | AUTOHEADER = @AUTOHEADER@
111 | AUTOMAKE = @AUTOMAKE@
112 | AWK = @AWK@
113 | CC = @CC@
114 | CCDEPMODE = @CCDEPMODE@
115 | CFLAGS = @CFLAGS@
116 | CLEANFILES = @CLEANFILES@
117 | CPP = @CPP@
118 | CPPFLAGS = @CPPFLAGS@
119 | CYGPATH_W = @CYGPATH_W@
120 | DEFS = @DEFS@
121 | DEPDIR = @DEPDIR@
122 | DLLTOOL = @DLLTOOL@
123 | DSYMUTIL = @DSYMUTIL@
124 | DUMPBIN = @DUMPBIN@
125 | ECHO_C = @ECHO_C@
126 | ECHO_N = @ECHO_N@
127 | ECHO_T = @ECHO_T@
128 | EGREP = @EGREP@
129 | EXEEXT = @EXEEXT@
130 | FGREP = @FGREP@
131 | GREP = @GREP@
132 | INCLUDES = @INCLUDES@
133 | INSTALL = @INSTALL@
134 | INSTALL_DATA = @INSTALL_DATA@
135 | INSTALL_PROGRAM = @INSTALL_PROGRAM@
136 | INSTALL_SCRIPT = @INSTALL_SCRIPT@
137 | INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
138 | LD = @LD@
139 | LDFLAGS = @LDFLAGS@
140 | LEX = @LEX@
141 | LEXLIB = @LEXLIB@
142 | LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
143 | LIBCAP = @LIBCAP@
144 | LIBDBI = @LIBDBI@
145 | LIBEXPAT = @LIBEXPAT@
146 | LIBICONV = @LIBICONV@
147 | LIBNX = @LIBNX@
148 | LIBOBJS = @LIBOBJS@
149 | LIBS = @LIBS@
150 | LIBSSL = @LIBSSL@
151 | LIBTOOL = @LIBTOOL@
152 | LIBZLIB = @LIBZLIB@
153 | LIPO = @LIPO@
154 | LN_S = @LN_S@
155 | LTLIBOBJS = @LTLIBOBJS@
156 | MAINTAINERCLEANFILES = @MAINTAINERCLEANFILES@
157 | MAKEINFO = @MAKEINFO@
158 | MANIFEST_TOOL = @MANIFEST_TOOL@
159 | MKDIR_P = @MKDIR_P@
160 | NM = @NM@
161 | NMEDIT = @NMEDIT@
162 | NX_CACHEDIR = @NX_CACHEDIR@
163 | NX_CONFIGFILE = @NX_CONFIGFILE@
164 | NX_MODULEDIR = @NX_MODULEDIR@
165 | NX_PIDFILE = @NX_PIDFILE@
166 | OBJDUMP = @OBJDUMP@
167 | OBJEXT = @OBJEXT@
168 | OPENSSL = @OPENSSL@
169 | OTOOL = @OTOOL@
170 | OTOOL64 = @OTOOL64@
171 | PACKAGE = @PACKAGE@
172 | PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
173 | PACKAGE_NAME = @PACKAGE_NAME@
174 | PACKAGE_STRING = @PACKAGE_STRING@
175 | PACKAGE_TARNAME = @PACKAGE_TARNAME@
176 | PACKAGE_URL = @PACKAGE_URL@
177 | PACKAGE_VERSION = @PACKAGE_VERSION@
178 | PATH_SEPARATOR = @PATH_SEPARATOR@
179 | PCRECONFIG = @PCRECONFIG@
180 | PCRE_CFLAGS = @PCRE_CFLAGS@
181 | PCRE_LIBS = @PCRE_LIBS@
182 | PERL = @PERL@
183 | PERL_CFLAGS = @PERL_CFLAGS@
184 | PERL_LIBS = @PERL_LIBS@
185 | RANLIB = @RANLIB@
186 | SED = @SED@
187 | SET_MAKE = @SET_MAKE@
188 | SHELL = @SHELL@
189 | STRIP = @STRIP@
190 | VERSION = @VERSION@
191 | WIN32 = @WIN32@
192 | YACC = @YACC@
193 | YFLAGS = @YFLAGS@
194 | abs_builddir = @abs_builddir@
195 | abs_srcdir = @abs_srcdir@
196 | abs_top_builddir = @abs_top_builddir@
197 | abs_top_srcdir = @abs_top_srcdir@
198 | ac_ct_AR = @ac_ct_AR@
199 | ac_ct_CC = @ac_ct_CC@
200 | ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
201 | am__include = @am__include@
202 | am__leading_dot = @am__leading_dot@
203 | am__quote = @am__quote@
204 | am__tar = @am__tar@
205 | am__untar = @am__untar@
206 | apr_config = @apr_config@
207 | bindir = @bindir@
208 | build = @build@
209 | build_alias = @build_alias@
210 | build_cpu = @build_cpu@
211 | build_os = @build_os@
212 | build_vendor = @build_vendor@
213 | builddir = @builddir@
214 | datadir = @datadir@
215 | datarootdir = @datarootdir@
216 | docdir = @docdir@
217 | dvidir = @dvidir@
218 | exec_prefix = @exec_prefix@
219 | host = @host@
220 | host_alias = @host_alias@
221 | host_cpu = @host_cpu@
222 | host_os = @host_os@
223 | host_vendor = @host_vendor@
224 | htmldir = @htmldir@
225 | includedir = @includedir@
226 | infodir = @infodir@
227 | install_sh = @install_sh@
228 | libdir = @libdir@
229 | libexecdir = @libexecdir@
230 | localedir = @localedir@
231 | localstatedir = @localstatedir@
232 | mandir = @mandir@
233 | mkdir_p = @mkdir_p@
234 | oldincludedir = @oldincludedir@
235 | pcre_config = @pcre_config@
236 | pdfdir = @pdfdir@
237 | prefix = @prefix@
238 | program_transform_name = @program_transform_name@
239 | psdir = @psdir@
240 | sbindir = @sbindir@
241 | sharedstatedir = @sharedstatedir@
242 | srcdir = @srcdir@
243 | sysconfdir = @sysconfdir@
244 | target = @target@
245 | target_alias = @target_alias@
246 | target_cpu = @target_cpu@
247 | target_os = @target_os@
248 | target_vendor = @target_vendor@
249 | top_build_prefix = @top_build_prefix@
250 | top_builddir = @top_builddir@
251 | top_srcdir = @top_srcdir@
252 | om_kafka_LTLIBRARIES = om_kafka.la
253 | om_kafka_la_SOURCES = om_kafka.c om_kafka.h
254 | om_kafka_la_LDFLAGS = -module -no-undefined -avoid-version
255 | om_kafka_la_LIBADD = $(LIBNX)
256 | om_kafkadir = $(NX_MODULEDIR)/output
257 | all: all-am
258 |
259 | .SUFFIXES:
260 | .SUFFIXES: .c .lo .o .obj
261 | $(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
262 | @for dep in $?; do \
263 | case '$(am__configure_deps)' in \
264 | *$$dep*) \
265 | ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
266 | && { if test -f $@; then exit 0; else break; fi; }; \
267 | exit 1;; \
268 | esac; \
269 | done; \
270 | echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/modules/output/kafka/Makefile'; \
271 | $(am__cd) $(top_srcdir) && \
272 | $(AUTOMAKE) --foreign src/modules/output/kafka/Makefile
273 | .PRECIOUS: Makefile
274 | Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
275 | @case '$?' in \
276 | *config.status*) \
277 | cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
278 | *) \
279 | echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
280 | cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
281 | esac;
282 |
283 | $(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
284 | cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
285 |
286 | $(top_srcdir)/configure: $(am__configure_deps)
287 | cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
288 | $(ACLOCAL_M4): $(am__aclocal_m4_deps)
289 | cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
290 | $(am__aclocal_m4_deps):
291 | install-om_kafkaLTLIBRARIES: $(om_kafka_LTLIBRARIES)
292 | @$(NORMAL_INSTALL)
293 | test -z "$(om_kafkadir)" || $(MKDIR_P) "$(DESTDIR)$(om_kafkadir)"
294 | @list='$(om_kafka_LTLIBRARIES)'; test -n "$(om_kafkadir)" || list=; \
295 | list2=; for p in $$list; do \
296 | if test -f $$p; then \
297 | list2="$$list2 $$p"; \
298 | else :; fi; \
299 | done; \
300 | test -z "$$list2" || { \
301 | echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(om_kafkadir)'"; \
302 | $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(om_kafkadir)"; \
303 | }
304 |
305 | uninstall-om_kafkaLTLIBRARIES:
306 | @$(NORMAL_UNINSTALL)
307 | @list='$(om_kafka_LTLIBRARIES)'; test -n "$(om_kafkadir)" || list=; \
308 | for p in $$list; do \
309 | $(am__strip_dir) \
310 | echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(om_kafkadir)/$$f'"; \
311 | $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(om_kafkadir)/$$f"; \
312 | done
313 |
314 | clean-om_kafkaLTLIBRARIES:
315 | -test -z "$(om_kafka_LTLIBRARIES)" || rm -f $(om_kafka_LTLIBRARIES)
316 | @list='$(om_kafka_LTLIBRARIES)'; for p in $$list; do \
317 | dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
318 | test "$$dir" != "$$p" || dir=.; \
319 | echo "rm -f \"$${dir}/so_locations\""; \
320 | rm -f "$${dir}/so_locations"; \
321 | done
322 | om_kafka.la: $(om_kafka_la_OBJECTS) $(om_kafka_la_DEPENDENCIES) $(EXTRA_om_kafka_la_DEPENDENCIES)
323 | $(om_kafka_la_LINK) -rpath $(om_kafkadir) $(om_kafka_la_OBJECTS) $(om_kafka_la_LIBADD) $(LIBS)
324 |
325 | mostlyclean-compile:
326 | -rm -f *.$(OBJEXT)
327 |
328 | distclean-compile:
329 | -rm -f *.tab.c
330 |
331 | @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/om_kafka.Plo@am__quote@
332 |
333 | .c.o:
334 | @am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
335 | @am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
336 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
337 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
338 | @am__fastdepCC_FALSE@ $(COMPILE) -c $<
339 |
340 | .c.obj:
341 | @am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
342 | @am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
343 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
344 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
345 | @am__fastdepCC_FALSE@ $(COMPILE) -c `$(CYGPATH_W) '$<'`
346 |
347 | .c.lo:
348 | @am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
349 | @am__fastdepCC_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
350 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
351 | @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
352 | @am__fastdepCC_FALSE@ $(LTCOMPILE) -c -o $@ $<
353 |
354 | mostlyclean-libtool:
355 | -rm -f *.lo
356 |
357 | clean-libtool:
358 | -rm -rf .libs _libs
359 |
360 | ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
361 | list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
362 | unique=`for i in $$list; do \
363 | if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
364 | done | \
365 | $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
366 | END { if (nonempty) { for (i in files) print i; }; }'`; \
367 | mkid -fID $$unique
368 | tags: TAGS
369 |
370 | TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
371 | $(TAGS_FILES) $(LISP)
372 | set x; \
373 | here=`pwd`; \
374 | list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
375 | unique=`for i in $$list; do \
376 | if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
377 | done | \
378 | $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
379 | END { if (nonempty) { for (i in files) print i; }; }'`; \
380 | shift; \
381 | if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
382 | test -n "$$unique" || unique=$$empty_fix; \
383 | if test $$# -gt 0; then \
384 | $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
385 | "$$@" $$unique; \
386 | else \
387 | $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
388 | $$unique; \
389 | fi; \
390 | fi
391 | ctags: CTAGS
392 | CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
393 | $(TAGS_FILES) $(LISP)
394 | list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
395 | unique=`for i in $$list; do \
396 | if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
397 | done | \
398 | $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
399 | END { if (nonempty) { for (i in files) print i; }; }'`; \
400 | test -z "$(CTAGS_ARGS)$$unique" \
401 | || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
402 | $$unique
403 |
404 | GTAGS:
405 | here=`$(am__cd) $(top_builddir) && pwd` \
406 | && $(am__cd) $(top_srcdir) \
407 | && gtags -i $(GTAGS_ARGS) "$$here"
408 |
409 | distclean-tags:
410 | -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
411 |
412 | distdir: $(DISTFILES)
413 | @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
414 | topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
415 | list='$(DISTFILES)'; \
416 | dist_files=`for file in $$list; do echo $$file; done | \
417 | sed -e "s|^$$srcdirstrip/||;t" \
418 | -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
419 | case $$dist_files in \
420 | */*) $(MKDIR_P) `echo "$$dist_files" | \
421 | sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
422 | sort -u` ;; \
423 | esac; \
424 | for file in $$dist_files; do \
425 | if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
426 | if test -d $$d/$$file; then \
427 | dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
428 | if test -d "$(distdir)/$$file"; then \
429 | find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
430 | fi; \
431 | if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
432 | cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
433 | find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
434 | fi; \
435 | cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
436 | else \
437 | test -f "$(distdir)/$$file" \
438 | || cp -p $$d/$$file "$(distdir)/$$file" \
439 | || exit 1; \
440 | fi; \
441 | done
442 | check-am: all-am
443 | check: check-am
444 | all-am: Makefile $(LTLIBRARIES)
445 | installdirs:
446 | for dir in "$(DESTDIR)$(om_kafkadir)"; do \
447 | test -z "$$dir" || $(MKDIR_P) "$$dir"; \
448 | done
449 | install: install-am
450 | install-exec: install-exec-am
451 | install-data: install-data-am
452 | uninstall: uninstall-am
453 |
454 | install-am: all-am
455 | @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
456 |
457 | installcheck: installcheck-am
458 | install-strip:
459 | if test -z '$(STRIP)'; then \
460 | $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
461 | install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
462 | install; \
463 | else \
464 | $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
465 | install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
466 | "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
467 | fi
468 | mostlyclean-generic:
469 |
470 | clean-generic:
471 | -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
472 |
473 | distclean-generic:
474 | -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
475 | -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
476 |
477 | maintainer-clean-generic:
478 | @echo "This command is intended for maintainers to use"
479 | @echo "it deletes files that may require special tools to rebuild."
480 | -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES)
481 | clean: clean-am
482 |
483 | clean-am: clean-generic clean-libtool clean-om_kafkaLTLIBRARIES \
484 | mostlyclean-am
485 |
486 | distclean: distclean-am
487 | -rm -rf ./$(DEPDIR)
488 | -rm -f Makefile
489 | distclean-am: clean-am distclean-compile distclean-generic \
490 | distclean-tags
491 |
492 | dvi: dvi-am
493 |
494 | dvi-am:
495 |
496 | html: html-am
497 |
498 | html-am:
499 |
500 | info: info-am
501 |
502 | info-am:
503 |
504 | install-data-am: install-om_kafkaLTLIBRARIES
505 |
506 | install-dvi: install-dvi-am
507 |
508 | install-dvi-am:
509 |
510 | install-exec-am:
511 |
512 | install-html: install-html-am
513 |
514 | install-html-am:
515 |
516 | install-info: install-info-am
517 |
518 | install-info-am:
519 |
520 | install-man:
521 |
522 | install-pdf: install-pdf-am
523 |
524 | install-pdf-am:
525 |
526 | install-ps: install-ps-am
527 |
528 | install-ps-am:
529 |
530 | installcheck-am:
531 |
532 | maintainer-clean: maintainer-clean-am
533 | -rm -rf ./$(DEPDIR)
534 | -rm -f Makefile
535 | maintainer-clean-am: distclean-am maintainer-clean-generic
536 |
537 | mostlyclean: mostlyclean-am
538 |
539 | mostlyclean-am: mostlyclean-compile mostlyclean-generic \
540 | mostlyclean-libtool
541 |
542 | pdf: pdf-am
543 |
544 | pdf-am:
545 |
546 | ps: ps-am
547 |
548 | ps-am:
549 |
550 | uninstall-am: uninstall-om_kafkaLTLIBRARIES
551 |
552 | .MAKE: install-am install-strip
553 |
554 | .PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
555 | clean-libtool clean-om_kafkaLTLIBRARIES ctags distclean \
556 | distclean-compile distclean-generic distclean-libtool \
557 | distclean-tags distdir dvi dvi-am html html-am info info-am \
558 | install install-am install-data install-data-am install-dvi \
559 | install-dvi-am install-exec install-exec-am install-html \
560 | install-html-am install-info install-info-am install-man \
561 | install-om_kafkaLTLIBRARIES install-pdf install-pdf-am \
562 | install-ps install-ps-am install-strip installcheck \
563 | installcheck-am installdirs maintainer-clean \
564 | maintainer-clean-generic mostlyclean mostlyclean-compile \
565 | mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
566 | tags uninstall uninstall-am uninstall-om_kafkaLTLIBRARIES
567 |
568 |
569 | # Tell versions [3.59,3.63) of GNU make to not export all variables.
570 | # Otherwise a system limit (for SysV at least) may be exceeded.
571 | .NOEXPORT:
572 |
--------------------------------------------------------------------------------
/src/kafka/om_kafka.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Free as in the oxygen you breathe, do whatever you like with this, at your own risk
3 | * Author: Filipe Almeida
4 | */
5 |
6 | #include "../../../common/module.h"
7 | #include "../../../common/event.h"
8 | #include "../../../common/error_debug.h"
9 | #include "../../../common/alloc.h"
10 | #include "om_kafka.h"
11 |
12 | #include
13 |
14 | #define NX_LOGMODULE NX_LOGMODULE_MODULE
15 |
16 | static void om_kafka_start(nx_module_t *module) {
17 | log_debug("Kafka module start entrypoint");
18 | //TODO: consolidate/merge with init
19 | }
20 |
21 | static void om_kafka_stop(nx_module_t *module) {
22 | log_debug("Kafka module stop entrypoint");
23 | ASSERT(module != NULL);
24 | //TODO: cleanup
25 | }
26 |
27 | static void om_kafka_config(nx_module_t *module) {
28 | const nx_directive_t* volatile curr;
29 | nx_om_kafka_conf_t* volatile modconf;
30 | nx_exception_t e;
31 | curr = module->directives;
32 | modconf = apr_pcalloc(module->pool, sizeof(nx_om_kafka_conf_t));
33 | module->config = modconf;
34 | modconf->partition = RD_KAFKA_PARTITION_UA;
35 |
36 | while (curr != NULL) {
37 | if (nx_module_common_keyword(curr->directive) == TRUE) { //ignore common configuration keywords
38 |
39 | } else if (strcasecmp(curr->directive, "brokerlist") == 0) {
40 | modconf->brokerlist = curr->args;
41 | } else if (strcasecmp(curr->directive, "topic") == 0) {
42 | modconf->topic = curr->args;
43 | } else if (strcasecmp(curr->directive, "compression") == 0) {
44 | modconf->compression = curr->args;
45 | } else if (strcasecmp(curr->directive, "partition") == 0) {
46 | modconf->partition = atoi(curr->args);
47 | } else {
48 | log_warn("Kafka output module ignored directive \"%s\" set to \"%s\"\n", curr->directive, curr->args);
49 | }
50 | curr = curr->next;
51 | }
52 |
53 | }
54 |
55 | static void om_kafka_init(nx_module_t *module) {
56 | log_debug("Kafka module init entrypoint");
57 | char errstr[512];
58 | nx_om_kafka_conf_t* modconf;
59 | modconf = (nx_om_kafka_conf_t*) module->config;
60 |
61 | rd_kafka_conf_t *conf;
62 | rd_kafka_topic_conf_t *topic_conf;
63 | /* Kafka configuration */
64 | conf = rd_kafka_conf_new();
65 | /* Topic configuration */
66 | topic_conf = rd_kafka_topic_conf_new();
67 |
68 | rd_kafka_conf_set_dr_cb(conf, msg_delivered);
69 |
70 | if (rd_kafka_conf_set(conf, "compression.codec", modconf->compression, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
71 | log_error("Unable to set compression codec %s", modconf->compression);
72 | } else {
73 | log_info("Kafka compression set to %s", modconf->compression);
74 | }
75 |
76 | if (!(modconf->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)))) {
77 | log_error("Failed to create new producer: %s\n", errstr);
78 | }
79 |
80 | if (rd_kafka_brokers_add(modconf->rk, modconf->brokerlist) == 0) {
81 | log_error("No valid brokers specified (%s)", modconf->brokerlist);
82 | } else {
83 | log_info("Kafka brokers set to %s", modconf->brokerlist);
84 | }
85 |
86 | modconf->rkt = rd_kafka_topic_new(modconf->rk, modconf->topic, topic_conf);
87 |
88 | modconf->kafka_conf = conf;
89 | modconf->topic_conf = topic_conf;
90 | }
91 |
92 | /**
93 | * Message delivery report callback.
94 | * Called once for each message.
95 | * See rdkafka.h for more information.
96 | */
97 | static void msg_delivered (rd_kafka_t *rk, void *payload, size_t len, int error_code, void *opaque, void *msg_opaque) {
98 | if (error_code) {
99 | log_error("Message delivery failed: %s\n", rd_kafka_err2str(error_code));
100 | } else {
101 | //log_debug(stderr, "Message delivered (%zd bytes)\n", len);
102 | log_debug("Message delivered (%zd bytes)\n", len);
103 | }
104 | }
105 |
106 | static void om_kafka_write(nx_module_t *module) {
107 | nx_om_kafka_conf_t* modconf;
108 | modconf = (nx_om_kafka_conf_t*) module->config;
109 | nx_logdata_t *logdata;
110 | if (nx_module_get_status(module) != NX_MODULE_STATUS_RUNNING) {
111 | log_warn("Kafka module not running.");
112 | return;
113 | }
114 |
115 | if (module->output.buflen == 0) {
116 | if ((logdata = nx_module_logqueue_peek(module)) != NULL) {
117 | module->output.logdata = logdata;
118 | if (rd_kafka_produce(modconf->rkt, modconf->partition, RD_KAFKA_MSG_F_COPY,
119 | /* Payload and length */
120 | logdata->raw_event->buf, (int) logdata->raw_event->len,
121 | /* Optional key and its length */
122 | NULL, 0,
123 | /* Message opaque, provided in delivery report callback as msg_opaque. */
124 | NULL) == -1) {
125 | log_error("Unable to produce message");
126 | rd_kafka_poll(modconf->rk, 0);
127 | } else {
128 | //TODO: report on message
129 | log_debug("Message sent");
130 | rd_kafka_poll(modconf->rk, 0);
131 | nx_module_logqueue_pop(module, module->output.logdata);
132 | nx_logdata_free(module->output.logdata);
133 | module->output.logdata = NULL;
134 | }
135 | }
136 | }
137 | }
138 |
139 | static void om_kafka_event(nx_module_t *module, nx_event_t *event) {
140 | log_debug("Kafka module event entrypoint");
141 | nx_exception_t e;
142 | ASSERT(module != NULL);
143 | ASSERT(event != NULL);
144 |
145 | nx_om_kafka_conf_t* modconf;
146 | modconf = (nx_om_kafka_conf_t*) module->config;
147 | switch (event->type) {
148 | case NX_EVENT_DATA_AVAILABLE:
149 | log_debug("Output buflen: %d, bufstart: %d", (int) module->output.buflen, (int) module->output.bufstart);
150 | try {
151 | om_kafka_write(module);
152 | } catch (e) {
153 | io_err_handler(module, &e);
154 | }
155 | break;
156 | case NX_EVENT_READ:
157 | break;
158 | case NX_EVENT_WRITE:
159 | break;
160 | case NX_EVENT_RECONNECT:
161 | break;
162 | case NX_EVENT_DISCONNECT:
163 | break;
164 | case NX_EVENT_TIMEOUT:
165 | break;
166 | case NX_EVENT_POLL:
167 | if (nx_module_get_status(module) == NX_MODULE_STATUS_RUNNING) {
168 | nx_module_pollset_poll(module, FALSE);
169 | }
170 | break;
171 | default:
172 | nx_panic("invalid event type: %d", event->type);
173 | }
174 | }
175 |
176 | static void io_err_handler(nx_module_t *module, nx_exception_t *e) NORETURN;
177 | static void io_err_handler(nx_module_t *module, nx_exception_t *e)
178 | {
179 | ASSERT(e != NULL);
180 | ASSERT(module != NULL);
181 |
182 | nx_module_stop_self(module);
183 | om_kafka_stop(module);
184 | rethrow(*e);
185 | }
186 |
187 | NX_MODULE_DECLARATION nx_om_kafka_module =
188 | {
189 | NX_MODULE_API_VERSION,
190 | NX_MODULE_TYPE_OUTPUT,
191 | NULL, // capabilities
192 | om_kafka_config, // config
193 | om_kafka_start, // start
194 | om_kafka_stop, // stop
195 | NULL, // pause
196 | NULL, // resume
197 | om_kafka_init, // init
198 | NULL, // shutdown
199 | om_kafka_event, // event
200 | NULL, // info
201 | NULL, // exports
202 | };
203 |
--------------------------------------------------------------------------------
/src/kafka/om_kafka.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Free as in the oxygen you breathe, do whatever you like with this, at your own risk
3 | * Author: Filipe Almeida
4 | */
5 |
6 | #ifndef __NX_OM_KAFKA_H
7 | #define __NX_OM_KAFKA_H
8 |
9 | #include "../../../common/types.h"
10 | #include
11 |
12 | static void msg_delivered (rd_kafka_t *rk, void *payload, size_t len, int error_code, void *opaque, void *msg_opaque);
13 | static void io_err_handler(nx_module_t *module, nx_exception_t *e);
14 |
15 | typedef struct nx_om_kafka_conf_t {
16 | char* brokerlist;
17 | char* topic;
18 | char* compression;
19 | int partition;
20 | rd_kafka_conf_t *kafka_conf;
21 | rd_kafka_topic_conf_t *topic_conf;
22 | rd_kafka_t *rk;
23 | rd_kafka_topic_t *rkt;
24 | nx_logdata_t *logdata;
25 | } nx_om_kafka_conf_t;
26 |
27 |
28 | #endif /* __NX_OM_KAFKA_H */
29 |
--------------------------------------------------------------------------------