├── .gitignore ├── ChangeLog.md ├── LICENSE ├── Makefile ├── README.md ├── coverity_model.c ├── docs └── fq_protocol.md ├── dtest.d ├── fq.d ├── fq.h ├── fq_bench.c ├── fq_client.c ├── fq_dtrace.blank.h ├── fq_dtrace.d ├── fq_msg.c ├── fq_rcvr.c ├── fq_sndr.c ├── fq_utils.c ├── fqc.c ├── fqd.c ├── fqd.h.in ├── fqd_ccs.c ├── fqd_config.c ├── fqd_dss.c ├── fqd_dyn_sample.c ├── fqd_http.c ├── fqd_listener.c ├── fqd_peer.c ├── fqd_private.h ├── fqd_prog.c ├── fqd_queue.c ├── fqd_queue_jlog.c ├── fqd_queue_mem.c ├── fqd_routemgr.c ├── fqs.c ├── fqtool.c ├── http_parser.c ├── http_parser.h ├── java ├── Makefile ├── fq_rcvr.java ├── pom.xml └── src │ ├── com │ └── omniti │ │ └── labs │ │ ├── FqClient.java │ │ ├── FqClientImplDebug.java │ │ ├── FqClientImplInterface.java │ │ ├── FqClientImplNoop.java │ │ ├── FqCommand.java │ │ ├── FqCommandProtocolError.java │ │ ├── FqDataProtocolError.java │ │ ├── FqHeartbeatException.java │ │ └── FqMessage.java │ └── main │ └── java │ └── com │ └── circonus │ ├── FqClient.java │ ├── FqClientImplDebug.java │ ├── FqClientImplInterface.java │ ├── FqClientImplNoop.java │ ├── FqCommand.java │ ├── FqCommandProtocolError.java │ ├── FqDataProtocolError.java │ ├── FqHeartbeatException.java │ └── FqMessage.java ├── lua ├── fq-proxy ├── fq-receiver ├── fq-sender ├── fqclient.lua.tail └── generatelua.sh ├── service-configs ├── 50-circonus-fq.preset ├── circonus-fq.service └── daemon_options ├── test ├── lua-support │ └── init.lua ├── run-tests.sh └── test_spec.lua └── web ├── css ├── bootstrap-theme.css ├── bootstrap-theme.min.css ├── bootstrap.css ├── bootstrap.min.css ├── colorbrewer.css └── theme.css ├── favicon.png ├── fonts ├── glyphicons-halflings-regular.eot ├── glyphicons-halflings-regular.svg ├── glyphicons-halflings-regular.ttf └── glyphicons-halflings-regular.woff ├── i └── product-logo-rev2.png ├── index.html └── js ├── bootstrap.js ├── bootstrap.min.js ├── colorbrewer.js ├── d3.min.js ├── fq.js └── jquery-3.5.1.min.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.o 3 | *.lo 4 | *.so* 5 | fqd.h 6 | fqd 7 | fqc 8 | fqs 9 | fqtool 10 | fq_bench 11 | fq_dtrace.h 12 | fq_rcvr 13 | fq_sndr 14 | lua/fqclient.lua 15 | Makefile.build 16 | Makefile.depend 17 | *.dSYM/ 18 | .*.swp 19 | ck-*/Makefile 20 | ck-*/build/ck.spec 21 | ck-*/doc/Makefile 22 | java/classes/ 23 | java/*.class 24 | java/fqclient.jar 25 | *~ 26 | .autotools 27 | .cproject 28 | .project 29 | .settings/ 30 | test/fqd.sqlite 31 | test/out.log 32 | test/test_detail.xml 33 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # ChangeLog 2 | 3 | ### v0.13.11 4 | 5 | * Fix test issue where libfq could not be found. 6 | * Minor change to compiler optimization flag. Level `O5` does not exist and is 7 | effectively `O3`. 8 | * Update systemd install path. The new path is equivalent to the old, but 9 | avoids potential issues with `/lib` being a symbolic link on popular Linux 10 | distros. 11 | * Fix potential race in host resolution. 12 | 13 | ### v0.13.10 14 | 15 | * Correct a build issue when libbcd support is disabled. 16 | * Fix race condition in client connection status. 17 | 18 | ### v0.13.9 19 | 20 | * Fix file descriptor leak on connection error. 21 | * Correct usage of `pthread_*` return values. Remove invalid use of `volatile`. 22 | * Add missing `volatile` in Java client, and replace `LinkedList` with 23 | `ArrayDeque`. 24 | 25 | ### v0.13.8 26 | 27 | * Upgrade jquery to 3.5.1 for FQ user interface 28 | 29 | ### v0.13.7 30 | 31 | * Queue drops are tracked as `dropped_to`. 32 | * Web UI updated to display queue drops/rate. 33 | * -b deprecated, -B added, and BCD is disabled by default. 34 | 35 | ### v0.13.6 36 | 37 | * Track drops to queues as `dropped_in` in status. 38 | 39 | ### v0.13.5 40 | 41 | * Force disconnect on message read/write error. 42 | * Reuse listener threads. 43 | 44 | ### v0.13.4 45 | 46 | * Various code cleanups. 47 | * Better bounds checking on auth handshake (allow full size). 48 | * Fix BCD integration. 49 | 50 | ### v0.13.3 51 | 52 | * Set `SO_REUSEPORT = 1` for listener. 53 | * Add `-b` to disable BCD/backtrace integration. 54 | 55 | ### v0.13.2 56 | 57 | * Name threads on Linux to aid debugging. 58 | * Prevent abort when queue removal fails. 59 | 60 | ### v0.13.1 61 | 62 | * Add libbcd support for catching faults with backtrace. 63 | 64 | ### v0.13.0 65 | 66 | * Place fq modules in $(LIBEXECDIR)/fq 67 | * Automatically load all available modules 68 | 69 | ### v0.12.1 70 | 71 | * Move the `valnode_t` definition into fq.h. 72 | * Fix hex construction macro. 73 | * Support var-args in loadable program functions. 74 | * Fix multi-argument parsing in routing grammar. 75 | 76 | ### v0.12.0 77 | 78 | * Omit unneeded library dependencies on Illumos. 79 | * Make poll() calls resume after signal interruption. 80 | 81 | ### v0.11.0 82 | 83 | * Use socket keep-alives for client/server connections. 84 | * Fix use-after-free bug in lua ffi client bindings. 85 | * Fix test suite. 86 | * Explicit registration of local function to better navigate 87 | changing dlsym "self" personalities. 88 | * ENABLE_DTRACE=1 Linux build flag. 89 | 90 | ### v0.10.14 91 | 92 | * Fixes to fq-client.lua on OmniOS 93 | 94 | ### v0.10.13 95 | 96 | * Add `fqs` tool for sending messages from stdin 97 | * Test suite utilizing `mtevbusted` from 98 | [libmtev](https://github.com/circonus-labs/libmtev/) (PR #37) 99 | 100 | ### v0.10.12 101 | 102 | * Fix misuse of stack for freeing messages (0.10.11 fix was bad). 103 | * Add Linux futex support for lower-latency idle network wake-up. 104 | * Ensure message ordering on per-client data connections. 105 | 106 | ### v0.10.11 107 | 108 | * Fix crash when shutting down client that has never seen a message. 109 | 110 | ### v0.10.10 111 | 112 | * Fix source management issue. 0.10.9 tag exluded commits. 113 | * Change message free-lists to prevent use-after-free on thread exit. 114 | 115 | ### v0.10.9 116 | 117 | * Fix builds on newer Mac OS X 118 | * Change message free-lists to prevent use-after-free on thread exit. 119 | * Fix bug in server->client heartbeats not beating. 120 | 121 | ### v0.10.8 122 | 123 | * Fix querystring parsing crash when parameters are not k=v form. 124 | * Resume on-disk queues at the right checkpoint location. 125 | 126 | ### v0.10.7 127 | 128 | * Fix bug in route binding prefix matching causing misdirected messages 129 | (clients could get more than they asked for). 130 | * Fix bug on some Linux systems regarding exposed symbols. 131 | 132 | ### v0.10.6 133 | 134 | * Fix crashing issue enqueueing messages due to unsafe use of spsc fifos. 135 | * Add dynamic loading of routing program extensions. 136 | * Move the "sample" function to a dynamic extension for example purposes. 137 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 OmniTI Computer Consulting, Inc. 2 | All rights reserved. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including without limitation the 7 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # If you want a verbose make (visible commands) add V=1 to you invocation 2 | 3 | .SUFFIXES: .lo 4 | 5 | CC=gcc 6 | LD=gcc 7 | LN_S=ln -s 8 | COPT=-O3 9 | TAR=tar 10 | SED=sed 11 | PREFIX=/usr/local 12 | INCLUDEDIR=$(PREFIX)/include 13 | LIBDIR=$(PREFIX)/lib 14 | LIBEXECDIR=$(PREFIX)/libexec 15 | BINDIR=$(PREFIX)/bin 16 | SBINDIR=$(PREFIX)/sbin 17 | VARLIBFQ=$(PREFIX)/var/lib/fq 18 | LUADIR=$(PREFIX)/share/lua/5.1 19 | INSTALL=install 20 | SHLD=$(LD) -shared 21 | MODULELD=$(LD) -shared 22 | LIBEXT=so 23 | SHCFLAGS=-fPIC 24 | DTRACE=/usr/sbin/dtrace 25 | OS=$(shell uname) 26 | 27 | FQ_MAJOR=0 28 | FQ_MINOR=13 29 | FQ_MICRO=12 30 | 31 | Q= 32 | ifeq ($(V),) 33 | Q=@ 34 | endif 35 | 36 | VENDOR_CFLAGS= 37 | VENDOR_LDFLAGS= 38 | DTRACEFLAGS= 39 | EXTRA_CFLAGS=$(VENDOR_CFLAGS) -g -D_REENTRANT -std=gnu99 -pedantic -Wall 40 | EXTRA_CFLAGS+=-DVARLIBFQDIR=\"$(VARLIBFQ)\" 41 | EXTRA_CFLAGS+=-DLIBEXECDIR=\"$(LIBEXECDIR)/fq\" 42 | #EXTRA_CFLAGS+=-DDEBUG 43 | 44 | CLIENT_OBJ=fq_client.o fq_msg.o fq_utils.o 45 | CLIENT_OBJ_LO=$(CLIENT_OBJ:%.o=%.lo) 46 | FQD_OBJ=fqd.o fqd_listener.o fqd_ccs.o fqd_dss.o fqd_config.o \ 47 | fqd_queue.o fqd_routemgr.o fqd_queue_mem.o fqd_queue_jlog.o \ 48 | fqd_http.o fqd_prog.o fqd_peer.o http_parser.o \ 49 | $(CLIENT_OBJ) 50 | FQC_OBJ=fqc.o $(CLIENT_OBJ) 51 | FQD_SAMPLE_OBJ=fqd_dyn_sample.lo 52 | FQD_DTRACE_OBJ= 53 | 54 | skip_bcd=$(NO_BCD) 55 | ifdef skip_bcd 56 | FQDLIBS=-ljlog -lsqlite3 57 | EXTRA_CFLAGS+=-DNO_BCD 58 | else 59 | FQDLIBS=-ljlog -lsqlite3 -lbcd 60 | endif 61 | LIBS+=-lck 62 | 63 | SHLDFLAGS= 64 | ifeq ($(OS),SunOS) 65 | SHLDFLAGS+=-R$(LIBDIR) 66 | LIBS+=-lcrypto -lsocket -lnsl -lumem -luuid 67 | LIBLIBS+=-luuid -lsocket -lnsl 68 | EXTRA_CFLAGS+=-D_XOPEN_SOURCE=600 69 | EXTRA_CFLAGS+=-D_BSD_SOURCE 70 | EXTRA_CFLAGS+=-D__EXTENSIONS__ -DHAVE_UINTXX_T -DSIZEOF_LONG_LONG_INT=8 -m64 -D_REENTRANT -DHAVE_GETHOSTBYNAME_R 71 | EXTRA_SHLDFLAGS=-m64 72 | FQD_DTRACE_OBJ=fq_dtrace.o 73 | DTRACEFLAGS=-xnolibs 74 | else 75 | ifeq ($(OS),Darwin) 76 | MODULELD=ld -bundle 77 | LOADER=-bundle_loader fqd -lc 78 | EXTRA_CFLAGS+=-D_DARWIN_C_SOURCE -DHAVE_U_INTXX_T -DHAVE_INTXX_T -DHAVE_U_INT64_T -DHAVE_INT64_T \ 79 | -Wno-dollar-in-identifier-extension -Wno-gnu-statement-expression -Wno-deprecated-declarations 80 | #EXTRA_CFLAGS+=-Weverything 81 | LIBEXT=dylib 82 | else 83 | ifeq ($(OS),Linux) 84 | EXTRA_CFLAGS+=-D_XOPEN_SOURCE=600 85 | EXTRA_CFLAGS+=-D_DEFAULT_SOURCE -DBYTE_ORDER=__BYTE_ORDER -DBIG_ENDIAN=__BIG_ENDIAN 86 | SHLDFLAGS+=-Wl,-rpath=$(LIBDIR) 87 | LDFLAGS+=-rdynamic -export-dynamic 88 | LIBS+=-lcrypto -lpthread -ldl -luuid -lrt 89 | LIBLIBS+=-lpthread -luuid -lrt 90 | DTRACE= 91 | ifeq ($(ENABLE_DTRACE),1) 92 | DTRACE=/bin/dtrace 93 | FQD_DTRACE_OBJ=fq_dtrace.o 94 | endif 95 | else 96 | ifeq ($(OS),FreeBSD) 97 | SHLDFLAGS+=-Wl,-rpath=$(LIBDIR) 98 | LDFLAGS+=-rdynamic 99 | LIBS+=-lcrypto -lpthread -luuid -lexecinfo 100 | LIBLIBS+=-lpthread -luuid -lexecinfo 101 | FQD_DTRACE_OBJ=fq_dtrace.o 102 | endif 103 | endif 104 | endif 105 | endif 106 | 107 | all: libfq.$(LIBEXT) libfq.a fqd fqc fqs fqtool fq_sndr fq_rcvr fq_bench \ 108 | fq-sample.so lua/fqclient.lua 109 | 110 | include Makefile.depend 111 | 112 | SHLDFLAGS+=$(VENDOR_LDFLAGS) -L$(LIBDIR) 113 | ifeq ($(OS),Darwin) 114 | SHLDFLAGS+=-current_version $(FQ_MAJOR).$(FQ_MINOR).$(FQ_MICRO) -install_name $(LIBDIR)/libfq.$(FQ_MAJOR).dylib 115 | SOLONG=libfq.$(FQ_MAJOR).$(FQ_MINOR).$(FQ_MICRO).dylib 116 | SOSHORT=libfq.$(FQ_MAJOR).dylib 117 | LIBNAME=libfq.dylib 118 | else 119 | SHLDFLAGS+=-Wl,-soname,libfq.so.$(FQ_MAJOR) 120 | SOLONG=libfq.so.$(FQ_MAJOR).$(FQ_MINOR).$(FQ_MICRO) 121 | SOSHORT=libfq.so.$(FQ_MAJOR) 122 | LIBNAME=libfq.so 123 | endif 124 | 125 | CFLAGS+=$(EXTRA_CFLAGS) 126 | SHCFLAGS+=$(EXTRA_CFLAGS) 127 | LDFLAGS+=$(VENDOR_LDFLAGS) 128 | 129 | fqd.h: fqd.h.in 130 | sed -e 's/@@FQ_MAJOR@@/'$(FQ_MAJOR)'/g;' \ 131 | -e 's/@@FQ_MINOR@@/'$(FQ_MINOR)'/g;' \ 132 | -e 's/@@FQ_MICRO@@/'$(FQ_MICRO)'/g;' < fqd.h.in > fqd.h 133 | 134 | fq_dtrace.h: fq_dtrace.d 135 | -$(DTRACE) $(DTRACEFLAGS) -h -o $@ -s $< 136 | if [ ! -f $@ ]; then cp fq_dtrace.blank.h $@; fi 137 | 138 | fq_dtrace.o: $(FQD_OBJ) 139 | $(DTRACE) $(DTRACEFLAGS) -64 -G -s fq_dtrace.d -o $@ $(FQD_OBJ) 140 | 141 | fq_dtrace.blank.h: fq_dtrace.h 142 | awk 'BEGIN{print "#if 0"} /#else/,/#endif/{print}' $< > $@ 143 | 144 | fqd: $(FQD_OBJ) $(FQD_DTRACE_OBJ) 145 | @echo " - linking $@" 146 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(FQD_OBJ) $(FQD_DTRACE_OBJ) $(LIBS) $(FQDLIBS) 147 | 148 | fqc: $(FQC_OBJ) 149 | @echo " - linking $@" 150 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(FQC_OBJ) $(LIBS) 151 | 152 | fq-sample.so: fqd $(FQD_SAMPLE_OBJ) 153 | $(Q)$(MODULELD) $(LOADER) $(EXTRA_SHLDFLAGS) -o $@ $(FQD_SAMPLE_OBJ) 154 | 155 | fq_sndr: fq_sndr.o libfq.a 156 | @echo " - linking $@" 157 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -L. -lfq -o $@ $^ $(LIBS) 158 | 159 | fqs: fqs.o libfq.a 160 | @echo " - linking $@" 161 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -L. -lfq -o $@ $^ $(LIBS) 162 | 163 | fq_rcvr: fq_rcvr.o libfq.a 164 | @echo " - linking $@" 165 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -L. -lfq -o $@ $^ $(LIBS) 166 | 167 | fqtool: fqtool.o libfq.a 168 | @echo " - linking $@" 169 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -L. -lfq -o $@ $^ $(LIBS) 170 | 171 | fq_bench: fq_bench.o libfq.a 172 | @echo " - linking $@" 173 | $(Q)$(CC) $(CFLAGS) $(LDFLAGS) -L. -lfq -o $@ $^ $(LIBS) 174 | 175 | libfq.$(LIBEXT): $(CLIENT_OBJ_LO) 176 | @echo " - creating $@" 177 | $(Q)$(SHLD) $(EXTRA_SHLDFLAGS) $(SHLDFLAGS) -o $@ $(CLIENT_OBJ_LO) $(LIBLIBS) 178 | $(LN_S) -f $@ $(SOSHORT) 179 | 180 | libfq.a: $(CLIENT_OBJ) 181 | @echo " - creating $@" 182 | $(Q)ar cr $@ $(CLIENT_OBJ) 183 | 184 | .c.o: $< 185 | @echo " - compiling $<" 186 | $(Q)$(CC) $(CPPFLAGS) $(CFLAGS) $(COPT) -o $@ -c $< 187 | 188 | .c.lo: $< 189 | @echo " - compiling $<" 190 | $(Q)$(CC) $(CPPFLAGS) $(SHCFLAGS) -o $@ -c $< 191 | 192 | Makefile.depend: fq_dtrace.h fqd.h 193 | @echo " - make depend" 194 | $(Q)$(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c > Makefile.depend 195 | 196 | java/fqclient.jar: 197 | (cd java && $(MAKE) fqclient.jar) 198 | 199 | lua/fqclient.lua: 200 | (cd lua; ./generatelua.sh) 201 | 202 | install: all 203 | $(INSTALL) -d $(DESTDIR)$(INCLUDEDIR) 204 | $(INSTALL) -m 0444 fq.h $(DESTDIR)$(INCLUDEDIR)/fq.h 205 | $(INSTALL) -d $(DESTDIR)$(LIBDIR) 206 | $(INSTALL) -m 0444 libfq.a $(DESTDIR)$(LIBDIR)/libfq.a 207 | $(INSTALL) -m 0555 libfq.$(LIBEXT) $(DESTDIR)$(LIBDIR)/$(SOLONG) 208 | $(LN_S) -f $(SOLONG) $(DESTDIR)$(LIBDIR)/$(SOSHORT) 209 | $(LN_S) -f $(SOLONG) $(DESTDIR)$(LIBDIR)/$(LIBNAME) 210 | $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/fq 211 | $(INSTALL) -m 0555 fq-sample.so $(DESTDIR)$(LIBEXECDIR)/fq/fq-sample.so 212 | $(INSTALL) -d $(DESTDIR)$(BINDIR) 213 | $(INSTALL) -m 0555 fqtool $(DESTDIR)$(BINDIR)/fqtool 214 | $(INSTALL) -m 0555 fqs $(DESTDIR)$(BINDIR)/fqs 215 | $(INSTALL) -d $(DESTDIR)$(SBINDIR) 216 | $(INSTALL) -m 0555 fqd $(DESTDIR)$(SBINDIR)/fqd 217 | $(INSTALL) -d $(DESTDIR)$(VARLIBFQ) 218 | $(TAR) cf - web | (cd $(DESTDIR)$(VARLIBFQ) && $(TAR) xf -) 219 | $(INSTALL) -d $(DESTDIR)/usr/lib/dtrace 220 | $(INSTALL) -m 0444 fq.d $(DESTDIR)/usr/lib/dtrace/fq.d 221 | $(INSTALL) -d $(DESTDIR)$(LUADIR) 222 | $(INSTALL) -m 0644 lua/fqclient.lua $(DESTDIR)$(LUADIR) 223 | $(INSTALL) -m 0555 lua/fq-sender lua/fq-receiver lua/fq-proxy $(DESTDIR)$(BINDIR) 224 | 225 | install-systemd: install 226 | $(INSTALL) -d $(DESTDIR)/usr/lib/systemd/system 227 | $(INSTALL) -d $(DESTDIR)/usr/lib/systemd/system-preset 228 | $(INSTALL) -m 0644 service-configs/circonus-fq.service $(DESTDIR)/usr/lib/systemd/system/circonus-fq.service 229 | $(INSTALL) -m 0644 service-configs/50-circonus-fq.preset $(DESTDIR)/usr/lib/systemd/system-preset/50-circonus-fq.preset 230 | $(INSTALL) -m 0644 service-configs/daemon_options $(DESTDIR)$(VARLIBFQ)/daemon_options 231 | 232 | clean: 233 | rm -f *.o *.a fqc fqd fqs *.$(LIBEXT) $(SOSHORT) fq_dtrace.h lua/fqclient.lua 234 | 235 | .PHONY: test 236 | test: lua/fqclient.lua 237 | ./test/run-tests.sh 238 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fq. 2 | 3 | 4 | Coverity Scan Build Status 6 | 7 | 8 | fq is a *brokered* message queue using a publish subscribe model. It is architected for performance and isn't (today) designed for large numbers of connected clients. 9 | 10 | 11 | +------------+ +-----------+ 12 | |- exchange -|<-- (msg publication) --|- client0 -| 13 | +------------+ +-----------+ 14 | |- routemap -| 15 | +------------+ 16 | | | +---------+ 17 | | +--------------|- queue -| 18 | | +---------+ 19 | +---------+ | 20 | |- queue -| | +-----------+ 21 | +---------+ +---|- client1 -| 22 | | +-----------+ 23 | | +-----------+ 24 | +--|- client2 -| 25 | | +-----------+ 26 | | 27 | +-----------+ 28 | |- client3 -| 29 | +-----------+ 30 | 31 | ## Terminology 32 | 33 | ### Broker 34 | 35 | The `fqd` process. The daemon through which all knowledge passes. 36 | 37 | ### Peers 38 | 39 | Peers are connected `fqd` processes. It is important to note that peers are unidirectional. If A peers with B, then A will act as a client to B. If you want bidirectional peering, you must specify that A peers with B and B peers with A. The system aims to prevent cyclic delivery of messages efficiently. 40 | 41 | Adding peers is done directly via fqd's sqlite DB store: 42 | 43 | ``` 44 | ; sqlite3 /var/lib/fq/fqd.sqlite 45 | sqlite> INSERT INTO "upstream" 46 | (host, port, source, password, exchange, program, permanent_binding) 47 | VALUES('peerB',8765,'fqd-peera//mem:drop,private,backlog=4096','none','logging','prefix:"http.access.json."','false'); 48 | ``` 49 | 50 | ### Client 51 | 52 | * [C client - libfq](https://github.com/postwait/fq/blob/master/fq.h#L164-L205) 53 | * [Java client - fq.jar](https://github.com/postwait/fq/blob/master/java/src/com/omniti/labs/FqClientImplInterface.java) 54 | * [Node.js client - fq](https://www.npmjs.com/package/fq) 55 | * [Go client - fq](https://godoc.org/github.com/postwait/gofq) 56 | * submission-only /submit API (see below) 57 | 58 | A client is an applications connection to fq over TCP/IP to send or receive messages. A client makes two TCP/IP connections to fq. An application can present itself to fq as multiple clients at one time (by opening new pairs of connections). See Queues for reasons why. 59 | 60 | ### Exchanges 61 | 62 | Exchanges are like buses on which messages may be sent. You cannot send a message without doing so on an exchange. Exchanges are created within fq on-demand. 63 | 64 | ### Queues 65 | 66 | Queues are queues. If you stick something in one end, you should expect it to come out the other. A single queue may have multiple clients subscribed. When a client connects, it is attached to one and only one queue. If an application wishes to attach to more than one queue, it should present as multiple clients. Queues use a competitive consumption model meaning that if multiple clients are attached to a single queue, the messages sent to that queue will be distributed over the clients such that no two clients will see the same message. 67 | 68 | #### Queue Types 69 | 70 | Queues can be of type `mem` or `disk`. The contents of memory queues will not survive restarts. 71 | 72 | Various parameters can be set on a queue using the syntax `type:param1,param2`. 73 | 74 | #### Sharing 75 | 76 | Queues with the `public` parameter can have multiple clients connected to them (in which case they compete for messages). If you want a private queue you can specify the `private` parameter. 77 | 78 | #### Policy 79 | 80 | Queues can either have a `block` or `drop` policy. The drop policy means that messages that would be routed to a queue that is full will be dropped and never delivered. The block policy will cause the publisher to wait until there is room in the queue. The block policy makes no sense on a disk queue. 81 | 82 | #### Backlog 83 | 84 | The `backlog=` parameter will specify how many messages may be held in the queue before the block or drop policies are applied. 85 | 86 | #### Permanence 87 | 88 | If you want a queue to be remembered by fqd, you can specify `permanent` as a flag. If you'd like for fqd to forget the queue after all clients have disconnected, you can specify the `transient` flag. If neither flag is specified, then an existing queue will retain its previous permanence setting or a new transient queue will be created. 89 | 90 | #### Examples: 91 | 92 | A queue called `bob` will be in memory, allowed to have multiple clients connected to it, with a drop policy and an allowable message backlog of 100000 messages: `bob/mem:public,drop,backlog=100000` 93 | 94 | A connection client will specify username/queue. A user "USER" connecting to the aforementioned queue would connect as `USER/bob/mem:public,drop,backlog=100000` 95 | 96 | ### Messages 97 | 98 | Messages are, of course, a payload and metadata. 99 | 100 | #### Message metadata 101 | 102 | Some are set by the broker. 103 | 104 | * sender [set by the broker] 105 | * hops (a list of fqd via which the message passed) 106 | 107 | Others are set by the sender. 108 | 109 | * exchange (up to 127 bytes) 110 | * route (up to 127 bytes) 111 | * id (128 bits). The first 64 bits the sender shall control, the latter 64bits the broker *might* control. 112 | 113 | ### Routes and Programs 114 | 115 | Routes and programs define how messages sent on exchanges are placed in queues: 116 | 117 | - A receiver that connects to an fq-broker specifies a program that filters the messages on the exchange. 118 | - A sender specifies a route for every message as part of the metadata 119 | 120 | Programs follow the following syntax (cf. `fqd.h`): 121 | 122 | ``` 123 | PROGRAM: :string RULES* 124 | RULE: (RULE) 125 | RULE: (RULE && RULE) 126 | RULE: (RULE || RULE) 127 | RULE: EXPR 128 | EXPR: function(args) 129 | args: arg 130 | args: arg, args 131 | arg: "string" 132 | arg: true|false 133 | arg: [0-9][0-9]*(?:.[0-9]*) 134 | 135 | functions are dynamically loadable with type signature 136 | strings: s, booleans: b, integers: d 137 | function: substr_eq(9.3,10,"tailorings",true) 138 | C symbol: fqd_route_prog__substr_eq__ddsb(int nargs, valnode_t *args); 139 | ``` 140 | 141 | In particular: 142 | 143 | - Every program starts with either `prefix:` or `exact:` 144 | - The program `prefix:` matches all rules 145 | - The program string is matched against the message route 146 | 147 | The following rule functions are defined in `fq_prog.c`: 148 | 149 | - `fqd_route_prog__sample__d()` -- subsample the stream 150 | - `fqd_route_prog__route_contains__s()` -- check if route contains a string 151 | - `fqd_route_prog__payload_prefix__s()` -- check if payload starts with prefix 152 | - `fqd_route_prog__payload_contains__s()` -- check if payload contains a string 153 | - `fqd_route_prog__true__()` -- always true 154 | 155 | Examples: 156 | 157 | - `prefix:` -- matches all messages 158 | - `prefix:bla` or `prefix:"bla"` -- matches all messages with rules starting with the sting 'bla' 159 | - `prefix: payload_prefix("M")` -- matches messages where the payload starts with 'M' 160 | - `prefix:foo (payload_prefix("M") && route_contains("bar"))` -- matches messages where the payload starts with 'M' and route starts with "foo" and moreover contains "bar" 161 | 162 | ## Protocol 163 | 164 | Information on command and message protocol is found in `docs/fq_protocol.md` 165 | 166 | ### HTTP superposition 167 | 168 | The Fq protocol also acts as a non-compliant HTTP server (though compliant enough of most clients and browsers). Fq ships with a web UI that allows inspecting real-time state and performance. 169 | 170 | #### GET /stats.json 171 | 172 | exposes current exchange, queue, and client information. 173 | 174 | #### POST /submit 175 | 176 | An endpoint allowing message submission without a full and stateful Fq connection. It expects the following headers: 177 | 178 | * ```X-Fq-User```, 179 | * ```X-Fq-Route```, and 180 | * ```X-Fq-Exchange```. 181 | 182 | The HTTP client *MUST* provide a Content-Length header corresponding to the payload content (no chunked submission). The payload is treated as the raw message box without any special encoding. 183 | 184 | Example: 185 | 186 | ``` 187 | curl -X POST -H "X-Fq-User: user" -H 'X-Fq-Route: bla' -H 'X-Fq-Exchange: test' localhost:8765/submit --data "TEST" 188 | ``` 189 | 190 | ## Building 191 | 192 | Requirements: 193 | * C compiler 194 | * GNU make 195 | * libuuid 196 | * sqlite3 197 | * [jlog](https://github.com/omniti-labs/jlog) 198 | * [libbcd](https://github.com/backtrace-labs/bcd) (optional, for crash tracing) 199 | 200 | Generally: 201 | ``` 202 | make 203 | make install 204 | ``` 205 | 206 | To build without libbcd support: 207 | ``` 208 | NO_BCD=1 make 209 | ``` 210 | 211 | ## Debugging 212 | 213 | FQ can be run in debug mode from the command line. 214 | 215 | To run FQ in debug mode, kill any and all existing FQ processes, then enter the 216 | following command: 217 | ``` 218 | fq -g fq FQ_DEBUG= /fqd -D -c /fqd.sqlite -p 219 | ``` 220 | 221 | Flag values determine debug output type and can have the following values: 222 | ``` 223 | FQ_DEBUG_MEM = 0x00000001, 224 | FQ_DEBUG_MSG = 0x00000002, 225 | FQ_DEBUG_ROUTE = 0x00000004, 226 | FQ_DEBUG_IO = 0x00000008, 227 | FQ_DEBUG_CONN = 0x00000010, 228 | FQ_DEBUG_CONFIG = 0x00000020, 229 | FQ_DEBUG = 0x00000040, 230 | FQ_DEBUG_PEER = 0x00000080, 231 | FQ_DEBUG_HTTP = 0x00000100, 232 | FQ_DEBUG_PANIC = 0x40000000 233 | ``` 234 | 235 | To debug more than one flag, simply OR the flag values. For example, to output 236 | connection, configuration, and route information, set `FQ_DEBUG` equal to 237 | `0x00000034 (FQ_DEBUG_CONFIG|FQ_DEBUG_CONN|FQ_DEBUG_ROUTE)`. 238 | 239 | For example, you can run FQ in debug mode with the variables shown below to 240 | output configuration, connection, and route information to the console: 241 | ``` 242 | fq -g fq FQ_DEBUG=0x00000034 /opt/circonus/sbin/fqd -D -c /opt/circonus/var/lib/fq/fqd.sqlite -p 8765 243 | ``` 244 | -------------------------------------------------------------------------------- /coverity_model.c: -------------------------------------------------------------------------------- 1 | typedef unsigned int uint32_t; 2 | typedef _Bool bool; 3 | 4 | void 5 | ck_pr_dec_uint_zero(uint32_t *v, bool *zero) { 6 | *v = (*v) - 1; 7 | if(zero) *zero = ((*v) == 0); 8 | } 9 | 10 | void 11 | ck_pr_inc_uint(uint32_t *v) { 12 | *v = (*v) + 1; 13 | } 14 | 15 | uint32_t 16 | ck_pr_load_uint(uint32_t *v) { 17 | return *v; 18 | } 19 | 20 | -------------------------------------------------------------------------------- /docs/fq_protocol.md: -------------------------------------------------------------------------------- 1 | # fq Protocol 2 | 3 | ## Client 4 | 5 | Clients maintain two paired tcp connections to fq. After the connections are made, some preliminary data is sent over the command socket. A "plain auth" command is then issued, return a client key. The client key is then used in some preliminary data sent over the data socket, pairing the two sockets for the session. 6 | 7 | ### Prefixes 8 | 9 | * Cmd Mode: `0xcc50cafe` 10 | * Data Mode: `0xcc50face` 11 | * Peer Mode: `0xcc50fade` 12 | 13 | ### Command Socket 14 | 15 | Length | Description 16 | ---------+----------------------------- 17 | 4 bytes | Cmd Mode 18 | 19 | ### Data Socket 20 | 21 | Length | Description 22 | ---------+----------------------------- 23 | 4 bytes | Data Mode 24 | 2 bytes | Client Key Length 25 | variable | Client Key 26 | 27 | ## Commands 28 | 29 | General form `(command prefix)(command)`, big endian. All non-heartbeat related commands have in-band responses corresponding to the order in which requests they were sent to fq. Heartbeat requests are used simply to tell fq to look for and to send heartbeats at a specific interval. Heartbeats should be checked for during normal command processing and not as a response to a specific request. 30 | 31 | ### Prefixes 32 | 33 | Commad prefixes are two bytes at the beginning of the command 34 | 35 | * Error: `0xeeee` 36 | * Heartbeat: `0xbea7` 37 | * Auth CMD: `0xaaaa` 38 | * Auth Plain: `0x0000` 39 | * Auth Response: `0xaa00` 40 | * Heartbeat Request: `0x4848` 41 | * Bind: `0xb171` 42 | * Bind Request: `0xb170` 43 | * Unbind: `0x171b` 44 | * Unbind Request: `0x071b` 45 | * Status: `0x57a7` 46 | * Status Request: `0xc7a7` 47 | 48 | #### Plain Auth 49 | 50 | Plain Auth is a subset of Auth and will have both prefixs 51 | 52 | ##### Request 53 | 54 | Length | Description 55 | ---------+----------------------------- 56 | 2 bytes | Auth CMD Prefix 57 | 2 bytes | Auth Plain Prefix 58 | 2 bytes | User Length 59 | variable | User 60 | 2 bytes | Queue Length (Queue + 1 byte + Queue type) 61 | variable | Queue 62 | 1 byte | 0 63 | variable | Queue type ("mem" or "disk") + ":param,param" 64 | 2 bytes | Password Length (16 bit) 65 | variable | Password 66 | 67 | 68 | ##### Response 69 | 70 | Length | Description 71 | ---------+----------------------------- 72 | 2 bytes | Auth Response Prefix 73 | 2 bytes | Client Key Length Length 74 | variable | Client Key (0 < length < 127 bytes) 75 | 76 | #### Bind 77 | 78 | ##### Request 79 | Length | Description 80 | ---------+----------------------------- 81 | 2 bytes | Bind Request Prefix 82 | 2 bytes | Flags: Peer Mode (0 or 1) | perm(0110)/trans(0100) 83 | 2 bytes | Exchange Length 84 | 2 bytes | Exchange 85 | 2 bytes | Program Length 86 | variable | Program 87 | 88 | ##### Response 89 | 90 | Length | Description 91 | ---------+----------------------------- 92 | 2 bytes | Bind Prefix 93 | 4 bytes | Binding ID 94 | 95 | #### Unbind 96 | 97 | ##### Request 98 | 99 | Length | Description 100 | ---------+----------------------------- 101 | 2 bytes | Unbind Request Prefix 102 | 4 bytes | Binding ID 103 | 2 bytes | Exchange Length 104 | variable | Exchange 105 | 106 | ##### Response 107 | 108 | Length | Description 109 | ---------+----------------------------- 110 | 2 bytes | Unbind Prefix 111 | 4 bytes | Binding ID 112 | 113 | On success, the response binding id will be the same as the one sent in the request. 114 | 115 | #### Status 116 | 117 | ##### Request 118 | 119 | Length | Description 120 | ---------+----------------------------- 121 | 2 bytes | Status Request Prefix 122 | 123 | ##### Response 124 | 125 | Length | Description 126 | ---------+----------------------------- 127 | 2 bytes | Status Prefix 128 | 2 bytes | Key Length 129 | variable | Key 130 | 4 bytes | Value 131 | … (repeat key length, key, value sets) 132 | 2 bytes | Key Length 0 133 | 134 | The response contains serveral key-value pairs. Each key is prefixed by a length and parsing of kv pairs should continue until a key length of 0 is read. 135 | 136 | #### Heartbeat Request 137 | 138 | ##### Request 139 | 140 | Length | Description 141 | ---------+----------------------------- 142 | 2 bytes | Heartbeat Request Prefix 143 | 2 bytes | Heartbeat Interval (milliseconds) 144 | 145 | ##### Response 146 | 147 | None 148 | 149 | #### Heartbeat 150 | 151 | ##### Request 152 | 153 | Length | Description 154 | ---------+----------------------------- 155 | 2 bytes | Heartbeat Prefix 156 | 157 | ##### Response 158 | 159 | Length | Description 160 | ---------+----------------------------- 161 | 2 bytes | Heartbeat Prefix 162 | 163 | ## Messages 164 | 165 | Length | Description | Note 166 | ---------+-----------------------------------+------------------------ 167 | 1 byte | Exchange Length | 168 | variable | Exchange | 169 | 1 byte | Route Length | 170 | 16 bytes | Message ID | 171 | 1 byte | Sender Length | Send for Peer Mode Only 172 | variable | Sender | Send for Peer Mode Only 173 | 1 byte | Number of Hops | Send for Peer Mode Only 174 | variable | Hops (numHops sets of 4-byte IPs) | Send for Peer Mode Only 175 | 4 bytes | Payload Length | 176 | variable | Payload (<= 128kb) | 177 | 178 | All properties will be present when receiving a message, while some propertiers are only sent when in peer mode. 179 | -------------------------------------------------------------------------------- /dtest.d: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | fq*:::message-receive{ 24 | printf("sender: %s\n", args[2]->sender); 25 | printf("exchange: %s\n", args[2]->exchange); 26 | printf("route: %s\n", args[2]->route); 27 | printf("message len: %d\n", args[2]->payload_len); 28 | printf("message: %.*s\n", args[2]->payload_len, args[2]->payload); 29 | 30 | printf("client: %s\n", args[0]->pretty); 31 | printf("client: %s\n", args[1]->pretty); 32 | 33 | printf("latency: %d\n", args[2]->latency); 34 | } 35 | 36 | fq*:::queue-drop{ 37 | q = ((fq_queue_t *)arg0); 38 | printf("dropped message on queue %s\n", q->name); 39 | } 40 | 41 | fq*:::queue-block{ 42 | printf("blocking queue %s\n", ((fq_queue_t *)arg0)->name); 43 | } 44 | -------------------------------------------------------------------------------- /fq.d: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | typedef struct { 25 | uintptr_t route; 26 | uintptr_t sender; 27 | uintptr_t exchange; 28 | uintptr_t payload; 29 | uint32_t payload_len; 30 | uint64_t latency; 31 | } fq_dtrace_msg_t; 32 | 33 | typedef struct { 34 | string route; 35 | string exchange; 36 | string sender; 37 | string payload; 38 | uint32_t payload_len; 39 | uint64_t latency; 40 | } fq_msg_t; 41 | 42 | translator fq_msg_t { 43 | route = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->route, sizeof(uintptr_t))); 44 | exchange = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->exchange, sizeof(uintptr_t))); 45 | sender = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->sender, sizeof(uintptr_t))); 46 | payload_len = *(uint32_t *)copyin((uintptr_t)&m->payload_len, sizeof(uint32_t)); 47 | payload = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->payload, sizeof(uintptr_t)), *(uint32_t *)copyin((uintptr_t)&m->payload_len, sizeof(uint32_t))); 48 | latency = *(uint64_t *)copyin((uintptr_t)&m->latency, sizeof(uint64_t)); 49 | }; 50 | 51 | typedef struct { 52 | uintptr_t name; 53 | int32_t isprivate; 54 | int32_t policy; 55 | uintptr_t type; 56 | } fq_dtrace_queue_t; 57 | 58 | typedef struct { 59 | string name; 60 | int32_t isprivate; 61 | int32_t policy; 62 | string type; 63 | } fq_queue_t; 64 | 65 | translator fq_queue_t { 66 | name = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->name, sizeof(uintptr_t))); 67 | type = copyinstr(*(uintptr_t *)copyin((uintptr_t)&m->type, sizeof(uintptr_t))); 68 | isprivate = *(uint32_t *)copyin((uintptr_t)&m->isprivate, sizeof(int32_t)); 69 | policy = *(uint32_t *)copyin((uintptr_t)&m->policy, sizeof(int32_t)); 70 | }; 71 | 72 | typedef struct { 73 | int32_t fd; 74 | uintptr_t pretty; 75 | } fq_dtrace_remote_anon_client_t; 76 | 77 | typedef struct { 78 | int32_t fd; 79 | string pretty; 80 | } fq_remote_anon_client_t; 81 | 82 | typedef struct { 83 | int32_t fd; 84 | uintptr_t pretty; 85 | } fq_dtrace_remote_client_t; 86 | 87 | typedef struct { 88 | int32_t fd; 89 | string pretty; 90 | } fq_remote_client_t; 91 | 92 | typedef struct { 93 | int32_t fd; 94 | uintptr_t pretty; 95 | } fq_dtrace_remote_data_client_t; 96 | 97 | typedef struct { 98 | int32_t fd; 99 | string pretty; 100 | } fq_remote_data_client_t; 101 | 102 | translator fq_remote_anon_client_t { 103 | fd = *(uint32_t *)copyin((uintptr_t)&c->fd, sizeof(int32_t)); 104 | pretty = copyinstr(*(uintptr_t *)copyin((uintptr_t)&c->pretty, sizeof(uintptr_t))); 105 | }; 106 | 107 | -------------------------------------------------------------------------------- /fq_bench.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "fq.h" 30 | 31 | void logger(fq_client c, const char *s) { 32 | (void)c; 33 | fprintf(stderr, "fq_logger: %s\n", s); 34 | } 35 | 36 | static void 37 | debug_status(char *key, uint32_t value, void *unused) { 38 | (void)unused; 39 | fq_debug(FQ_DEBUG_CONN, " ---> %s : %u\n", key, value); 40 | } 41 | 42 | static void 43 | print_rate(fq_client c, hrtime_t s, hrtime_t f, uint64_t cnt, uint64_t icnt) { 44 | double d; 45 | fq_client_status(c, debug_status, NULL); 46 | if(cnt) { 47 | d = (double)cnt * 1000000000; 48 | d /= (double)(f-s); 49 | printf("[%d backlog] output %0.2f msg/sec\n", 50 | fq_client_data_backlog(c), d); 51 | } 52 | if(icnt) { 53 | d = (double)icnt * 1000000000; 54 | d /= (double)(f-s); 55 | printf("[%d backlog] input %0.2f msg/sec\n", 56 | fq_client_data_backlog(c), d); 57 | } 58 | } 59 | 60 | static void usage(const char *prog) { 61 | printf("%s:\n", prog); 62 | printf("\t-H\t\tthis help message\n"); 63 | printf("\t-h\t\tthe fq host to connect to\n"); 64 | printf("\t-p \tspecify connecting port (default: 8765)\n"); 65 | printf("\t-u \tuser name\n"); 66 | printf("\t-P \tpassword\n"); 67 | printf("\t-t \t'disk' or 'mem' for type of queue to test\n"); 68 | printf("\t-q \tname of queue to test (default: benchmark_queue)\n"); 69 | printf("\t-c \tnumber of messages to bench with\n"); 70 | printf("\t-s \tsize of each message\n"); 71 | } 72 | 73 | static char *host; 74 | static int port = 8765; 75 | static char *user; 76 | static char *exchange; 77 | static char *pass; 78 | static char *type; 79 | static char *queue_name; 80 | static int count = 100000; 81 | static int size = 100; 82 | 83 | static void parse_cli(int argc, char **argv) { 84 | int c; 85 | const char *debug = getenv("FQ_DEBUG"); 86 | while((c = getopt(argc, argv, "Hh:p:u:P:t:q:e:c:s:")) != EOF) { 87 | switch(c) { 88 | case 'H': 89 | usage(argv[0]); 90 | exit(0); 91 | case 'h': 92 | free(host); 93 | host = strdup(optarg); 94 | break; 95 | case 'p': 96 | port = atoi(optarg); 97 | break; 98 | case 'u': 99 | free(user); 100 | user = strdup(optarg); 101 | break; 102 | case 'P': 103 | free(pass); 104 | pass = strdup(optarg); 105 | break; 106 | case 't': 107 | free(type); 108 | type = strdup(optarg); 109 | break; 110 | case 'q': 111 | free(queue_name); 112 | queue_name = strdup(optarg); 113 | break; 114 | case 'e': 115 | free(exchange); 116 | exchange = strdup(optarg); 117 | break; 118 | case 'c': 119 | count = atoi(optarg); 120 | break; 121 | case 's': 122 | size = atoi(optarg); 123 | break; 124 | default: 125 | usage(argv[0]); 126 | exit(-1); 127 | } 128 | } 129 | if(debug) fq_debug_set_string(debug); 130 | if(!host) host = strdup("localhost"); 131 | if(!user) user = strdup("user"); 132 | if(!pass) pass = strdup("pass"); 133 | if(!queue_name) queue_name = strdup("benchmark_queue"); 134 | if(!exchange) exchange = strdup("maryland"); 135 | if(!type) type = strdup("mem"); 136 | } 137 | 138 | 139 | /** 140 | * Benchmark a single thread against a type of queue at some host. 141 | */ 142 | int main(int argc, char **argv) { 143 | char queue[256] = {0}; 144 | hrtime_t s0, s, f, f0; 145 | uint64_t cnt = 0, icnt = 0; 146 | int i = 0; 147 | fq_client c; 148 | fq_msg *m; 149 | char *fq_debug = getenv("FQ_DEBUG"); 150 | if(fq_debug) fq_debug_set_bits(atoi(fq_debug)); 151 | signal(SIGPIPE, SIG_IGN); 152 | fq_client_init(&c, 0, logger); 153 | 154 | parse_cli(argc, argv); 155 | 156 | sprintf(queue, "%s/%s/%s:public,drop,backlog=10000,permanent", user, queue_name, type); 157 | 158 | printf("using queue -> %s\n", queue); 159 | 160 | fq_client_creds(c, host, port, queue, pass); 161 | fq_client_heartbeat(c, 1000); 162 | fq_client_set_backlog(c, 10000, 100); 163 | fq_client_connect(c); 164 | 165 | printf("payload size -> %d\n", size); 166 | printf("message count -> %d\n", count); 167 | 168 | s0 = s = fq_gethrtime(); 169 | f = s; 170 | 171 | m = fq_msg_alloc_BLANK(size); 172 | memset(m->payload, 'X', size); 173 | fq_msg_exchange(m, exchange, strlen(exchange)); 174 | fq_msg_route(m, "test.bench.foo", 14); 175 | 176 | while(i < count || fq_client_data_backlog(c) > 0) { 177 | if(i < count) { 178 | m->arrival_time = fq_gethrtime(); 179 | fq_msg_id(m, NULL); 180 | fq_client_publish(c, m); 181 | cnt++; 182 | i++; 183 | } 184 | else usleep(10); 185 | 186 | if (i % 1000 == 0) { 187 | f = fq_gethrtime(); 188 | } 189 | 190 | if(f-s > 1000000000) { 191 | print_rate(c, s, f, cnt, icnt); 192 | icnt = 0; 193 | cnt = 0; 194 | s = f; 195 | } 196 | } 197 | f0 = fq_gethrtime(); 198 | print_rate(c, s0, f0, i, 0); 199 | 200 | free(host); 201 | free(user); 202 | free(pass); 203 | free(exchange); 204 | free(queue_name); 205 | free(type); 206 | return 0; 207 | } 208 | -------------------------------------------------------------------------------- /fq_dtrace.blank.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #if 0 25 | #else 26 | 27 | #define FQ_QUEUE_ENQUEUE(arg0, arg1) \ 28 | do { \ 29 | } while (0) 30 | #define FQ_QUEUE_ENQUEUE_ENABLED() (0) 31 | #define FQ_CLIENT_AUTH(arg0) \ 32 | do { \ 33 | } while (0) 34 | #define FQ_CLIENT_AUTH_ENABLED() (0) 35 | #define FQ_CLIENT_AUTH_DATA(arg0) \ 36 | do { \ 37 | } while (0) 38 | #define FQ_CLIENT_AUTH_DATA_ENABLED() (0) 39 | #define FQ_CLIENT_CONNECT(arg0, arg1) \ 40 | do { \ 41 | } while (0) 42 | #define FQ_CLIENT_CONNECT_ENABLED() (0) 43 | #define FQ_CLIENT_DISCONNECT(arg0, arg1) \ 44 | do { \ 45 | } while (0) 46 | #define FQ_CLIENT_DISCONNECT_ENABLED() (0) 47 | #define FQ_CONFIG_ROTATE(arg0) \ 48 | do { \ 49 | } while (0) 50 | #define FQ_CONFIG_ROTATE_ENABLED() (0) 51 | #define FQ_MESSAGE_DELIVER(arg0, arg1, arg2) \ 52 | do { \ 53 | } while (0) 54 | #define FQ_MESSAGE_DELIVER_ENABLED() (0) 55 | #define FQ_MESSAGE_RECEIVE(arg0, arg1, arg2) \ 56 | do { \ 57 | } while (0) 58 | #define FQ_MESSAGE_RECEIVE_ENABLED() (0) 59 | #define FQ_QUEUE_BLOCK(arg0, arg1) \ 60 | do { \ 61 | } while (0) 62 | #define FQ_QUEUE_BLOCK_ENABLED() (0) 63 | #define FQ_QUEUE_CREATE_FAILURE(arg0, arg1, arg2) \ 64 | do { \ 65 | } while (0) 66 | #define FQ_QUEUE_CREATE_FAILURE_ENABLED() (0) 67 | #define FQ_QUEUE_CREATE_SUCCESS(arg0, arg1, arg2, arg3, arg4, arg5) \ 68 | do { \ 69 | } while (0) 70 | #define FQ_QUEUE_CREATE_SUCCESS_ENABLED() (0) 71 | #define FQ_QUEUE_DESTROY(arg0, arg1) \ 72 | do { \ 73 | } while (0) 74 | #define FQ_QUEUE_DESTROY_ENABLED() (0) 75 | #define FQ_QUEUE_DROP(arg0, arg1) \ 76 | do { \ 77 | } while (0) 78 | #define FQ_QUEUE_DROP_ENABLED() (0) 79 | #define FQ_ROUTE_PROGRAM_ENTRY(arg0, arg1) \ 80 | do { \ 81 | } while (0) 82 | #define FQ_ROUTE_PROGRAM_ENTRY_ENABLED() (0) 83 | #define FQ_ROUTE_PROGRAM_RETURN(arg0, arg1, arg2) \ 84 | do { \ 85 | } while (0) 86 | #define FQ_ROUTE_PROGRAM_RETURN_ENABLED() (0) 87 | 88 | #endif /* !defined(DTRACE_PROBES_DISABLED) || !DTRACE_PROBES_DISABLED */ 89 | -------------------------------------------------------------------------------- /fq_dtrace.d: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | typedef struct { 25 | int dummy; 26 | } fq_dtrace_msg_t; 27 | 28 | typedef struct { 29 | int dummy; 30 | } fq_msg_t; 31 | 32 | typedef struct { 33 | int dummy; 34 | } fq_dtrace_queue_t; 35 | 36 | typedef struct { 37 | int dummy; 38 | } fq_queue_t; 39 | 40 | typedef struct { 41 | int dummy; 42 | } fq_dtrace_remote_client_t; 43 | 44 | typedef struct { 45 | int dummy; 46 | } fq_remote_client_t; 47 | 48 | typedef struct { 49 | int dummy; 50 | } fq_dtrace_remote_anon_client_t; 51 | 52 | typedef struct { 53 | int dummy; 54 | } fq_remote_anon_client_t; 55 | 56 | typedef struct { 57 | int dummy; 58 | } fq_dtrace_remote_data_client_t; 59 | 60 | typedef struct { 61 | int dummy; 62 | } fq_remote_data_client_t; 63 | 64 | provider fq { 65 | probe client__connect(fq_dtrace_remote_anon_client_t *c, int m) : 66 | (fq_remote_anon_client_t *c, int m); 67 | probe client__disconnect(fq_dtrace_remote_anon_client_t *c, int m) : 68 | (fq_remote_anon_client_t *c, int m); 69 | probe client__auth(fq_dtrace_remote_client_t *c) : 70 | (fq_remote_client_t *c); 71 | probe client__auth__data(fq_dtrace_remote_data_client_t *c) : 72 | (fq_remote_data_client_t *c); 73 | probe queue__create__success(int, char *, int, char *, int, int); 74 | probe queue__create__failure(int, char *, char *); 75 | probe queue__destroy(int, char *); 76 | probe queue__drop(fq_dtrace_queue_t *q, fq_dtrace_msg_t *m) : 77 | (fq_queue_t *q, fq_msg_t *m); 78 | probe queue__block(fq_dtrace_queue_t *q, fq_dtrace_msg_t *m) : 79 | (fq_queue_t *q, fq_msg_t *m); 80 | probe queue__enqueue(fq_dtrace_queue_t *q, fq_dtrace_msg_t *m) : 81 | (fq_queue_t *q, fq_msg_t *m); 82 | probe config__rotate(int); 83 | probe message__receive(fq_dtrace_remote_client_t *c, 84 | fq_dtrace_remote_data_client_t *d, 85 | fq_dtrace_msg_t *m) : 86 | (fq_remote_client_t *c, 87 | fq_remote_data_client_t *d, 88 | fq_msg_t *m); 89 | probe message__deliver(fq_dtrace_remote_client_t *c, 90 | fq_dtrace_remote_data_client_t *d, 91 | fq_dtrace_msg_t *m) : 92 | (fq_remote_client_t *c, 93 | fq_remote_data_client_t *d, 94 | fq_msg_t *m); 95 | probe route__program__entry(char *p, fq_dtrace_msg_t *m) : 96 | (char *p, fq_msg_t *m); 97 | probe route__program__return(char *p, fq_dtrace_msg_t *m, int32_t u) : 98 | (char *p, fq_msg_t *m, int32_t u); 99 | }; 100 | 101 | #pragma D attributes Evolving/Evolving/ISA provider fq provider 102 | #pragma D attributes Private/Private/Unknown provider fq module 103 | #pragma D attributes Private/Private/Unknown provider fq function 104 | #pragma D attributes Private/Private/ISA provider fq name 105 | #pragma D attributes Evolving/Evolving/ISA provider fq args 106 | -------------------------------------------------------------------------------- /fq_msg.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "fq.h" 30 | #include "ck_pr.h" 31 | #include "ck_malloc.h" 32 | #include "ck_hs.h" 33 | #include "ck_fifo.h" 34 | 35 | #define MSG_ALIGN sizeof(void *) 36 | #define MAX_FREE_LIST_SIZE 1000000 37 | 38 | #define unlikely(x) __builtin_expect(!!(x), 0) 39 | 40 | static fq_msgid local_msgid = { 41 | .id = { 42 | .u32 = { 43 | .p1 = 0, 44 | .p2 = 0, 45 | .p3 = 0, 46 | .p4 = 0 47 | } 48 | } 49 | }; 50 | 51 | static void 52 | pull_next_local_msgid(fq_msgid *msgid) { 53 | uint32_t last; 54 | fq_msgid g; 55 | again: 56 | memcpy(&g, &local_msgid, sizeof(fq_msgid)); 57 | memcpy(msgid, &g, sizeof(fq_msgid)); 58 | last = ck_pr_faa_32(&local_msgid.id.u32.p1, 1); 59 | msgid->id.u32.p1 = last + 1; 60 | if(last == 0xffffffffUL) { 61 | last = ck_pr_faa_32(&local_msgid.id.u32.p2, 1); 62 | msgid->id.u32.p2 = last + 1; 63 | if(last == 0xffffffffUL) { 64 | last = ck_pr_faa_32(&local_msgid.id.u32.p3, 1); 65 | msgid->id.u32.p3 = last + 1; 66 | if(last == 0xffffffffUL) { 67 | last = ck_pr_faa_32(&local_msgid.id.u32.p4, 1); 68 | msgid->id.u32.p4 = last + 1; 69 | } 70 | } 71 | } 72 | if(msgid->id.u32.p4 < g.id.u32.p4) goto again; 73 | if(msgid->id.u32.p4 > g.id.u32.p4) return; 74 | if(msgid->id.u32.p3 < g.id.u32.p3) goto again; 75 | if(msgid->id.u32.p3 > g.id.u32.p3) return; 76 | if(msgid->id.u32.p2 < g.id.u32.p2) goto again; 77 | if(msgid->id.u32.p2 > g.id.u32.p2) return; 78 | if(msgid->id.u32.p1 > g.id.u32.p1) return; 79 | goto again; 80 | } 81 | 82 | static inline fq_msg* 83 | msg_allocate(const size_t s) 84 | { 85 | fq_msg *m = calloc(1, sizeof(fq_msg) + s); 86 | if(!m) return NULL; 87 | m->payload_len = s; 88 | return m; 89 | } 90 | 91 | static inline void 92 | msg_free(fq_msg *m) 93 | { 94 | if (m->free_fn != NULL) { 95 | m->free_fn(m); 96 | } else { 97 | free(m); 98 | } 99 | } 100 | 101 | fq_msg * 102 | fq_msg_alloc(const void *data, size_t s) { 103 | fq_msg *m = msg_allocate(s); 104 | if (unlikely(m == NULL)) { 105 | return NULL; 106 | } 107 | if(s) memcpy(m->payload, data, s); 108 | #ifdef DEBUG 109 | fq_debug(FQ_DEBUG_MSG, "msg(%p) -> alloc\n", (void *)m); 110 | #endif 111 | m->arrival_time = fq_gethrtime(); 112 | m->refcnt = 1; 113 | return m; 114 | } 115 | 116 | fq_msg * 117 | fq_msg_alloc_BLANK(size_t s) { 118 | fq_msg *m = msg_allocate(s); 119 | if (unlikely(m == NULL)) { 120 | return NULL; 121 | } 122 | #ifdef DEBUG 123 | fq_debug(FQ_DEBUG_MSG, "msg(%p) -> alloc\n", (void *)m); 124 | #endif 125 | m->arrival_time = fq_gethrtime(); 126 | m->refcnt = 1; 127 | return m; 128 | } 129 | 130 | void 131 | fq_msg_ref(fq_msg *msg) { 132 | ck_pr_inc_uint(&msg->refcnt); 133 | #ifdef DEBUG 134 | fq_debug(FQ_DEBUG_MSG, "msg(%p) -> ref: %d\n", (void *)msg, msg->refcnt); 135 | #endif 136 | } 137 | void 138 | fq_msg_deref(fq_msg *msg) { 139 | bool zero; 140 | 141 | ck_pr_dec_uint_zero(&msg->refcnt, &zero); 142 | if(zero) { 143 | #ifdef DEBUG 144 | fq_debug(FQ_DEBUG_MSG, "msg(%p) -> free\n", (void *)msg); 145 | #endif 146 | msg_free(msg); 147 | } 148 | #ifdef DEBUG 149 | else { 150 | fq_debug(FQ_DEBUG_MSG, "msg(%p) -> deref: %d\n", (void *)msg, msg->refcnt); 151 | } 152 | #endif 153 | } 154 | void 155 | fq_msg_exchange(fq_msg *msg, const void *exchange, int rlen) { 156 | if(rlen <= 0) { 157 | msg->exchange.len = 0; 158 | return; 159 | } 160 | if(rlen > MAX_RK_LEN) rlen = MAX_RK_LEN; 161 | msg->exchange.len = rlen; 162 | memcpy(msg->exchange.name, exchange, rlen); 163 | } 164 | void 165 | fq_msg_route(fq_msg *msg, const void *route, int rlen) { 166 | if(rlen <= 0) { 167 | msg->route.len = 0; 168 | return; 169 | } 170 | if(rlen > MAX_RK_LEN) rlen = MAX_RK_LEN; 171 | msg->route.len = rlen; 172 | memcpy(msg->route.name, route, rlen); 173 | } 174 | void 175 | fq_msg_id(fq_msg *msg, fq_msgid *id) { 176 | if(!id) pull_next_local_msgid(&msg->sender_msgid); 177 | else memcpy(&msg->sender_msgid, id, sizeof(*id)); 178 | } 179 | -------------------------------------------------------------------------------- /fq_rcvr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "fq.h" 30 | 31 | char *exchange = "maryland"; 32 | char *program = "prefix:\"\""; 33 | int output = 1; 34 | 35 | void logger(fq_client, const char *); 36 | 37 | void logger(fq_client c, const char *s) { 38 | (void)c; 39 | fprintf(stderr, "fq_logger: %s\n", s); 40 | } 41 | static void 42 | print_rate(fq_client c, hrtime_t s, hrtime_t f, uint64_t cnt, uint64_t icnt) { 43 | double d; 44 | if(cnt) { 45 | d = (double)cnt * 1000000000; 46 | d /= (double)(f-s); 47 | printf("[%d backlog] output %0.2f msg/sec\n", 48 | fq_client_data_backlog(c), d); 49 | } 50 | if(icnt) { 51 | d = (double)icnt * 1000000000; 52 | d /= (double)(f-s); 53 | printf("[%d backlog] input %0.2f msg/sec\n", 54 | fq_client_data_backlog(c), d); 55 | } 56 | } 57 | 58 | 59 | static void 60 | my_auth_handler(fq_client c, int error) { 61 | fq_bind_req *breq; 62 | 63 | if(error) return; 64 | 65 | printf("attempting bind\n"); 66 | breq = malloc(sizeof(*breq)); 67 | memset(breq, 0, sizeof(*breq)); 68 | int exchange_len = strlen(exchange); 69 | memcpy(breq->exchange.name, exchange, exchange_len); 70 | breq->exchange.len = exchange_len; 71 | breq->flags = FQ_BIND_TRANS; 72 | breq->program = program; 73 | fq_client_bind(c, breq); 74 | } 75 | 76 | static void 77 | my_bind_handler(fq_client c, fq_bind_req *breq) { 78 | (void)c; 79 | printf("route set -> %u\n", breq->out__route_id); 80 | if(breq->out__route_id == FQ_BIND_ILLEGAL) { 81 | fprintf(stderr, "Failure to bind...\n"); 82 | exit(-1); 83 | } 84 | } 85 | 86 | fq_hooks hooks = { 87 | .version = FQ_HOOKS_V1, 88 | .auth = my_auth_handler, 89 | .bind = my_bind_handler 90 | }; 91 | 92 | int main(int argc, char **argv) { 93 | hrtime_t s, f; 94 | uint64_t cnt = 0, icnt = 0, icnt_total = 0; 95 | int rcvd = 0; 96 | fq_client c; 97 | fq_msg *m; 98 | 99 | char *fq_debug = getenv("FQ_DEBUG"); 100 | if(fq_debug) fq_debug_set_bits(atoi(fq_debug)); 101 | signal(SIGPIPE, SIG_IGN); 102 | fq_client_init(&c, 0, logger); 103 | if(fq_client_hooks(c, &hooks)) { 104 | fprintf(stderr, "Can't register hooks\n"); 105 | exit(-1); 106 | } 107 | 108 | char *host = "localhost"; 109 | int port = 8765; 110 | char *user = "guest"; 111 | char *pass = "guest"; 112 | int o; 113 | while(-1 != (o = getopt(argc, argv, "h:p:u:P:e:b:s"))) { 114 | switch(o) { 115 | case 'h': host = strdup(optarg); break; 116 | case 'p': port = atoi(optarg); break; 117 | case 'u': user = strdup(optarg); break; 118 | case 'P': pass = strdup(optarg); break; 119 | case 'e': exchange = strdup(optarg); break; 120 | case 'b': program = strdup(optarg); break; 121 | case 's': output = 2; break; 122 | default: 123 | fprintf(stderr, "%s [-h host] [-p port] [-u user] [-P pass] [-e exchange] [-b program] [-s]\n", 124 | argv[0]); 125 | exit(-1); 126 | break; 127 | } 128 | } 129 | fq_client_hooks(c, &hooks); 130 | fq_client_creds(c, host, port, user, pass); 131 | fq_client_heartbeat(c, 1000); 132 | fq_client_set_backlog(c, 10000, 100); 133 | fq_client_connect(c); 134 | 135 | s = fq_gethrtime(); 136 | while(1) { 137 | f = fq_gethrtime(); 138 | while(NULL != (m = fq_client_receive(c))) { 139 | icnt++; 140 | icnt_total++; 141 | rcvd++; 142 | if(output == 1) { 143 | int ending = m->payload[m->payload_len-1] == '\n' ? 1 : 0; 144 | printf("[%.*s] %.*s\n", m->route.len, m->route.name, m->payload_len - ending, m->payload); 145 | } 146 | fq_msg_deref(m); 147 | } 148 | usleep(1000); 149 | if(f-s > 1000000000) { 150 | if(output == 2) { 151 | print_rate(c, s, f, cnt, icnt); 152 | printf("total: %llu\n", (unsigned long long)icnt_total); 153 | } 154 | icnt = 0; 155 | cnt = 0; 156 | s = f; 157 | } 158 | } 159 | (void) argc; 160 | return 0; 161 | } 162 | -------------------------------------------------------------------------------- /fq_sndr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "fq.h" 30 | 31 | #define SEND_COUNT 1000 32 | int send_count = SEND_COUNT; 33 | void logger(fq_client c, const char *); 34 | 35 | void logger(fq_client c, const char *s) { 36 | (void)c; 37 | fprintf(stderr, "fq_logger: %s\n", s); 38 | } 39 | static void 40 | debug_status(char *key, uint32_t value, void *unused) { 41 | (void)unused; 42 | fq_debug(FQ_DEBUG_CONN, " ---> %s : %u\n", key, value); 43 | } 44 | static void 45 | print_rate(fq_client c, hrtime_t s, hrtime_t f, uint64_t cnt, uint64_t icnt) { 46 | double d; 47 | fq_client_status(c, debug_status, NULL); 48 | if(cnt) { 49 | d = (double)cnt * 1000000000; 50 | d /= (double)(f-s); 51 | printf("[%d backlog] output %0.2f msg/sec\n", 52 | fq_client_data_backlog(c), d); 53 | } 54 | if(icnt) { 55 | d = (double)icnt * 1000000000; 56 | d /= (double)(f-s); 57 | printf("[%d backlog] input %0.2f msg/sec\n", 58 | fq_client_data_backlog(c), d); 59 | } 60 | } 61 | int main(int argc, char **argv) { 62 | hrtime_t s0, s, f, f0; 63 | uint64_t cnt = 0, icnt = 0; 64 | int psize = 0, i = 0; 65 | fq_client c; 66 | fq_msg *m; 67 | char *fq_debug = getenv("FQ_DEBUG"); 68 | if(fq_debug) fq_debug_set_bits(atoi(fq_debug)); 69 | signal(SIGPIPE, SIG_IGN); 70 | fq_client_init(&c, 0, logger); 71 | if(argc < 5) { 72 | fprintf(stderr, "%s [size [count]]\n", 73 | argv[0]); 74 | exit(-1); 75 | } 76 | fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]); 77 | fq_client_heartbeat(c, 1000); 78 | fq_client_set_backlog(c, 10000, 100); 79 | fq_client_connect(c); 80 | 81 | if(argc > 5) { 82 | psize = atoi(argv[5]); 83 | } 84 | printf("payload size -> %d\n", psize); 85 | if(argc > 6) { 86 | send_count = atoi(argv[6]); 87 | } 88 | printf("message count -> %d\n", send_count); 89 | 90 | s0 = s = fq_gethrtime(); 91 | while(i < send_count || fq_client_data_backlog(c) > 0) { 92 | if(i < send_count) { 93 | m = fq_msg_alloc_BLANK(psize); 94 | memset(m->payload, 0, psize); 95 | fq_msg_exchange(m, "maryland", 8); 96 | fq_msg_route(m, "test.prefix.boo", 15); 97 | fq_msg_id(m, NULL); 98 | fq_client_publish(c, m); 99 | cnt++; 100 | i++; 101 | fq_msg_free(m); 102 | } 103 | else usleep(100); 104 | 105 | 106 | f = fq_gethrtime(); 107 | if(f-s > 1000000000) { 108 | print_rate(c, s, f, cnt, icnt); 109 | icnt = 0; 110 | cnt = 0; 111 | s = f; 112 | } 113 | } 114 | f0 = fq_gethrtime(); 115 | print_rate(c, s0, f0, i, 0); 116 | (void) argc; 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /fqc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include "fq.h" 30 | 31 | #define SEND_COUNT 1000 32 | int send_count = SEND_COUNT; 33 | void logger(fq_client, const char *); 34 | 35 | void logger(fq_client c, const char *s) { 36 | (void)c; 37 | fprintf(stderr, "fq_logger: %s\n", s); 38 | } 39 | static void 40 | print_rate(fq_client c, hrtime_t s, hrtime_t f, uint64_t cnt, uint64_t icnt) { 41 | double d; 42 | if(cnt) { 43 | d = (double)cnt * 1000000000; 44 | d /= (double)(f-s); 45 | printf("[%d backlog] output %0.2f msg/sec\n", 46 | fq_client_data_backlog(c), d); 47 | } 48 | if(icnt) { 49 | d = (double)icnt * 1000000000; 50 | d /= (double)(f-s); 51 | printf("[%d backlog] input %0.2f msg/sec\n", 52 | fq_client_data_backlog(c), d); 53 | } 54 | } 55 | int main(int argc, char **argv) { 56 | hrtime_t s0, s, f, f0; 57 | uint64_t cnt = 0, icnt = 0; 58 | int psize = 0, i = 0, rcvd = 0; 59 | fq_client c; 60 | fq_bind_req breq; 61 | fq_msg *m; 62 | signal(SIGPIPE, SIG_IGN); 63 | fq_client_init(&c, 0, logger); 64 | if(argc < 5) { 65 | fprintf(stderr, "%s [size [count]]\n", 66 | argv[0]); 67 | exit(-1); 68 | } 69 | fq_client_creds(c, argv[1], atoi(argv[2]), argv[3], argv[4]); 70 | fq_client_heartbeat(c, 1000); 71 | fq_client_set_backlog(c, 10000, 100); 72 | fq_client_connect(c); 73 | 74 | memset(&breq, 0, sizeof(breq)); 75 | memcpy(breq.exchange.name, "maryland", 8); 76 | breq.exchange.len = 8; 77 | breq.flags = 0; 78 | breq.program = (char *)"prefix:\"test.prefix.\""; 79 | 80 | fq_client_bind(c, &breq); 81 | while(ck_pr_load_32(&breq.out__route_id) == 0) usleep(100); 82 | printf("route set -> %u\n", breq.out__route_id); 83 | if(breq.out__route_id == FQ_BIND_ILLEGAL) { 84 | fprintf(stderr, "Failure to bind...\n"); 85 | exit(-1); 86 | } 87 | 88 | if(argc > 5) { 89 | psize = atoi(argv[5]); 90 | if(psize <= 0 || psize > 100000000) { 91 | fprintf(stderr, "invalid size must be > 0 and < 100000000\n"); 92 | exit(-1); 93 | } 94 | } 95 | printf("payload size -> %d\n", psize); 96 | if(argc > 6) { 97 | send_count = atoi(argv[6]); 98 | if(send_count <= 0 || send_count > 10000000) { 99 | fprintf(stderr, "invalid send count must be > 0 and < 10000000\n"); 100 | exit(-1); 101 | } 102 | } 103 | printf("message count -> %d\n", send_count); 104 | 105 | s0 = s = fq_gethrtime(); 106 | while(i < send_count || fq_client_data_backlog(c) > 0) { 107 | if(i < send_count) { 108 | m = fq_msg_alloc_BLANK(psize); 109 | memset(m->payload, 0, psize); 110 | fq_msg_exchange(m, "maryland", 8); 111 | fq_msg_route(m, "test.prefix.foo", 15); 112 | fq_msg_id(m, NULL); 113 | fq_client_publish(c, m); 114 | cnt++; 115 | i++; 116 | fq_msg_free(m); 117 | } 118 | else usleep(100); 119 | 120 | 121 | f = fq_gethrtime(); 122 | while(NULL != (m = fq_client_receive(c))) { 123 | icnt++; 124 | rcvd++; 125 | fq_msg_deref(m); 126 | } 127 | if(f-s > 1000000000) { 128 | print_rate(c, s, f, cnt, icnt); 129 | icnt = 0; 130 | cnt = 0; 131 | s = f; 132 | } 133 | } 134 | f0 = fq_gethrtime(); 135 | print_rate(c, s0, f0, i, 0); 136 | do { 137 | icnt=0; 138 | while(NULL != (m = fq_client_receive(c))) { 139 | icnt++; 140 | rcvd++; 141 | fq_msg_deref(m); 142 | } 143 | } while(rcvd < send_count); 144 | f0 = fq_gethrtime(); 145 | print_rate(c, s0, f0, 0, rcvd); 146 | printf("Total received during test: %d\n", rcvd); 147 | 148 | (void) argc; 149 | return 0; 150 | } 151 | -------------------------------------------------------------------------------- /fqd.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include "getopt.h" 37 | #include "fqd.h" 38 | #include "fqd_private.h" 39 | #ifndef NO_BCD 40 | #include 41 | 42 | static void bcd_signal_handler(int s, siginfo_t *si, void *unused) { 43 | (void)si; 44 | (void)unused; 45 | bcd_fatal("This is a fatal crash"); 46 | signal(s, SIG_DFL); 47 | return; 48 | } 49 | 50 | static void 51 | bcd_setup_sigaction(void) 52 | { 53 | struct sigaction sa; 54 | int signals[] = { 55 | SIGSEGV, 56 | SIGFPE, 57 | SIGABRT, 58 | SIGBUS, 59 | SIGILL, 60 | SIGFPE 61 | }; 62 | 63 | sa.sa_sigaction = bcd_signal_handler; 64 | sa.sa_flags = SA_SIGINFO | SA_ONSTACK; 65 | 66 | for (size_t i = 0; i < sizeof(signals) / sizeof(*signals); i++) { 67 | if (sigaction(signals[i], &sa, NULL) == -1) { 68 | fprintf(stderr, "warning: failed to set signal " 69 | "handler %d\n", signals[i]); 70 | } 71 | } 72 | 73 | return; 74 | } 75 | #else 76 | typedef void *bcd_t; 77 | #endif 78 | 79 | static uint32_t nodeid = 0; 80 | static unsigned short port = 8765; 81 | static int foreground = 0; 82 | static bool usebcd = false; 83 | static int worker_threads = 1; 84 | static char *config_path = NULL; 85 | static char *queue_path = NULL; 86 | static char *libexecdir = NULL; 87 | static bcd_t global_bcd = { 0 }; 88 | 89 | #define die(str) do { \ 90 | fprintf(stderr, "%s: %s\n", str, strerror(errno)); \ 91 | exit(-1); \ 92 | } while(0) 93 | 94 | void fqd_bcd_attach(void) { 95 | if(!usebcd) return; 96 | #ifndef NO_BCD 97 | bcd_error_t error; 98 | if (bcd_attach(&global_bcd, &error) == -1) { 99 | fprintf(stderr, "error: %s\n", 100 | bcd_error_message(&error)); 101 | exit(-1); 102 | } 103 | #endif 104 | } 105 | 106 | static void *listener_thread(void *unused) { 107 | (void)unused; 108 | fqd_start_worker_threads(worker_threads); 109 | fprintf(stderr, "Listening on port: %d\n", port); 110 | fqd_listener(NULL, port); 111 | fqd_stop_worker_threads(); 112 | return NULL; 113 | } 114 | static void usage(const char *prog) { 115 | printf("%s:\n", prog); 116 | printf("\t-h\t\tthis help message\n"); 117 | printf("\t-D\t\trun in the foreground\n"); 118 | printf("\t-B\t\tenable BCD backtrace reporting\n"); 119 | printf("\t-t \tnumber of worker threads to use (default 1)\n"); 120 | printf("\t-n \t\tnode self identifier (IPv4)\n"); 121 | printf("\t-p \tspecify listening port (default: 8765)\n"); 122 | printf("\t-c \tlocation of the configdb\n"); 123 | printf("\t-q \twhere persistent queues are stored\n"); 124 | printf("\t-w \twhere files for web services are available\n"); 125 | printf("\t-v \tprint additional debugging information, by overriding FQ_DEBUG (cf. fq.h)\n"); 126 | printf("\t-l \tuse this dir for relative module loads\n"); 127 | printf("\t-m \tmodule to load\n"); 128 | } 129 | static void parse_cli(int argc, char **argv) { 130 | int c; 131 | char *debug = NULL; 132 | if(getenv("FQ_DEBUG")) { 133 | debug = strdup(getenv("FQ_DEBUG")); 134 | } 135 | libexecdir = strdup(LIBEXECDIR); 136 | while((c = getopt(argc, argv, "Bbl:m:hDt:n:p:q:c:w:v:")) != EOF) { 137 | switch(c) { 138 | case 'B': 139 | usebcd = true; 140 | break; 141 | case 'b': 142 | usebcd = false; 143 | break; 144 | case 'l': 145 | free(libexecdir); 146 | libexecdir = strdup(optarg); 147 | break; 148 | case 'm': 149 | fqd_route_load_module(libexecdir, optarg, ".so"); 150 | break; 151 | case 'q': 152 | free(queue_path); 153 | queue_path = strdup(optarg); 154 | break; 155 | case 'w': 156 | fqd_http_set_root(optarg); 157 | break; 158 | case 'c': 159 | free(config_path); 160 | config_path = strdup(optarg); 161 | break; 162 | case 'D': 163 | foreground = 1; 164 | break; 165 | case 't': 166 | worker_threads = atoi(optarg); 167 | break; 168 | case 'h': 169 | usage(argv[0]); 170 | exit(0); 171 | case 'n': 172 | if(inet_pton(AF_INET, optarg, &nodeid) != 1) { 173 | fprintf(stderr, "Bad argument to -n, must be an IPv4 address.\n"); 174 | exit(-1); 175 | } 176 | if(nodeid == 0 || nodeid == htonl(0x7f000001)) { 177 | fprintf(stderr, "nodeid cannot be INADDR_ANY or loopback\n"); 178 | exit(-1); 179 | } 180 | break; 181 | case 'p': 182 | port = atoi(optarg); 183 | break; 184 | case 'v': 185 | free(debug); 186 | debug = strdup(optarg); 187 | break; 188 | default: 189 | usage(argv[0]); 190 | exit(-1); 191 | } 192 | } 193 | if(debug) fq_debug_set_string(debug); 194 | free(debug); 195 | } 196 | static uint32_t get_my_ip(void) { 197 | uint32_t ip; 198 | struct hostent *h; 199 | char buff[128]; 200 | gethostname(buff, sizeof(buff)); 201 | h = gethostbyname(buff); 202 | if(h && h->h_addrtype == AF_INET && h->h_length == 4) { 203 | memcpy(&ip, h->h_addr_list[0], h->h_length); 204 | if(ip == htonl(0x7f000001)) return 0; 205 | return ip; 206 | } 207 | return 0; 208 | } 209 | int main(int argc, char **argv) { 210 | nodeid = get_my_ip(); 211 | parse_cli(argc,argv); 212 | global_functions_init(libexecdir); 213 | if(nodeid == 0) { 214 | fprintf(stderr, "Could not determine host address, use -n \n"); 215 | exit(-1); 216 | } 217 | signal(SIGPIPE, SIG_IGN); 218 | if(!foreground) { 219 | int pid, fd; 220 | 221 | /* Handle stdin/stdout/stderr */ 222 | fd = open("/dev/null", O_RDONLY); 223 | if(fd < 0 || dup2(fd, STDIN_FILENO) < 0) die("Failed to setup stdin"); 224 | close(fd); 225 | fd = open("/dev/null", O_WRONLY); 226 | if(fd < 0 || dup2(fd, STDOUT_FILENO) < 0 || dup2(fd, STDERR_FILENO) < 0) 227 | die("Failed to setup std{out,err}"); 228 | close(fd); 229 | 230 | /* daemonize */ 231 | pid = fork(); 232 | if(pid < 0) die("Failed to fork"); 233 | if(pid > 0) exit(0); 234 | setsid(); 235 | pid = fork(); 236 | if(pid < 0) die("Failed to fork"); 237 | if(pid > 0) exit(0); 238 | 239 | /* run */ 240 | } 241 | #ifndef NO_BCD 242 | bcd_error_t error; 243 | if(usebcd) { 244 | struct bcd_config config; 245 | 246 | /* Initialize BCD configuration. See bcd.h for options */ 247 | if (bcd_config_init(&config, &error) == -1) 248 | goto fatal; 249 | 250 | /* Initialize the library. */ 251 | if (bcd_init(&config, &error) == -1) 252 | goto fatal; 253 | 254 | /* Initialize a handle to BCD. This should be called by every thread interacting with BCD. */ 255 | if (bcd_attach(&global_bcd, &error) == -1) 256 | goto fatal; 257 | 258 | if (bcd_kv(&global_bcd, "application", "fqd", &error) == -1) 259 | goto fatal; 260 | 261 | if (bcd_kv(&global_bcd, "version", FQ_VERSION, &error) == -1) 262 | goto fatal; 263 | 264 | bcd_setup_sigaction(); 265 | } 266 | #endif 267 | fqd_config_init(nodeid, config_path, queue_path); 268 | listener_thread(NULL); 269 | fprintf(stderr, "Listener thread could not start. Exiting.\n"); 270 | exit(0); 271 | return 0; 272 | 273 | #ifndef NO_BCD 274 | fatal: 275 | fprintf(stderr, "error: %s\n", 276 | bcd_error_message(&error)); 277 | exit(-1); 278 | #endif 279 | } 280 | -------------------------------------------------------------------------------- /fqd_dyn_sample.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014 Circonus, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include "fqd.h" 25 | #include 26 | 27 | bool fqd_route_prog__sample__d(fq_msg *, int, valnode_t *); 28 | bool 29 | fqd_route_prog__sample__d(fq_msg *m, int nargs, valnode_t *args) { 30 | (void)m; 31 | assert(nargs == 1); 32 | assert(args[0].value_type == RP_VALUE_DOUBLE); 33 | if(drand48() < args[0].value.d) return true; 34 | return false; 35 | } 36 | 37 | -------------------------------------------------------------------------------- /fqd_listener.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include "fq.h" 41 | #include "fqd.h" 42 | #include "fqd_private.h" 43 | #include "fq_dtrace.h" 44 | 45 | void 46 | fqd_remote_client_ref(remote_client *r) { 47 | ck_pr_inc_uint(&r->refcnt); 48 | } 49 | bool 50 | fqd_remote_client_deref(remote_client *r) { 51 | bool zero; 52 | ck_pr_dec_uint_zero(&r->refcnt, &zero); 53 | fq_debug(FQ_DEBUG_CONN, "deref client -> %u%s\n", 54 | r->refcnt, zero ? " dropping" : ""); 55 | if(zero) { 56 | close(r->fd); 57 | free(r); 58 | return true; 59 | } 60 | return false; 61 | } 62 | 63 | static void 64 | service_connection(remote_anon_client *client) { 65 | uint32_t cmd; 66 | uint32_t peer_id = 0; 67 | int rv, on = 1; 68 | char buf[40]; 69 | buf[0] = '\0'; 70 | inet_ntop(AF_INET, &client->remote.sin_addr, buf, sizeof(buf)); 71 | fq_thread_setname("fqd:c:%s", client->pretty); 72 | snprintf(client->pretty, sizeof(client->pretty), 73 | "(pre-auth)@%s:%d", buf, ntohs(client->remote.sin_port)); 74 | gettimeofday(&client->connect_time, NULL); 75 | fq_debug(FQ_DEBUG_CONN, "client(%s) connected\n", client->pretty); 76 | 77 | /* We do nothing if this fails. */ 78 | (void)setsockopt(client->fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)); 79 | 80 | while((rv = read(client->fd, &cmd, sizeof(cmd))) == -1 && errno == EINTR); 81 | if(rv != 4) goto disconnect; 82 | if(FQ_CLIENT_DISCONNECT_ENABLED()) { 83 | fq_dtrace_remote_anon_client_t dc; 84 | DTRACE_PACK_ANON_CLIENT(&dc, client); 85 | FQ_CLIENT_CONNECT(&dc, ntohl(cmd)); 86 | } 87 | fq_debug(FQ_DEBUG_CONN, "read(%d) cmd -> %08x\n", client->fd, ntohl(cmd)); 88 | switch(ntohl(cmd)) { 89 | case FQ_PROTO_CMD_MODE: 90 | { 91 | remote_client *newc = calloc(1, sizeof(*newc)); 92 | memcpy(newc, client, sizeof(*client)); 93 | newc->refcnt = 1; 94 | client->fd=-1; 95 | fqd_command_and_control_server(newc); 96 | (void)fqd_remote_client_deref((remote_client *)newc); 97 | } 98 | break; 99 | 100 | case FQ_PROTO_PEER_MODE: 101 | while((rv = read(client->fd, &peer_id, sizeof(peer_id))) == -1 && errno == EINTR); 102 | if(rv != 4) goto disconnect; 103 | /* FALLTHROUGH */ 104 | case FQ_PROTO_OLD_PEER_MODE: 105 | cmd = FQ_PROTO_PEER_MODE; 106 | /* FALLTHROUGH */ 107 | case FQ_PROTO_DATA_MODE: 108 | { 109 | remote_data_client *newc = calloc(1, sizeof(*newc)); 110 | memcpy(newc, client, sizeof(*client)); 111 | newc->mode = ntohl(cmd); 112 | newc->peer_id = peer_id; 113 | newc->refcnt=1; 114 | client->fd=-1; 115 | fqd_data_subscription_server(newc); 116 | (void)fqd_remote_client_deref((remote_client *)newc); 117 | } 118 | break; 119 | 120 | case FQ_PROTO_HTTP_GET: 121 | case FQ_PROTO_HTTP_HEAD: 122 | case FQ_PROTO_HTTP_POST: 123 | case FQ_PROTO_HTTP_PUT: 124 | { 125 | remote_client *newc = calloc(1, sizeof(*newc)); 126 | memcpy(newc, client, sizeof(*client)); 127 | newc->refcnt = 1; 128 | client->fd=-1; 129 | fqd_http_loop(newc, cmd); 130 | (void)fqd_remote_client_deref((remote_client *)newc); 131 | } 132 | break; 133 | 134 | default: 135 | fq_debug(FQ_DEBUG_CONN, "client protocol violation in initial cmd\n"); 136 | close(client->fd); 137 | client->fd = -1; 138 | break; 139 | } 140 | 141 | disconnect: 142 | if(FQ_CLIENT_DISCONNECT_ENABLED()) { 143 | fq_dtrace_remote_anon_client_t dc; 144 | DTRACE_PACK_ANON_CLIENT(&dc, client); 145 | FQ_CLIENT_DISCONNECT(&dc, ntohl(cmd)); 146 | } 147 | 148 | close(client->fd); 149 | free(client); 150 | } 151 | 152 | static void * 153 | conn_handler(void *vc) { 154 | fqd_bcd_attach(); 155 | while(1) { 156 | fq_thread_setname("fqd:c:idle"); 157 | remote_anon_client *client = fqd_ccs_dequeue_work(); 158 | service_connection(client); 159 | } 160 | return NULL; 161 | } 162 | 163 | typedef struct fqd_ccs_work_queue { 164 | remote_anon_client *client; 165 | struct fqd_ccs_work_queue *next; 166 | } fqd_ccs_work_queue_t; 167 | 168 | static pthread_mutex_t fqd_ccs_work_queue_lock; 169 | static pthread_cond_t fqd_ccs_work_queue_cv; 170 | static fqd_ccs_work_queue_t *fqd_ccs_work_queue; 171 | static int fqd_ccs_idle_threads = 0; 172 | 173 | static void 174 | fqd_ccs_enqueue_work(remote_anon_client *client) { 175 | fqd_ccs_work_queue_t *node = calloc(sizeof(*client), 1); 176 | int err; 177 | node->client = client; 178 | err = pthread_mutex_lock(&fqd_ccs_work_queue_lock); 179 | if(err != 0) { 180 | fprintf(stderr, "pthread_mutex_lock: %s\n", strerror(err)); 181 | exit(2); 182 | } 183 | if(fqd_ccs_idle_threads < 1) { 184 | pthread_t client_task; 185 | assert(pthread_create(&client_task, NULL, conn_handler, NULL) == 0); 186 | } 187 | node->next = (fqd_ccs_work_queue_t *)fqd_ccs_work_queue; 188 | fqd_ccs_work_queue = node; 189 | err = pthread_mutex_unlock(&fqd_ccs_work_queue_lock); 190 | if(err != 0) { 191 | fprintf(stderr, "pthread_mutex_unlock: %s\n", strerror(err)); 192 | exit(2); 193 | } 194 | err = pthread_cond_signal(&fqd_ccs_work_queue_cv); 195 | if(err != 0) { 196 | fprintf(stderr, "pthread_cond_signal: %s\n", strerror(err)); 197 | exit(2); 198 | } 199 | } 200 | 201 | remote_anon_client * 202 | fqd_ccs_dequeue_work(void) { 203 | remote_anon_client *client = NULL; 204 | int err; 205 | err = pthread_mutex_lock(&fqd_ccs_work_queue_lock); 206 | if(err != 0) { 207 | fprintf(stderr, "pthread_mutex_lock: %s\n", strerror(err)); 208 | exit(2); 209 | } 210 | fqd_ccs_idle_threads++; 211 | while(fqd_ccs_work_queue == NULL) { 212 | err = pthread_cond_wait(&fqd_ccs_work_queue_cv, &fqd_ccs_work_queue_lock); 213 | if(err != 0) { 214 | fprintf(stderr, "pthread_cond_wait: %s\n", strerror(err)); 215 | exit(2); 216 | } 217 | } 218 | client = fqd_ccs_work_queue->client; 219 | fqd_ccs_work_queue_t *tofree = (fqd_ccs_work_queue_t *)fqd_ccs_work_queue; 220 | fqd_ccs_work_queue = fqd_ccs_work_queue->next; 221 | fqd_ccs_idle_threads--; 222 | err = pthread_mutex_unlock(&fqd_ccs_work_queue_lock); 223 | if(err != 0) { 224 | fprintf(stderr, "pthread_mutex_unlock: %s\n", strerror(err)); 225 | exit(2); 226 | } 227 | free(tofree); 228 | return client; 229 | } 230 | 231 | int 232 | fqd_listener(const char *host, unsigned short port) { 233 | int fd; 234 | remote_anon_client *client = NULL; 235 | unsigned int on = 1; 236 | struct sockaddr_in laddr; 237 | 238 | pthread_mutex_init(&fqd_ccs_work_queue_lock, NULL); 239 | pthread_cond_init(&fqd_ccs_work_queue_cv, NULL); 240 | 241 | memset(&laddr, 0, sizeof(laddr)); 242 | laddr.sin_family = AF_INET; 243 | laddr.sin_addr.s_addr = INADDR_ANY; 244 | if(host && inet_pton(AF_INET, host, &laddr.sin_addr) != 0) { 245 | return -1; 246 | } 247 | laddr.sin_port = htons(port); 248 | 249 | fd = socket(AF_INET, SOCK_STREAM 250 | #ifdef SOCK_CLOEXEC 251 | |SOCK_CLOEXEC 252 | #endif 253 | , 0); 254 | if(fd < 0) return -1; 255 | 256 | if(setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) != 0 || 257 | #ifdef SO_REUSEPORT 258 | setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &on, sizeof(on)) != 0 || 259 | #endif 260 | bind(fd, (struct sockaddr *)&laddr, sizeof(laddr)) < 0 || 261 | listen(fd, 16) < 0) { 262 | close(fd); 263 | return -1; 264 | } 265 | 266 | while(1) { 267 | struct sockaddr_in raddr; 268 | socklen_t raddr_len; 269 | 270 | if(client == NULL) client = calloc(1, sizeof(*client)); 271 | raddr_len = sizeof(raddr); 272 | client->fd = accept(fd, (struct sockaddr *)&client->remote, &raddr_len); 273 | if(client->fd < 0) continue; 274 | fq_keepalive_fd(client->fd, 10, 5, 2); 275 | client->refcnt = 1; 276 | fqd_ccs_enqueue_work(client); 277 | client = NULL; 278 | } 279 | return -1; 280 | } 281 | -------------------------------------------------------------------------------- /fqd_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 Circonus, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #ifndef FQD_PRIVATE_H 25 | #define FQD_PRIVATE_H 26 | 27 | #include "fq.h" 28 | 29 | #define MAX_QUEUE_CLIENTS 16 30 | 31 | struct fqd_route_stats { 32 | /* These are just estimates */ 33 | uint64_t invocations; 34 | uint32_t avg_ns; 35 | uint32_t refcnt; 36 | }; 37 | struct fqd_route_rule { 38 | fq_rk prefix; 39 | int match_maxlen; 40 | char *program; 41 | rulenode_t *compiled_program; 42 | uint32_t route_id; 43 | bool permanent; 44 | int peermode; 45 | fqd_queue *queue; 46 | struct fqd_route_stats *stats; 47 | struct fqd_route_rule *next; 48 | }; 49 | 50 | struct prefix_jumptable { 51 | enum { JUMPTABLE, RULETABLE } tabletype; 52 | struct fqd_route_rule *rules; 53 | struct { 54 | uint64_t pattern; 55 | uint64_t checkbits; 56 | struct prefix_jumptable *jt; 57 | } *pats; 58 | int pat_len; 59 | }; 60 | 61 | struct fqd_route_rules { 62 | struct prefix_jumptable master; 63 | }; 64 | 65 | struct fqd_queue { 66 | fq_rk name; 67 | bool permanent; 68 | bool private; 69 | remote_client *downstream[MAX_QUEUE_CLIENTS]; 70 | /* referenced by: routes and connections */ 71 | queue_policy_t policy; 72 | uint32_t backlog_limit; 73 | uint32_t backlog; 74 | uint32_t dropped_to; 75 | 76 | /* These are only use for FQ_POLICY_BLOCK */ 77 | pthread_cond_t cv; 78 | pthread_mutex_t lock; 79 | 80 | uint32_t refcnt; 81 | fqd_queue_impl *impl; 82 | fqd_queue_impl_data *impl_data; 83 | }; 84 | 85 | extern int 86 | for_each_route_rule_do(struct fqd_route_rules *set, 87 | int (*f)(struct fqd_route_rule *, int, void *), 88 | void *closure); 89 | 90 | void fqd_bcd_attach(void); 91 | void fqd_start_worker_threads(int thread_count); 92 | void fqd_stop_worker_threads(void); 93 | void fqd_routemgr_add_handle(void *); 94 | void global_function_register(const char *name, void (*f)(void)); 95 | void global_functions_init(const char *dir); 96 | void fqd_route_load_module(const char *libexecdir, const char *file, const char *ext); 97 | void fq_thread_setname(const char *format, ...); 98 | remote_anon_client *fqd_ccs_dequeue_work(void); 99 | 100 | #if defined(linux) || defined(__linux) || defined(__linux__) 101 | #if ((__GLIBC__ == 2) && (__GLIBC_MINOR__ < 38)) 102 | static inline size_t strlcpy(char *dst, const char *src, size_t size) 103 | { 104 | if(size > 0) { 105 | strncpy(dst, src, size-1); 106 | dst[size-1] = '\0'; 107 | return size; 108 | } 109 | 110 | dst[0] = '\0'; 111 | return 0; 112 | } 113 | static inline size_t strlcat(char *dst, const char *src, size_t size) 114 | { 115 | int dl = strlen(dst); 116 | int sz = size-dl-1; 117 | 118 | if(sz >= 0) { 119 | strncat(dst, src, sz); 120 | dst[dl+sz] = '\0'; 121 | } 122 | 123 | return dl+strlen(src); 124 | } 125 | #endif /* glibc */ 126 | #endif /* linux */ 127 | 128 | #endif 129 | -------------------------------------------------------------------------------- /fqd_prog.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #define _GNU_SOURCE 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include "fqd.h" 32 | #include "fqd_private.h" 33 | 34 | bool fqd_route_prog__true__(fq_msg *, int, valnode_t *); 35 | bool fqd_route_prog__route_contains__s(fq_msg *, int, valnode_t *); 36 | bool fqd_route_prog__payload_prefix__s(fq_msg *, int, valnode_t *); 37 | 38 | void fqd_route_load_module(const char *libexecdir, const char *file, const char *ext) { 39 | char path[PATH_MAX]; 40 | if(*file != '/') { 41 | snprintf(path, sizeof(path), "%s/%s%s", libexecdir, file, ext ? ext : ""); 42 | file = path; 43 | } 44 | void *handle = dlopen(file, RTLD_NOW|RTLD_GLOBAL); 45 | if(handle == NULL) { 46 | fprintf(stderr, "Failed to load %s: %s\n", file, dlerror()); 47 | } 48 | fqd_routemgr_add_handle(handle); 49 | } 50 | 51 | void global_functions_init(const char *libexecdir) { 52 | #define GFR(a) global_function_register(#a, (void (*)(void))a) 53 | GFR(fqd_route_prog__true__); 54 | GFR(fqd_route_prog__route_contains__s); 55 | GFR(fqd_route_prog__payload_prefix__s); 56 | #undef GFR 57 | 58 | DIR *dir = opendir(libexecdir); 59 | if(!dir) return; 60 | struct dirent *de; 61 | while(NULL != (de = readdir(dir))) { 62 | char path[PATH_MAX]; 63 | struct stat sb; 64 | int namelen = strlen(de->d_name); 65 | if(namelen < 4 || memcmp(de->d_name + namelen - 3, ".so", 3)) continue; 66 | snprintf(path, sizeof(path), "%s/%s", libexecdir, de->d_name); 67 | if(stat(path, &sb) == -1) continue; 68 | if((sb.st_mode & S_IFMT) == S_IFREG || (sb.st_mode & S_IFMT) == S_IFLNK) { 69 | fqd_route_load_module(libexecdir, de->d_name, NULL); 70 | } 71 | } 72 | closedir(dir); 73 | } 74 | 75 | bool fqd_route_prog__true__(fq_msg *m, int nargs, valnode_t *args) { 76 | fq_assert(nargs == 0); 77 | (void)m; 78 | (void)nargs; 79 | (void)args; 80 | return true; 81 | } 82 | 83 | bool 84 | fqd_route_prog__route_contains__s(fq_msg *m, int nargs, valnode_t *args) { 85 | int flen; 86 | fq_assert(nargs == 1); 87 | fq_assert(args[0].value_type == RP_VALUE_STRING); 88 | flen = strlen(args[0].value.s); 89 | if(flen > m->route.len) return false; 90 | return memmem(m->route.name, m->route.len, args[0].value.s, flen) != NULL; 91 | } 92 | 93 | bool 94 | fqd_route_prog__payload_prefix__s(fq_msg *m, int nargs, valnode_t *args) { 95 | uint32_t flen; 96 | fq_assert(nargs == 1); 97 | fq_assert(args[0].value_type == RP_VALUE_STRING); 98 | flen = strlen(args[0].value.s); 99 | if(flen > m->payload_len) return false; 100 | if(memcmp(args[0].value.s, m->payload, flen) == 0) 101 | return true; 102 | return false; 103 | } 104 | 105 | bool 106 | fqd_route_prog__payload_contains__s(fq_msg *m, int nargs, valnode_t *args) { 107 | int flen; 108 | fq_assert(nargs == 1); 109 | fq_assert(args[0].value_type == RP_VALUE_STRING); 110 | flen = strlen(args[0].value.s); 111 | if(flen > m->payload_len) return false; 112 | return memmem(m->payload, m->payload_len, args[0].value.s, flen) != NULL; 113 | } 114 | -------------------------------------------------------------------------------- /fqd_queue_mem.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include "fqd.h" 25 | #include 26 | #include 27 | 28 | struct queue_mem { 29 | uint32_t qlen; 30 | ck_fifo_spsc_t q; 31 | ck_fifo_spsc_entry_t *qhead; 32 | }; 33 | 34 | static void queue_mem_enqueue(fqd_queue_impl_data f, fq_msg *m) { 35 | struct queue_mem *d = (struct queue_mem *)f; 36 | ck_fifo_spsc_enqueue_lock(&d->q); 37 | ck_fifo_spsc_entry_t *fifo_entry = ck_fifo_spsc_recycle(&d->q); 38 | if (fifo_entry == NULL) { 39 | fifo_entry = malloc(sizeof(ck_fifo_spsc_entry_t)); 40 | } 41 | fq_msg_ref(m); 42 | ck_fifo_spsc_enqueue(&d->q, fifo_entry, m); 43 | ck_fifo_spsc_enqueue_unlock(&d->q); 44 | ck_pr_inc_uint(&d->qlen); 45 | } 46 | static fq_msg *queue_mem_dequeue(fqd_queue_impl_data f) { 47 | struct queue_mem *d = (struct queue_mem *)f; 48 | fq_msg *m = NULL; 49 | ck_fifo_spsc_dequeue_lock(&d->q); 50 | if(ck_fifo_spsc_dequeue(&d->q, &m) == true) { 51 | ck_fifo_spsc_dequeue_unlock(&d->q); 52 | ck_pr_dec_uint(&d->qlen); 53 | return m; 54 | } 55 | ck_fifo_spsc_dequeue_unlock(&d->q); 56 | return NULL; 57 | } 58 | static fqd_queue_impl_data queue_mem_setup(fq_rk *qname, uint32_t *count) { 59 | struct queue_mem *d; 60 | d = calloc(1, sizeof(*d)); 61 | d->qhead = malloc(sizeof(ck_fifo_spsc_entry_t)); 62 | *count = 0; 63 | ck_fifo_spsc_init(&d->q, d->qhead); 64 | (void)qname; 65 | return d; 66 | } 67 | static void queue_mem_dispose(fq_rk *qname, fqd_queue_impl_data f) { 68 | struct queue_mem *d = (struct queue_mem *)f; 69 | fq_msg *m; 70 | (void)qname; 71 | while(NULL != (m = queue_mem_dequeue(d))) { 72 | fq_msg_deref(m); 73 | } 74 | ck_fifo_spsc_entry_t *garbage = NULL; 75 | ck_fifo_spsc_deinit(&d->q, &garbage); 76 | while (garbage != NULL) { 77 | ck_fifo_spsc_entry_t *n = garbage->next; 78 | free(garbage); 79 | garbage = n; 80 | } 81 | free(d); 82 | } 83 | 84 | /* not supported for now */ 85 | static int queue_mem_add_checkpoint(fqd_queue_impl_data data, const char *name, const fq_msgid *id) { 86 | return -1; 87 | } 88 | 89 | /* not supported for now */ 90 | static int queue_mem_remove_checkpoint(fqd_queue_impl_data data, const char *name) { 91 | return -1; 92 | } 93 | 94 | /* not supported for now */ 95 | static int queue_mem_reset_to_checkpoint(fqd_queue_impl_data data, const char *name) { 96 | return -1; 97 | } 98 | 99 | fqd_queue_impl fqd_queue_mem_impl = { 100 | .name = "mem", 101 | .setup = queue_mem_setup, 102 | .enqueue = queue_mem_enqueue, 103 | .dequeue = queue_mem_dequeue, 104 | .dispose = queue_mem_dispose, 105 | .add_checkpoint = queue_mem_add_checkpoint, 106 | .remove_checkpoint = queue_mem_remove_checkpoint, 107 | .reset_checkpoint = queue_mem_reset_to_checkpoint 108 | }; 109 | -------------------------------------------------------------------------------- /fqs.c: -------------------------------------------------------------------------------- 1 | /* 2 | * fqs 3 | * 4 | * fqs reads messages from stdin, one pre line, and sends it to the specified fq exchange. 5 | * 6 | */ 7 | 8 | #define _GNU_SOURCE 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "fq.h" 16 | 17 | void logger(fq_client c, const char *s) { 18 | fprintf(stderr, "fq_logger: %s\n", s); 19 | } 20 | 21 | static void usage(const char *prog) { 22 | printf("%s:\n", prog); 23 | printf("\t-h \tshow this help message\n"); 24 | printf("\t-a :\tspecify the address to connect to (default: 127.0.0.1:8765)\n"); 25 | printf("\t-x \tspecify the exchange to relay messages on (required)\n"); 26 | printf("\t-r \tspecify the message route (required)\n"); 27 | printf("\t-u \tspecify the user (default: user)\n"); 28 | printf("\t-p \tspecify the password (default: pass)\n"); 29 | } 30 | 31 | int main(int argc, char **argv) { 32 | char *host = NULL, *exchange = NULL, *route = NULL; 33 | char *address = strdup("127.0.0.1:8765"); 34 | char *user = "user"; 35 | char *pass = "pass"; 36 | int port = 0; 37 | int c = 0; 38 | if (argc == 1) { 39 | usage(argv[0]); 40 | exit(-1); 41 | } 42 | while((c = getopt(argc, argv, "ha:x:r:u:p:")) != EOF) { 43 | switch(c) { 44 | case 'h': 45 | usage(argv[0]); 46 | exit(0); 47 | break; 48 | case 'a': 49 | address = strdup(optarg); 50 | break; 51 | case 'x': 52 | exchange = strdup(optarg); 53 | break; 54 | case 'r': 55 | route = strdup(optarg); 56 | break; 57 | case 'u': 58 | user = strdup(optarg); 59 | break; 60 | case 'p': 61 | pass = strdup(optarg); 62 | break; 63 | default: 64 | usage(argv[0]); 65 | exit(-1); 66 | } 67 | } 68 | // Separate host and port in address string 69 | // We have strdup'ed address so we can mutate it 70 | host = address; 71 | for (char *p = address; *p != 0; p++) { 72 | if (*p == ':') { 73 | *p = 0; 74 | port = strtol(p+1, NULL, 10); 75 | break; 76 | } 77 | } 78 | if (!port) { 79 | printf("Illegal port specification\n"); 80 | exit(-1); 81 | } 82 | if (!exchange) { 83 | printf("Exchange argument required"); 84 | exit(-1); 85 | } 86 | if (!route) { 87 | printf("Route argument required"); 88 | exit(-1); 89 | } 90 | 91 | fq_client cli; 92 | fq_msg *m = NULL; 93 | size_t exchange_len = strlen(exchange); 94 | size_t route_len = strlen(route); 95 | signal(SIGPIPE, SIG_IGN); // ignore SIGPIPE 96 | fq_client_init(&cli, 0, logger); 97 | fq_client_creds(cli, host, port, user, pass); 98 | fq_client_heartbeat(cli, 1000); 99 | fq_client_set_backlog(cli, 10000, 100); 100 | fq_client_connect(cli); 101 | while(true) { 102 | char *line = NULL; 103 | size_t line_cap = 0; 104 | int line_len; 105 | line_len = getline(&line, &line_cap, stdin); 106 | if(line_len < 0) { 107 | // wait for queues to drain 108 | while(fq_client_data_backlog(cli) > 0) { 109 | usleep(100); 110 | } 111 | exit(0); 112 | } 113 | m = fq_msg_alloc(line, line_len); 114 | fq_msg_exchange(m, exchange, exchange_len); 115 | fq_msg_route(m, route, route_len); 116 | fq_msg_id(m, NULL); 117 | fq_client_publish(cli, m); 118 | fq_msg_free(m); 119 | } 120 | return 0; 121 | } 122 | -------------------------------------------------------------------------------- /fqtool.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014 Circonus, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include "getopt.h" 36 | #include "fq.h" 37 | 38 | int usage(const char *); 39 | void logger(fq_client, const char *); 40 | 41 | void logger(fq_client c, const char *s) { 42 | (void)c; 43 | fprintf(stderr, "fq_logger: %s\n", s); 44 | } 45 | 46 | int permanent = 1; 47 | int binding_trans = 1; 48 | char *binding_prog = NULL; 49 | char *exchange = NULL; 50 | char *output = NULL; 51 | char *format = "payload"; 52 | 53 | enum { 54 | M_ROUTE = 0, 55 | M_HOST, 56 | M_PAYLOAD, 57 | M_NONE 58 | } o_fmt[M_NONE]; 59 | #define M_COUNT M_NONE 60 | int output_fd = -1; 61 | bool inject_json = true; 62 | 63 | static void 64 | my_auth_handler(fq_client c, int error) { 65 | fq_bind_req *breq; 66 | 67 | if(error) { 68 | fprintf(stderr, "Error encoutered.\n"); 69 | exit(-1); 70 | } 71 | 72 | if(!output) 73 | fprintf(stderr, "Queue made %s.\n", permanent ? "permanent" : "transient"); 74 | if(!binding_prog && !output) exit(0); 75 | 76 | breq = malloc(sizeof(*breq)); 77 | memset(breq, 0, sizeof(*breq)); 78 | assert(strlen(exchange) < sizeof(breq->exchange.name)); 79 | memcpy(breq->exchange.name, exchange, strlen(exchange)); 80 | breq->exchange.len = strlen(exchange); 81 | breq->flags = binding_trans ? FQ_BIND_TRANS : FQ_BIND_PERM; 82 | breq->program = binding_prog; 83 | fq_client_bind(c, breq); 84 | } 85 | 86 | static void 87 | my_bind_handler(fq_client c, fq_bind_req *breq) { 88 | (void)c; 89 | if(breq->out__route_id == FQ_BIND_ILLEGAL) { 90 | fprintf(stderr, "Failure to bind...\n"); 91 | exit(-1); 92 | } 93 | if(!output) { 94 | fprintf(stderr, "Binding made %s.\n", binding_trans ? "transient" : "permanent"); 95 | exit(0); 96 | } 97 | } 98 | 99 | static bool 100 | my_message_handler(fq_client c, fq_msg *m) { 101 | int i; 102 | char *payload = (char *)m->payload; 103 | long payload_len = m->payload_len; 104 | char host[32] = ""; 105 | bool lineend = true; 106 | bool started = false; 107 | if(payload_len > 0 && payload[payload_len-1] == '\n') { 108 | payload_len--; 109 | } 110 | 111 | struct in_addr addr; 112 | for(i=0;ihops[i+1] == 0 || /* IN ADDR ANY */ 114 | m->hops[i+1] == 0xffffffff || /* IN ADDR ANY */ 115 | (ntohl(m->hops[i+1]) & 0xff000000) == 0x7f000000) /* LOCALHOST */ 116 | break; 117 | memcpy(&addr, m->hops+i, sizeof(unsigned int)); 118 | snprintf(host, sizeof(host), "%s", inet_ntoa(addr)); 119 | 120 | #define OUTSTART() do { \ 121 | if(started) write(output_fd, " ", 1); started = true; \ 122 | } while(0) 123 | for(i=0;iroute.name, m->route.len); 133 | break; 134 | case M_PAYLOAD: 135 | OUTSTART(); 136 | if(inject_json && payload[0] == '{' && payload[payload_len-1] == '}') { 137 | struct iovec iov[8]; 138 | write(output_fd, payload, payload_len-1); 139 | iov[0].iov_base = "\"_host\":\""; iov[0].iov_len = 9; 140 | if(payload_len > 2) { 141 | iov[0].iov_base = ",\"_host\":\""; iov[0].iov_len = 10; 142 | } 143 | iov[1].iov_base = host; iov[1].iov_len = strlen(host); 144 | iov[2].iov_base = "\""; iov[2].iov_len = 1; 145 | writev(output_fd, iov, 3); 146 | iov[0].iov_base = ",\"_route\":\""; iov[0].iov_len = 11; 147 | iov[1].iov_base = m->route.name; iov[1].iov_len = m->route.len; 148 | iov[2].iov_base = "\""; iov[2].iov_len = 1; 149 | writev(output_fd, iov, 3); 150 | write(output_fd, "}", 1); 151 | } 152 | else { 153 | write(output_fd, payload, payload_len); 154 | } 155 | break; 156 | } 157 | } 158 | if(started && lineend) write(output_fd, "\n", 1); 159 | return true; 160 | } 161 | 162 | fq_hooks hooks = { 163 | .version = FQ_HOOKS_V4, 164 | .auth = my_auth_handler, 165 | .bind = my_bind_handler, 166 | .message = my_message_handler 167 | }; 168 | 169 | int usage(const char *prog) { 170 | printf("%s [-h host] [-P port] [-u user] [-p pass]\n", prog); 171 | printf("\t<-e name> <-q name> [-t ] [-D]\n"); 172 | printf("\t[-(b|B) 'program'] <-d backlog>\n"); 173 | printf("\n"); 174 | printf("\t-h \t\tdefault: localhost\n"); 175 | printf("\t-e \t\texchange name\n"); 176 | printf("\t-P \t\tdefault: 8765\n"); 177 | printf("\t-u \t\tdefault: nobody\n"); 178 | printf("\t-p \t\tdefault: nopass\n"); 179 | printf("\t-q \t\tqueue name\n"); 180 | printf("\t-t \t\tqueue type\n"); 181 | printf("\t-d \t\t\tdefine queue depth (backlog)\n"); 182 | printf("\t-D\t\t\tdelete queue (make transient)\n"); 183 | printf("\t-B \t\tinstall binding\n"); 184 | printf("\t-b \t\tuninstall binding\n"); 185 | printf("\t-o \t\toutput file (- for stdout)\n"); 186 | return 0; 187 | } 188 | 189 | int main(int argc, char **argv) { 190 | int opt; 191 | char *user = (char *)"nobody"; 192 | char *pass = (char *)"nopass"; 193 | char *host = (char *)"localhost"; 194 | int port = 8765; 195 | char *queuename = NULL; 196 | char *qtype = (char *)"mem"; 197 | int backlog = -1; 198 | char connstr[256]; 199 | fq_client c; 200 | 201 | while((opt = getopt(argc, argv, "Jo:f:Dh:e:P:u:p:q:t:B:b:d:")) != EOF) { 202 | switch(opt) { 203 | case 'o': output = strdup(optarg); break; 204 | case 'f': format = strdup(optarg); break; 205 | case 'e': exchange = strdup(optarg); break; 206 | case 'h': host = strdup(optarg); break; 207 | case 'P': port = atoi(optarg); break; 208 | case 'u': user = strdup(optarg); break; 209 | case 'p': pass = strdup(optarg); break; 210 | case 'q': queuename = strdup(optarg); break; 211 | case 't': qtype = strdup(optarg); break; 212 | case 'D': permanent = 0; break; 213 | case 'B': binding_prog = strdup(optarg); 214 | binding_trans = 0; break; 215 | case 'b': binding_prog = strdup(optarg); 216 | binding_trans = 1; break; 217 | case 'd': backlog = atoi(optarg); break; 218 | case 'J': inject_json = false; break; 219 | default: exit(usage(argv[0])); 220 | } 221 | } 222 | if(format) { 223 | char *word, *brkt; 224 | int o_i = 0; 225 | for (word = strtok_r(format, ",", &brkt); 226 | word && o_i < M_COUNT; 227 | word = strtok_r(NULL, ",", &brkt)) { 228 | if(!strcmp(word, "host")) o_fmt[o_i++] = M_HOST; 229 | else if(!strcmp(word, "route")) o_fmt[o_i++] = M_ROUTE; 230 | else if(!strcmp(word, "payload")) o_fmt[o_i++] = M_PAYLOAD; 231 | else { 232 | fprintf(stderr, "Error: unknown format '%s'\n", word); 233 | exit(-2); 234 | } 235 | } 236 | for(;o_i "); 40 | System.exit(-1); 41 | } 42 | System.err.println(args[0]); 43 | FqClient client = null; 44 | FqTest impl = new FqTest(); 45 | try { 46 | client = new FqClient(impl); 47 | client.creds(args[0], new Integer(args[1]), args[2], args[3]); 48 | client.connect(); 49 | } catch(Exception e) { 50 | throw new RuntimeException(e); 51 | } 52 | 53 | while(true) { 54 | client.send(new FqCommand.StatusRequest()); 55 | try { Thread.sleep(1000); } catch(InterruptedException ignore) { } 56 | } 57 | 58 | //if(client != null) client.shutdown(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /java/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.circonus 4 | fqclient 5 | 0.0.1 6 | jar 7 | 8 | Fq Java Client 9 | Fq Client for Java 10 | https://github.com/circonus-labs/fq 11 | 12 | 13 | 14 | BSD 15 | https://github.com/circonus-labs/fq/blob/master/LICENSE 16 | repo 17 | 18 | 19 | 20 | 21 | https://github.com/circonus-labs/fq 22 | 23 | 24 | 25 | postwait 26 | Theo Schlossnagle 27 | theo.schlossnagle (at) circonus.com 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-compiler-plugin 36 | 2.3.1 37 | 38 | 1.5 39 | 1.5 40 | 41 | 42 | 43 | org.apache.maven.plugins 44 | maven-deploy-plugin 45 | 2.7 46 | 47 | 48 | org.apache.maven.plugins 49 | maven-source-plugin 50 | 2.3 51 | 52 | 53 | attach-sources 54 | 55 | jar-no-fork 56 | 57 | 58 | 59 | 60 | 61 | org.apache.maven.plugins 62 | maven-javadoc-plugin 63 | 2.10.1 64 | 65 | 66 | attach-javadocs 67 | 68 | jar 69 | 70 | 71 | -Xdoclint:none 72 | 73 | 74 | 75 | 76 | 77 | org.sonatype.plugins 78 | nexus-staging-maven-plugin 79 | 1.6.3 80 | true 81 | 82 | ossrh 83 | https://oss.sonatype.org/ 84 | false 85 | 86 | 87 | 88 | org.apache.maven.plugins 89 | maven-gpg-plugin 90 | 1.5 91 | 92 | 93 | sign-artifacts 94 | verify 95 | 96 | sign 97 | 98 | 99 | 100 | 101 | 102 | org.apache.maven.plugins 103 | maven-release-plugin 104 | 2.5 105 | 106 | true 107 | false 108 | release 109 | deploy 110 | 111 | 112 | 113 | 114 | 115 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqClientImplDebug.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | import java.util.Date; 27 | import java.util.Map; 28 | 29 | public class FqClientImplDebug implements FqClientImplInterface { 30 | protected FqClient client = null; 31 | public void setClient(FqClient c) throws InUseException { 32 | if(client != null) throw new InUseException(); 33 | client = c; 34 | } 35 | protected void genericError(Throwable e) { 36 | e.printStackTrace(); 37 | } 38 | public void connectError(Throwable e) { genericError(e); } 39 | public void commandError(Throwable e) { genericError(e); } 40 | public void dataError(Throwable e) { genericError(e); } 41 | public void dispatch(FqMessage m) { 42 | byte b[] = m.getPayload(); 43 | int len = (b == null) ? 0 : b.length; 44 | System.err.println("m[" + len + "] via " + m.getRoute() + 45 | " over " + m.getExchange() + " from " + m.getSender()); 46 | } 47 | public void dispatch(FqCommand cmd) { 48 | System.err.println(cmd); 49 | } 50 | public void dispatchAuth(FqCommand.Auth cmd) { 51 | dispatch(cmd); 52 | } 53 | public void dispatchHeartbeatRequest(FqCommand.HeartbeatRequest cmd) { 54 | dispatch(cmd); 55 | } 56 | public void dispatchHeartbeat(FqCommand.Heartbeat cmd) { dispatch(cmd); } 57 | public void dispatchBindRequest(FqCommand.BindRequest cmd) { 58 | System.err.println(cmd.toString() + cmd.getBinding()); 59 | } 60 | public void dispatchUnbindRequest(FqCommand.UnbindRequest cmd) { 61 | System.err.println(cmd.toString() + cmd.getBinding() + " " + cmd.getSuccess()); 62 | } 63 | public void dispatchStatusRequest(FqCommand.StatusRequest cmd) { 64 | Date d = cmd.getDate(); 65 | Map m = cmd.getMap(); 66 | System.err.println("Status: " + d); 67 | for(Map.Entry entry : m.entrySet()) { 68 | System.err.println(" " + entry.getKey() + " : " + entry.getValue()); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqClientImplInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | import com.omniti.labs.FqCommand; 27 | 28 | public interface FqClientImplInterface { 29 | public class InUseException extends Exception { } 30 | public void setClient(FqClient c) throws InUseException; 31 | public void connectError(Throwable e); 32 | public void commandError(Throwable e); 33 | public void dataError(Throwable e); 34 | public void dispatch(FqMessage m); 35 | public void dispatch(FqCommand cmd); 36 | public void dispatchAuth(FqCommand.Auth cmd); 37 | public void dispatchHeartbeat(FqCommand.Heartbeat cmd); 38 | public void dispatchHeartbeatRequest(FqCommand.HeartbeatRequest cmd); 39 | public void dispatchBindRequest(FqCommand.BindRequest cmd); 40 | public void dispatchUnbindRequest(FqCommand.UnbindRequest cmd); 41 | public void dispatchStatusRequest(FqCommand.StatusRequest cmd); 42 | } 43 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqClientImplNoop.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | public class FqClientImplNoop implements FqClientImplInterface { 27 | protected FqClient client; 28 | public void setClient(FqClient c) throws InUseException { 29 | if(client != null) throw new InUseException(); 30 | client = c; 31 | } 32 | protected void genericError(Throwable e) { } 33 | public void connectError(Throwable e) { genericError(e); } 34 | public void commandError(Throwable e) { genericError(e); } 35 | public void dataError(Throwable e) { genericError(e); } 36 | public void dispatch(FqMessage m) { } 37 | public void dispatch(FqCommand cmd) { } 38 | public void dispatchAuth(FqCommand.Auth cmd) { dispatch(cmd); } 39 | public void dispatchHeartbeatRequest(FqCommand.HeartbeatRequest cmd) { 40 | dispatch(cmd); 41 | } 42 | public void dispatchHeartbeat(FqCommand.Heartbeat cmd) { dispatch(cmd); } 43 | public void dispatchBindRequest(FqCommand.BindRequest cmd) { dispatch(cmd); }; 44 | public void dispatchUnbindRequest(FqCommand.UnbindRequest cmd) { dispatch(cmd); }; 45 | public void dispatchStatusRequest(FqCommand.StatusRequest cmd) { dispatch(cmd); }; 46 | } 47 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | import java.io.IOException; 27 | import java.net.Socket; 28 | import java.nio.ByteBuffer; 29 | import java.nio.ByteOrder; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.Date; 32 | import java.util.HashMap; 33 | import java.util.Map; 34 | import com.omniti.labs.FqClient; 35 | 36 | public abstract class FqCommand { 37 | public final static short FQ_PROTO_ERROR = (short)0xeeee; 38 | public final static short FQ_PROTO_HB = (short)0xbea7; 39 | public final static short FQ_PROTO_AUTH_CMD = (short)0xaaaa; 40 | public final static short FQ_PROTO_AUTH_PLAIN = (short)0; 41 | public final static short FQ_PROTO_AUTH_RESP = (short)0xaa00; 42 | public final static short FQ_PROTO_HBREQ = (short)0x4848; 43 | public final static short FQ_PROTO_BIND = (short)0xb171; 44 | public final static short FQ_PROTO_BINDREQ = (short)0xb170; 45 | public final static short FQ_PROTO_UNBIND = (short)0x171b; 46 | public final static short FQ_PROTO_UNBINDREQ = (short)0x071b; 47 | public final static short FQ_PROTO_STATUS = (short)0x57a7; 48 | public final static short FQ_PROTO_STATUSREQ = (short)0xc7a7; 49 | 50 | protected ByteBuffer bb; 51 | private boolean composed = false; 52 | public abstract short cmd(); 53 | public short response_cmd() { return cmd(); } 54 | public void compose() { } 55 | public void send(FqClient c) throws IOException { 56 | if(composed) { 57 | bb.position(0); 58 | } else { 59 | bb.putShort(cmd()); 60 | compose(); 61 | bb.flip(); 62 | composed = true; 63 | } 64 | c.cmd_write(bb); 65 | } 66 | public abstract boolean hasInBandResponse(); 67 | private static Heartbeat hb = new Heartbeat(); 68 | 69 | public Short getShortCmd(FqClient c) 70 | throws IOException, FqCommandProtocolError { 71 | Short cmd; 72 | do { 73 | ByteBuffer bb = c.cmd_read(2); 74 | if(bb == null) return null; 75 | bb.flip(); 76 | cmd = bb.getShort(); 77 | if(cmd == FQ_PROTO_HB) { c.recvHeartbeat(); } 78 | if(cmd == FQ_PROTO_ERROR) { 79 | throw new FqCommandProtocolError(c.cmd_read_short_string()); 80 | } 81 | } while(cmd() != FQ_PROTO_HB && cmd == FQ_PROTO_HB); 82 | return cmd; 83 | } 84 | public void process(FqClient c) throws IOException, FqCommandProtocolError { 85 | Short cmd = getShortCmd(c); 86 | // the hearbeat happens magically in getShortCmd 87 | if(cmd == null) { 88 | throw new FqCommandProtocolError("null cmd"); 89 | } 90 | else if(cmd != response_cmd()) { 91 | throw new FqCommandProtocolError(response_cmd(), cmd); 92 | } 93 | } 94 | 95 | protected void alloc(int size) { 96 | bb = ByteBuffer.allocate(size); 97 | bb.order(ByteOrder.BIG_ENDIAN); 98 | } 99 | public FqCommand(int size) { 100 | alloc(size + 2); 101 | } 102 | public FqCommand() { 103 | } 104 | 105 | public static class Heartbeat extends FqCommand { 106 | public Heartbeat() { super(0); } 107 | public boolean hasInBandResponse() { return false; } 108 | public short cmd() { return FQ_PROTO_HB; } 109 | } 110 | public static class HeartbeatRequest extends FqCommand { 111 | short ms; 112 | public HeartbeatRequest(int _ms) { 113 | super(2); 114 | ms = (short)(_ms & 0xffff); 115 | } 116 | public boolean hasInBandResponse() { return false; } 117 | public short cmd() { return FQ_PROTO_HBREQ; } 118 | public void compose() { bb.putShort(ms); } 119 | } 120 | public static abstract class Auth extends FqCommand { 121 | protected byte[] key = null; 122 | public boolean success() { return (key != null); } 123 | public byte[] getKey() { return key; } 124 | } 125 | public static class PlainAuth extends Auth { 126 | private byte b_user[]; 127 | private byte b_pass[]; 128 | private byte b_queue[]; 129 | private byte b_queue_type[]; 130 | 131 | public PlainAuth(String user, String pass, 132 | String queue, String queue_type) { 133 | b_user = user.getBytes(StandardCharsets.UTF_8); 134 | b_queue = queue.getBytes(StandardCharsets.UTF_8); 135 | b_queue_type = queue_type.getBytes(StandardCharsets.UTF_8); 136 | b_pass = pass.getBytes(StandardCharsets.UTF_8); 137 | int extra_space = 138 | 2 + /* plain */ 139 | 2 + b_user.length + /* user */ 140 | 2 + b_queue.length + 1 + b_queue_type.length + /* queue */ 141 | 2 + b_pass.length; 142 | alloc(2+extra_space); 143 | } 144 | public short cmd() { return FQ_PROTO_AUTH_CMD; } 145 | public boolean hasInBandResponse() { return true; } 146 | public void compose() { 147 | bb.putShort(FQ_PROTO_AUTH_PLAIN); 148 | bb.putShort((short)b_user.length); 149 | bb.put(b_user); 150 | bb.putShort((short)(b_queue.length + 1 + b_queue_type.length)); 151 | bb.put(b_queue); 152 | bb.put((byte) 0); 153 | bb.put(b_queue_type); 154 | bb.putShort((short)b_pass.length); 155 | bb.put(b_pass); 156 | } 157 | public void process(FqClient c) throws IOException, FqCommandProtocolError { 158 | Short cmd, len; 159 | bb = c.cmd_read(2); 160 | if(bb == null) return; 161 | bb.flip(); 162 | cmd = bb.getShort(); 163 | switch(cmd) { 164 | case FQ_PROTO_AUTH_RESP: 165 | key = c.cmd_read_short_bytearray(); 166 | if(key == null || key.length > 127) 167 | throw new FqCommandProtocolError("bad key"); 168 | break; 169 | case FQ_PROTO_ERROR: 170 | String error = c.cmd_read_short_string(); 171 | if(error != null) throw new FqCommandProtocolError(error); 172 | /* fall through */ 173 | default: 174 | throw new FqCommandProtocolError(cmd); 175 | } 176 | c.getImpl().dispatchAuth(this); 177 | } 178 | } 179 | public static class BindRequest extends FqCommand { 180 | public static final short FQ_BIND_PEER = 0x0001; 181 | public static final short FQ_BIND_PERM = 0x0110; 182 | public static final short FQ_BIND_TRANS = 0x0100; 183 | private Integer binding; 184 | private byte exchange[]; 185 | private byte program[]; 186 | private short flags; 187 | 188 | public BindRequest(byte _exchange[], String _program, short _flags) { 189 | program = _program.getBytes(StandardCharsets.UTF_8); 190 | exchange = _exchange; 191 | flags = _flags; 192 | int extra_space = 193 | 2 + /* flags */ 194 | 2 + exchange.length + /* user */ 195 | 2 + program.length; 196 | alloc(2+extra_space); 197 | } 198 | public BindRequest(byte _exchange[], String _program, boolean _peermode) { 199 | this(_exchange, _program, _peermode ? FQ_BIND_PEER : 0); 200 | } 201 | public BindRequest(String exchange, String p, boolean m) { 202 | this(exchange.getBytes(StandardCharsets.UTF_8), p, m); 203 | } 204 | public short cmd() { return FQ_PROTO_BINDREQ; } 205 | public short response_cmd() { return FQ_PROTO_BIND; } 206 | public boolean hasInBandResponse() { return true; } 207 | public void compose() { 208 | bb.putShort(flags); 209 | bb.putShort((short)exchange.length); 210 | bb.put(exchange); 211 | bb.putShort((short)program.length); 212 | bb.put(program); 213 | } 214 | public void process(FqClient c) throws IOException, FqCommandProtocolError { 215 | super.process(c); 216 | Integer cmd; 217 | bb = c.cmd_read(4); 218 | if(bb == null) return; 219 | bb.flip(); 220 | binding = bb.getInt(); 221 | c.getImpl().dispatchBindRequest(this); 222 | } 223 | public Integer getBinding() { return binding; } 224 | public byte[] getExchange() { return exchange; } 225 | } 226 | public static class UnbindRequest extends FqCommand { 227 | private BindRequest bind; 228 | private Integer success; 229 | 230 | public UnbindRequest(BindRequest b) { 231 | bind = b; 232 | int extra_space = 233 | 2 + /* peermode */ 234 | 4 + /* route_id */ 235 | 2 + bind.getExchange().length; 236 | alloc(2+extra_space); 237 | } 238 | public short cmd() { return FQ_PROTO_UNBINDREQ; } 239 | public short response_cmd() { return FQ_PROTO_UNBIND; } 240 | public boolean hasInBandResponse() { return true; } 241 | public void compose() { 242 | bb.putInt(bind.getBinding()); 243 | bb.putShort((short)bind.getExchange().length); 244 | bb.put(bind.getExchange()); 245 | } 246 | public void process(FqClient c) throws IOException, FqCommandProtocolError { 247 | super.process(c); 248 | Integer cmd; 249 | bb = c.cmd_read(4); 250 | if(bb == null) return; 251 | bb.flip(); 252 | success = bb.getInt(); 253 | c.getImpl().dispatchUnbindRequest(this); 254 | } 255 | public Integer getBinding() { return bind.getBinding(); } 256 | public boolean getSuccess() { 257 | return (success != null && success.equals(bind.getBinding())); 258 | } 259 | } 260 | 261 | public static class StatusRequest extends FqCommand { 262 | protected Date last_update; 263 | protected HashMap status = new HashMap(); 264 | public StatusRequest() { super(0); } 265 | public short cmd() { return FQ_PROTO_STATUSREQ; } 266 | public short response_cmd() { return FQ_PROTO_STATUS; } 267 | public boolean hasInBandResponse() { return true; } 268 | public void process(FqClient c) throws IOException, FqCommandProtocolError { 269 | super.process(c); 270 | last_update = new Date(); 271 | while(true) { 272 | String key = c.cmd_read_short_string(); 273 | if(key == null || key.length() == 0) break; 274 | Integer ivalue; 275 | bb = c.cmd_read(4); 276 | if(bb == null) throw new FqCommandProtocolError("status read failure"); 277 | bb.flip(); 278 | ivalue = bb.getInt(); 279 | Long value = ivalue & (long)0xffffffff; 280 | status.put(key,value); 281 | } 282 | c.getImpl().dispatchStatusRequest(this); 283 | } 284 | public Date getDate() { return last_update; } 285 | public Map getMap() { return status; } 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqCommandProtocolError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | public class FqCommandProtocolError extends Exception { 27 | private short expected; 28 | private short recvd; 29 | private String msg; 30 | public String toString() { 31 | if(msg != null) return msg; 32 | return "Expected " + String.format("0x%04x", expected) + 33 | ", but received " + String.format("0x%04x", recvd); 34 | } 35 | public FqCommandProtocolError(short _expected, short _recvd) { 36 | expected = _expected; 37 | recvd = _recvd; 38 | } 39 | public FqCommandProtocolError(short _expected) { 40 | expected = _expected; 41 | recvd = 0; 42 | } 43 | public FqCommandProtocolError(String _msg) { 44 | msg = _msg; 45 | recvd = 0; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqDataProtocolError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | public class FqDataProtocolError extends Exception { 27 | public FqDataProtocolError(String a) { 28 | super(a); 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqHeartbeatException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | public class FqHeartbeatException extends Exception { 27 | public FqHeartbeatException() { super(); } 28 | } 29 | -------------------------------------------------------------------------------- /java/src/com/omniti/labs/FqMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining a copy 6 | * of this software and associated documentation files (the "Software"), to 7 | * deal in the Software without restriction, including without limitation the 8 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 9 | * sell copies of the Software, and to permit persons to whom the Software is 10 | * furnished to do so, subject to the following conditions: 11 | * 12 | * The above copyright notice and this permission notice shall be included in 13 | * all copies or substantial portions of the Software. 14 | * 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 | * IN THE SOFTWARE. 22 | */ 23 | 24 | package com.omniti.labs; 25 | 26 | import java.io.IOException; 27 | import java.net.InetAddress; 28 | import java.nio.ByteBuffer; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.UUID; 31 | import com.omniti.labs.FqDataProtocolError; 32 | 33 | public class FqMessage { 34 | public static class MsgId { 35 | protected byte d[]; 36 | public MsgId(byte v[]) { 37 | d = new byte[16]; 38 | System.arraycopy(v,0,d,0,16); 39 | } 40 | } 41 | 42 | private boolean _complete = false; 43 | 44 | private int nhops = -1; 45 | private InetAddress hops[]; 46 | private int route_len = -1; 47 | private byte route[]; 48 | private int sender_len = -1; 49 | private byte sender[]; 50 | private int exchange_len = -1; 51 | private byte exchange[]; 52 | private MsgId sender_msgid; 53 | private int payload_len = -1; 54 | private byte payload[]; 55 | 56 | private ByteBuffer[] iovec; 57 | 58 | public void setRoute(byte[] _r) { route = _r; route_len = _r.length; } 59 | public void setSender(byte[] _r) { sender = _r; sender_len = _r.length; } 60 | public void setExchange(byte[] _r) { exchange = _r; exchange_len = _r.length; } 61 | public void setMsgId() { 62 | UUID uuid = UUID.randomUUID(); 63 | ByteBuffer bb = ByteBuffer.wrap(new byte[16]); 64 | bb.putLong(uuid.getMostSignificantBits()); 65 | bb.putLong(uuid.getLeastSignificantBits()); 66 | sender_msgid = new MsgId(bb.array()); 67 | } 68 | public void setPayload(byte[] _r) { payload = _r; payload_len = _r.length; } 69 | 70 | public String getRoute() { return new String(route, StandardCharsets.UTF_8); } 71 | public String getExchange() { return new String(exchange, StandardCharsets.UTF_8); } 72 | public String getSender() { return new String(sender, StandardCharsets.UTF_8); } 73 | public MsgId getMsgId() { return sender_msgid; } 74 | public byte[] getPayload() { return payload; } 75 | public InetAddress[] getPath() { return hops; } 76 | 77 | public boolean isComplete(boolean peermode) { 78 | if(peermode) { 79 | if(nhops < 0 || hops == null || sender_len < 0 || sender == null) 80 | return false; 81 | } 82 | if(route_len <= 0 || route == null || 83 | exchange_len <= 0 || exchange == null || 84 | payload_len < 0 || payload == null || sender_msgid == null) 85 | return false; 86 | return true; 87 | } 88 | public boolean isComplete() { return _complete; } 89 | public boolean read(FqClient c) throws IOException, FqDataProtocolError { 90 | boolean success; 91 | int limit, position; 92 | if(isComplete()) return true; 93 | ByteBuffer bb = c.fill_data_buffer(false); 94 | // Save fill location 95 | position = bb.position(); 96 | limit = bb.limit(); 97 | // Set read location 98 | bb.reset(); 99 | bb.limit(position); 100 | 101 | success = readInternal(c, bb); 102 | 103 | if(!success) { 104 | // compact while reading 105 | bb.compact(); 106 | // after compaction, position is a fill position 107 | position = bb.position(); 108 | // mark at zero (as we've compacted) 109 | bb.position(0); 110 | bb.mark(); 111 | // restore the fill position 112 | bb.position(position); 113 | } else { 114 | // restore fill position 115 | bb.limit(limit); 116 | bb.position(position); 117 | } 118 | return success; 119 | } 120 | private boolean readInternal(FqClient c, ByteBuffer bb) 121 | throws IOException, FqDataProtocolError { 122 | if(isComplete()) return true; 123 | if(exchange_len == -1) { 124 | if(bb.remaining() < 1) return false; 125 | byte len = bb.get(); 126 | exchange_len = len; 127 | bb.mark(); 128 | if(exchange_len <= 0 || exchange_len > 127) 129 | throw new FqDataProtocolError("invalid exchange_len: " + exchange_len); 130 | } 131 | if(exchange == null) { 132 | if(bb.remaining() < exchange_len) return false; 133 | exchange = new byte[exchange_len]; 134 | bb.get(exchange); 135 | bb.mark(); 136 | } 137 | if(route_len == -1) { 138 | if(bb.remaining() < 1) return false; 139 | route_len = (int)bb.get(); 140 | bb.mark(); 141 | if(route_len < 0 || route_len > 127) 142 | throw new FqDataProtocolError("invalid route_len: " + route_len); 143 | } 144 | if(route == null) { 145 | if(bb.remaining() < route_len) return false; 146 | route = new byte[route_len]; 147 | bb.get(route); 148 | bb.mark(); 149 | } 150 | if(sender_msgid == null) { 151 | if(bb.remaining() < 16) return false; 152 | byte[] m = new byte[16]; 153 | bb.get(m); 154 | bb.mark(); 155 | sender_msgid = new MsgId(m); 156 | } 157 | if(sender_len == -1) { 158 | if(bb.remaining() < 1) return false; 159 | sender_len = (int)bb.get(); 160 | bb.mark(); 161 | if(sender_len < 0 || sender_len > 127) 162 | throw new FqDataProtocolError("invalid sender_len: " + sender_len); 163 | } 164 | if(sender == null) { 165 | if(bb.remaining() < sender_len) return false; 166 | sender = new byte[sender_len]; 167 | bb.get(sender); 168 | bb.mark(); 169 | } 170 | if(nhops == -1) { 171 | if(bb.remaining() < 1) return false; 172 | nhops = (int)bb.get(); 173 | if(nhops < 0 || nhops > 32) 174 | throw new FqDataProtocolError("invalid nhops: " + nhops); 175 | bb.mark(); 176 | } 177 | if(hops == null) { 178 | if(bb.remaining() < nhops * 4) return false; 179 | hops = new InetAddress[nhops]; 180 | byte ip[] = new byte[4]; 181 | for(int i=0;i 0) { 193 | payload = new byte[payload_len]; 194 | if(bb.remaining() >= payload_len) { 195 | bb.get(payload); 196 | } else { 197 | int havenow = bb.remaining(); 198 | bb.get(payload, 0, havenow); 199 | int nread = c.blockingRead(payload, havenow, payload_len - havenow); 200 | if((nread+havenow) != payload_len) 201 | throw new FqDataProtocolError("payload read failure: " + nread + "+" + havenow + " != " + payload_len); 202 | } 203 | bb.mark(); 204 | } 205 | _complete = true; 206 | return true; 207 | } 208 | 209 | public boolean send(FqClient c) throws IOException, FqDataProtocolError { 210 | if(!isComplete(c.isPeermode())) throw new FqDataProtocolError("incomplete message"); 211 | if(iovec == null) { 212 | int i = 0; 213 | iovec = new ByteBuffer[c.isPeermode() ? 11 : 7]; 214 | iovec[i ] = ByteBuffer.allocate(1).put((byte)exchange_len); 215 | iovec[i++].flip(); 216 | iovec[i++] = ByteBuffer.wrap(exchange); 217 | iovec[i ] = ByteBuffer.allocate(1).put((byte)route_len); 218 | iovec[i++].flip(); 219 | iovec[i++] = ByteBuffer.wrap(route); 220 | iovec[i++] = ByteBuffer.wrap(sender_msgid.d); 221 | if(c.isPeermode()) { 222 | iovec[i ] = ByteBuffer.allocate(1).put((byte)sender_len); 223 | iovec[i++].flip(); 224 | iovec[i++] = ByteBuffer.wrap(sender); 225 | iovec[i ] = ByteBuffer.allocate(1).put((byte)nhops); 226 | iovec[i++].flip(); 227 | iovec[i ] = ByteBuffer.allocate(nhops * 4); 228 | for(int j=0; j m = cmd.getMap(); 67 | System.err.println("Status: " + d); 68 | for(Map.Entry entry : m.entrySet()) { 69 | System.err.println(" " + entry.getKey() + " : " + entry.getValue()); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqClientImplInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | import com.circonus.FqCommand; 28 | 29 | public interface FqClientImplInterface { 30 | public class InUseException extends Exception { } 31 | public void setClient(FqClient c) throws InUseException; 32 | public void connectError(Throwable e); 33 | public void commandError(Throwable e); 34 | public void dataError(Throwable e); 35 | public void dispatch(FqMessage m); 36 | public void dispatch(FqCommand cmd); 37 | public void dispatchAuth(FqCommand.Auth cmd); 38 | public void dispatchHeartbeat(FqCommand.Heartbeat cmd); 39 | public void dispatchHeartbeatRequest(FqCommand.HeartbeatRequest cmd); 40 | public void dispatchBindRequest(FqCommand.BindRequest cmd); 41 | public void dispatchUnbindRequest(FqCommand.UnbindRequest cmd); 42 | public void dispatchStatusRequest(FqCommand.StatusRequest cmd); 43 | } 44 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqClientImplNoop.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | public class FqClientImplNoop implements FqClientImplInterface { 28 | protected FqClient client; 29 | public void setClient(FqClient c) throws InUseException { 30 | if(client != null) throw new InUseException(); 31 | client = c; 32 | } 33 | protected void genericError(Throwable e) { } 34 | public void connectError(Throwable e) { genericError(e); } 35 | public void commandError(Throwable e) { genericError(e); } 36 | public void dataError(Throwable e) { genericError(e); } 37 | public void dispatch(FqMessage m) { } 38 | public void dispatch(FqCommand cmd) { } 39 | public void dispatchAuth(FqCommand.Auth cmd) { dispatch(cmd); } 40 | public void dispatchHeartbeatRequest(FqCommand.HeartbeatRequest cmd) { 41 | dispatch(cmd); 42 | } 43 | public void dispatchHeartbeat(FqCommand.Heartbeat cmd) { dispatch(cmd); } 44 | public void dispatchBindRequest(FqCommand.BindRequest cmd) { dispatch(cmd); }; 45 | public void dispatchUnbindRequest(FqCommand.UnbindRequest cmd) { dispatch(cmd); }; 46 | public void dispatchStatusRequest(FqCommand.StatusRequest cmd) { dispatch(cmd); }; 47 | } 48 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqCommandProtocolError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | public class FqCommandProtocolError extends Exception { 28 | private short expected; 29 | private short recvd; 30 | private String msg; 31 | public String toString() { 32 | if(msg != null) return msg; 33 | return "Expected " + String.format("0x%04x", expected) + 34 | ", but received " + String.format("0x%04x", recvd); 35 | } 36 | public FqCommandProtocolError(short _expected, short _recvd) { 37 | expected = _expected; 38 | recvd = _recvd; 39 | } 40 | public FqCommandProtocolError(short _expected) { 41 | expected = _expected; 42 | recvd = 0; 43 | } 44 | public FqCommandProtocolError(String _msg) { 45 | msg = _msg; 46 | recvd = 0; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqDataProtocolError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | public class FqDataProtocolError extends Exception { 28 | public FqDataProtocolError(String a) { 29 | super(a); 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqHeartbeatException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | public class FqHeartbeatException extends Exception { 28 | public FqHeartbeatException() { super(); } 29 | } 30 | -------------------------------------------------------------------------------- /java/src/main/java/com/circonus/FqMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 Circonus, Inc. 3 | * Copyright (c) 2013 OmniTI Computer Consulting, Inc. 4 | * All rights reserved. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to 8 | * deal in the Software without restriction, including without limitation the 9 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 | * IN THE SOFTWARE. 23 | */ 24 | 25 | package com.circonus; 26 | 27 | import java.io.IOException; 28 | import java.net.InetAddress; 29 | import java.nio.ByteBuffer; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.UUID; 32 | import com.circonus.FqDataProtocolError; 33 | 34 | public class FqMessage { 35 | public static class MsgId { 36 | protected byte d[]; 37 | public MsgId(byte v[]) { 38 | d = new byte[16]; 39 | System.arraycopy(v,0,d,0,16); 40 | } 41 | } 42 | 43 | private boolean _complete = false; 44 | 45 | private int nhops = -1; 46 | private InetAddress hops[]; 47 | private int route_len = -1; 48 | private byte route[]; 49 | private int sender_len = -1; 50 | private byte sender[]; 51 | private int exchange_len = -1; 52 | private byte exchange[]; 53 | private MsgId sender_msgid; 54 | private int payload_len = -1; 55 | private byte payload[]; 56 | 57 | private ByteBuffer[] iovec; 58 | 59 | public void setRoute(byte[] _r) { route = _r; route_len = _r.length; } 60 | public void setSender(byte[] _r) { sender = _r; sender_len = _r.length; } 61 | public void setExchange(byte[] _r) { exchange = _r; exchange_len = _r.length; } 62 | public void setMsgId() { 63 | UUID uuid = UUID.randomUUID(); 64 | ByteBuffer bb = ByteBuffer.wrap(new byte[16]); 65 | bb.putLong(uuid.getMostSignificantBits()); 66 | bb.putLong(uuid.getLeastSignificantBits()); 67 | sender_msgid = new MsgId(bb.array()); 68 | } 69 | public void setPayload(byte[] _r) { payload = _r; payload_len = _r.length; } 70 | 71 | public String getRoute() { return new String(route, StandardCharsets.UTF_8); } 72 | public String getExchange() { return new String(exchange, StandardCharsets.UTF_8); } 73 | public String getSender() { return new String(sender, StandardCharsets.UTF_8); } 74 | public MsgId getMsgId() { return sender_msgid; } 75 | public byte[] getPayload() { return payload; } 76 | public InetAddress[] getPath() { return hops; } 77 | 78 | public boolean isComplete(boolean peermode) { 79 | if(peermode) { 80 | if(nhops < 0 || hops == null || sender_len < 0 || sender == null) 81 | return false; 82 | } 83 | if(route_len <= 0 || route == null || 84 | exchange_len <= 0 || exchange == null || 85 | payload_len < 0 || payload == null || sender_msgid == null) 86 | return false; 87 | return true; 88 | } 89 | public boolean isComplete() { return _complete; } 90 | public boolean read(FqClient c) throws IOException, FqDataProtocolError { 91 | boolean success; 92 | int limit, position; 93 | if(isComplete()) return true; 94 | ByteBuffer bb = c.fill_data_buffer(false); 95 | // Save fill location 96 | position = bb.position(); 97 | limit = bb.limit(); 98 | // Set read location 99 | bb.reset(); 100 | bb.limit(position); 101 | 102 | success = readInternal(c, bb); 103 | 104 | if(!success) { 105 | // compact while reading 106 | bb.compact(); 107 | // after compaction, position is a fill position 108 | position = bb.position(); 109 | // mark at zero (as we've compacted) 110 | bb.position(0); 111 | bb.mark(); 112 | // restore the fill position 113 | bb.position(position); 114 | } else { 115 | // restore fill position 116 | bb.limit(limit); 117 | bb.position(position); 118 | } 119 | return success; 120 | } 121 | private boolean readInternal(FqClient c, ByteBuffer bb) 122 | throws IOException, FqDataProtocolError { 123 | if(isComplete()) return true; 124 | if(exchange_len == -1) { 125 | if(bb.remaining() < 1) return false; 126 | byte len = bb.get(); 127 | exchange_len = len; 128 | bb.mark(); 129 | if(exchange_len <= 0 || exchange_len > 127) 130 | throw new FqDataProtocolError("invalid exchange_len: " + exchange_len); 131 | } 132 | if(exchange == null) { 133 | if(bb.remaining() < exchange_len) return false; 134 | exchange = new byte[exchange_len]; 135 | bb.get(exchange); 136 | bb.mark(); 137 | } 138 | if(route_len == -1) { 139 | if(bb.remaining() < 1) return false; 140 | route_len = (int)bb.get(); 141 | bb.mark(); 142 | if(route_len < 0 || route_len > 127) 143 | throw new FqDataProtocolError("invalid route_len: " + route_len); 144 | } 145 | if(route == null) { 146 | if(bb.remaining() < route_len) return false; 147 | route = new byte[route_len]; 148 | bb.get(route); 149 | bb.mark(); 150 | } 151 | if(sender_msgid == null) { 152 | if(bb.remaining() < 16) return false; 153 | byte[] m = new byte[16]; 154 | bb.get(m); 155 | bb.mark(); 156 | sender_msgid = new MsgId(m); 157 | } 158 | if(sender_len == -1) { 159 | if(bb.remaining() < 1) return false; 160 | sender_len = (int)bb.get(); 161 | bb.mark(); 162 | if(sender_len < 0 || sender_len > 127) 163 | throw new FqDataProtocolError("invalid sender_len: " + sender_len); 164 | } 165 | if(sender == null) { 166 | if(bb.remaining() < sender_len) return false; 167 | sender = new byte[sender_len]; 168 | bb.get(sender); 169 | bb.mark(); 170 | } 171 | if(nhops == -1) { 172 | if(bb.remaining() < 1) return false; 173 | nhops = (int)bb.get(); 174 | if(nhops < 0 || nhops > 32) 175 | throw new FqDataProtocolError("invalid nhops: " + nhops); 176 | bb.mark(); 177 | } 178 | if(hops == null) { 179 | if(bb.remaining() < nhops * 4) return false; 180 | hops = new InetAddress[nhops]; 181 | byte ip[] = new byte[4]; 182 | for(int i=0;i 0) { 194 | payload = new byte[payload_len]; 195 | if(bb.remaining() >= payload_len) { 196 | bb.get(payload); 197 | } else { 198 | int havenow = bb.remaining(); 199 | bb.get(payload, 0, havenow); 200 | int nread = c.blockingRead(payload, havenow, payload_len - havenow); 201 | if((nread+havenow) != payload_len) 202 | throw new FqDataProtocolError("payload read failure: " + nread + "+" + havenow + " != " + payload_len); 203 | } 204 | bb.mark(); 205 | } 206 | _complete = true; 207 | return true; 208 | } 209 | 210 | public boolean send(FqClient c) throws IOException, FqDataProtocolError { 211 | if(!isComplete(c.isPeermode())) throw new FqDataProtocolError("incomplete message"); 212 | if(iovec == null) { 213 | int i = 0; 214 | iovec = new ByteBuffer[c.isPeermode() ? 11 : 7]; 215 | iovec[i ] = ByteBuffer.allocate(1).put((byte)exchange_len); 216 | iovec[i++].flip(); 217 | iovec[i++] = ByteBuffer.wrap(exchange); 218 | iovec[i ] = ByteBuffer.allocate(1).put((byte)route_len); 219 | iovec[i++].flip(); 220 | iovec[i++] = ByteBuffer.wrap(route); 221 | iovec[i++] = ByteBuffer.wrap(sender_msgid.d); 222 | if(c.isPeermode()) { 223 | iovec[i ] = ByteBuffer.allocate(1).put((byte)sender_len); 224 | iovec[i++].flip(); 225 | iovec[i++] = ByteBuffer.wrap(sender); 226 | iovec[i ] = ByteBuffer.allocate(1).put((byte)nhops); 227 | iovec[i++].flip(); 228 | iovec[i ] = ByteBuffer.allocate(nhops * 4); 229 | for(int j=0; j 0 do 45 | local key = shift() 46 | if key == "--src_host" then src_host = shift() 47 | elseif key == "--src_port" then src_port = shift() 48 | elseif key == "--src_user" then src_user = shift() 49 | elseif key == "--src_pass" then src_pass = shift() 50 | elseif key == "--src_exchange" then src_exchange = shift() 51 | elseif key == "--src_program" then src_program = shift() 52 | elseif key == "--dst_host" then dst_host = shift() 53 | elseif key == "--dst_port" then dst_port = shift() 54 | elseif key == "--dst_user" then dst_user = shift() 55 | elseif key == "--dst_pass" then dst_pass = shift() 56 | elseif key == "--dst_exchange" then dst_exchange = shift() 57 | elseif key == "--dst_route" then dst_route = shift() 58 | elseif key == "--filter" then filter = shift() 59 | elseif key == "--print" then do_print = true 60 | else 61 | die("Unknown argument " .. key) 62 | end 63 | end 64 | 65 | if not src_exchange then die("src_exchange not set") end 66 | if not dst_exchange then die("dst_exchange not set") end 67 | 68 | function main() 69 | local src_fq_client = fqclient.new(src_host, src_port, src_user, src_pass) 70 | src_fq_client:bind(src_exchange, src_program) 71 | src_fq_client:connect() 72 | local dst_fq_client = fqclient.new(dst_host, dst_port, dst_user, dst_pass) 73 | dst_fq_client:connect() 74 | if filter then print(string.format("Filtering messages by '%s'", filter)) end 75 | src_fq_client:listen(function(msg) 76 | if (not filter) or string.match(msg, filter) then 77 | if do_print then print(msg) end 78 | dst_fq_client:send(msg, dst_exchange, dst_route) 79 | end 80 | end) 81 | end 82 | main() 83 | -------------------------------------------------------------------------------- /lua/fq-receiver: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | local fqclient = require 'fqclient' 3 | 4 | local usage = [[ 5 | Arguments: 6 | --host (default localhost) 7 | --port (default 8765) 8 | --user (default user) 9 | --pass (default pass) 10 | --exchange (string) 11 | --program (default 'prefix:') 12 | --time add timestamp prefix 13 | --timef (default '%.6f') add fromatted timestamp 14 | -v,--verbose print metadata 15 | -h,--help print this message 16 | ]] 17 | 18 | local function die(msg) 19 | print(msg) 20 | print(usage) 21 | os.exit(1) 22 | end 23 | 24 | local host = "localhost" 25 | local port = 8765 26 | local program = 'prefix:' 27 | local exchange 28 | local user = "user" 29 | local pass = "pass" 30 | local verbose = false 31 | local separator = "\n" 32 | local time = false 33 | local time_format = "[%.3f]\t" 34 | 35 | local function shift() return table.remove(arg, 1) end 36 | while #arg > 0 do 37 | local key = shift() 38 | if key == "--host" then 39 | host = shift() 40 | elseif key == "--port" then 41 | port = shift() 42 | elseif key == "--user" then 43 | user = shift() 44 | elseif key == "--pass" then 45 | pass = shift() 46 | elseif key == "--program" then 47 | program = shift() 48 | elseif key == "--exchange" then 49 | exchange = shift() 50 | elseif key == "--separator" then 51 | separator = shift() 52 | elseif key == "--time" then 53 | time = true 54 | elseif key == "--timef" then 55 | time = true 56 | time_format = shift() 57 | elseif key == "-v" or key == "--verbose" then 58 | verbose = true 59 | elseif key == "-h" or key == "--help" then 60 | print(usage) 61 | os.exit(0) 62 | else 63 | die("Unknown argument " .. key) 64 | end 65 | end 66 | 67 | if not exchange then die("No exchange provided") end 68 | 69 | local fq_client = fqclient.new(host, port, user, pass) 70 | fq_client:bind(exchange, program) 71 | fq_client:connect() 72 | fq_client:listen_table(function(mtab) 73 | if verbose then 74 | io.write(string.format("--%s\n", time and string.format(time_format, fqclient.time()) or "")) 75 | io.write(string.format("arrival_time: %d\n", mtab.arrival_time)) 76 | io.write(string.format("sender : %s\n", mtab.sender)) 77 | io.write(string.format("route : %s\n", mtab.route)) 78 | io.write(string.format("payload : %s\n", mtab.payload)) 79 | else 80 | if time then io.write(string.format(time_format, fqclient.time())) end 81 | io.write(mtab.payload) 82 | io.write(separator) 83 | end 84 | end) 85 | -------------------------------------------------------------------------------- /lua/fq-sender: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | local fqclient = require 'fqclient' 3 | local ffi = require 'ffi'; 4 | 5 | local USAGE = [[ 6 | Arguments: 7 | --host (default localhost) fq host to connect 8 | --port (default 8765) port to connect 9 | --user (default user) 10 | --pass (default pass) 11 | --exchange (string) 12 | --route (default "") 13 | -v verbosity 14 | ]] 15 | 16 | local function die(msg) 17 | print(msg) 18 | print(USAGE) 19 | os.exit(1) 20 | end 21 | 22 | local host = "localhost" 23 | local port = 8765 24 | local user = "user" 25 | local pass = "pass" 26 | local exchange 27 | local route = "" 28 | local verbose = false 29 | 30 | local function shift() return table.remove(arg, 1) end 31 | while #arg > 0 do 32 | local key = shift() 33 | if key == "--host" then 34 | host = shift() 35 | elseif key == "--port" then 36 | port = shift() 37 | elseif key == "--user" then 38 | user = shift() 39 | elseif key == "--pass" then 40 | pass = shift() 41 | elseif key == "--exchange" then 42 | exchange = shift() 43 | elseif key == "--route" then 44 | route = shift() 45 | elseif key == "-v" or key == "--verbose" then 46 | verbose = true 47 | elseif key == "-h" or key == "--help" then 48 | print(USAGE) 49 | os.exit(0) 50 | else 51 | die("Unknown argument " .. key) 52 | end 53 | end 54 | 55 | if not exchange then 56 | die("Exchange not provided") 57 | end 58 | if not route then 59 | die("Route not provided") 60 | end 61 | 62 | local log 63 | if verbose then 64 | log = function(...) return io.stderr:write(string.format(...) .. "\n") end 65 | else 66 | log = function() end 67 | end 68 | 69 | local function exit() 70 | end 71 | 72 | log("Connecting to %s %d %s %s\n", host, port, user, pass) 73 | local fq_client = fqclient.new(host, port, user, pass) 74 | fq_client:connect() 75 | while true do 76 | local msg = io.read() 77 | if not msg then 78 | fqclient:close() 79 | os.exit(0) 80 | end 81 | fq_client:send(msg, exchange, route) 82 | log('sent\t{ "exchange":"%s", route:"%s", "message":"%s" }', exchange, route, msg) 83 | end 84 | -------------------------------------------------------------------------------- /lua/fqclient.lua.tail: -------------------------------------------------------------------------------- 1 | 2 | local gettimeofday_struct = ffi.new("timeval") 3 | local function gettimeofday() 4 | ffi.C.gettimeofday(gettimeofday_struct, nil) 5 | return tonumber(gettimeofday_struct.tv_sec) + tonumber(gettimeofday_struct.tv_usec) / 1000000 6 | end 7 | 8 | local function charstar(str) 9 | local len = string.len(str) 10 | local buf = ffi.new("char[?]", len+1, 0) 11 | ffi.copy(buf, str, len) 12 | return buf 13 | end 14 | 15 | local function m2tab(m) 16 | return { 17 | route = ffi.string(m.route.name, m.route.len), 18 | sender = ffi.string(m.sender.name, m.sender.len), 19 | exchange = ffi.string(m.exchange.name, m.exchange.len), 20 | arrival_time = tonumber(m.arrival_time), 21 | payload = ffi.string(m.payload, m.payload_len), 22 | } 23 | end 24 | 25 | local function new(host, port, user, pass) 26 | local binds = {} 27 | local bind_programs = {} -- refs the ffi C strings inside binds 28 | local conn = ffi.new("fq_client[?]", 1); 29 | local object = {} 30 | local hooks = ffi.new("fq_hooks[?]", 1); 31 | hooks[0].version = ffi.C.FQ_HOOKS_V3; 32 | hooks[0].sync = 1 33 | hooks[0].unbind = nil 34 | hooks[0].auth = function (c, err) 35 | if object.auth_cb ~= nil then 36 | object:auth_cb(err) 37 | end 38 | -- perform binds after auth has completed 39 | for i,v in ipairs(binds) do 40 | fq.fq_client_bind(c, v[0]) 41 | end 42 | end 43 | hooks[0].bind = function (c, breq) 44 | if object.bind_cb ~= nil then 45 | object:bind_cb(breq) 46 | end 47 | end 48 | rv = fq.fq_client_init(conn, 0, nil) 49 | fq.fq_client_hooks(conn[0], hooks) 50 | fq.fq_client_creds(conn[0], host, port, user, pass) 51 | fq.fq_client_heartbeat(conn[0], 1000); 52 | fq.fq_client_set_backlog(conn[0], 10000, 100); 53 | fq.fq_client_set_nonblock(conn[0], false); 54 | 55 | object.conn = conn[0] 56 | 57 | object.bind = function(object, exchange, program, flags) 58 | local breq = ffi.new("fq_bind_req[?]", 1) 59 | ffi.copy(breq[0].exchange.name, exchange) 60 | breq[0].exchange.len = exchange:len() 61 | breq[0].flags = flags or fq.FQ_BIND_TRANS 62 | local bind_program = charstar(program) 63 | table.insert(bind_programs, bind_program) 64 | breq[0].program = bind_program 65 | table.insert(binds, breq) 66 | return object 67 | end 68 | 69 | object.connect = function(object) 70 | local rc = fq.fq_client_connect(object.conn); 71 | if (rc == -1) then error("Connection failed") end 72 | end 73 | 74 | object.close = function(object) 75 | while fq.fq_client_data_backlog(object.conn) > 0 do 76 | ffi.C.usleep(100) 77 | end 78 | fq.fq_client_destroy(object.conn) 79 | end 80 | 81 | --- Recevie message if one is available, otherwise nil 82 | object.recv = function(object) 83 | jit.off() 84 | local m = fq.fq_client_receive(object.conn) 85 | if m ~= nil then -- need ~= nil check here. 86 | local arrival_time = tonumber(m.arrival_time) 87 | local payload = ffi.string(m.payload, m.payload_len) 88 | local route = ffi.string(m.route.name, m.route.len) 89 | local sender = ffi.string(m.sender.name, m.sender.len) 90 | local exchange = ffi.string(m.exchange.name, m.exchange.len) 91 | fq.fq_msg_deref(m) 92 | return arrival_time, payload, route, sender, exchange 93 | end 94 | jit.on() 95 | end 96 | 97 | object.listen_raw = function(object, callback) 98 | -- poll on socket and execute callback when message is found 99 | local sleep_micros = 1 100 | local sleep_micros_min = 2 101 | local sleep_micros_max = 10E3 102 | while true do 103 | jit.off() 104 | local m = fq.fq_client_receive(object.conn) 105 | if m ~= nil then 106 | callback(m) 107 | fq.fq_msg_deref(m) 108 | sleep_micros = 1 109 | elseif sleep_micros < sleep_micros_max then 110 | sleep_micros = sleep_micros * 2 111 | end 112 | if sleep_micros > sleep_micros_min then 113 | ffi.C.usleep(sleep_micros) 114 | end 115 | jit.on() 116 | end 117 | end 118 | 119 | object.listen = function(object, callback) 120 | object:listen_raw(function(m) callback(ffi.string(m.payload)) end) 121 | end 122 | 123 | object.listen_table = function(object, callback) 124 | object:listen_raw(function(m) callback(m2tab(m)) end) 125 | end 126 | 127 | object.send = function(object, message, exchange, route) 128 | local cmsg = charstar(message) 129 | local cexchange = charstar(exchange) 130 | local croute = charstar(route) 131 | local msg = fq.fq_msg_alloc(cmsg, string.len(message)) 132 | fq.fq_msg_exchange(msg, cexchange, string.len(exchange)) 133 | fq.fq_msg_route(msg, croute, string.len(route)) 134 | 135 | -- fq is set to be blocking so fq_client_publish will block 136 | fq.fq_client_publish(object.conn, msg) 137 | fq.fq_msg_deref(msg) 138 | end 139 | 140 | return object 141 | end 142 | 143 | return { 144 | new = new, 145 | usleep = ffi.C.usleep, 146 | time = gettimeofday, 147 | } 148 | 149 | -------------------------------------------------------------------------------- /lua/generatelua.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit # Exit script on first error. 4 | set -o nounset # Treat references to unset variables as errors. 5 | set -o pipefail 6 | 7 | AWK=' 8 | BEGIN { out=0 } 9 | /!lua start/ { out=1; next } 10 | /!lua stop/ { out=0; next } 11 | /^$/ {next} # skip whitespace 12 | /^#/ { while (/\\$/) { getline } next } 13 | (out == 1) { print } 14 | ' 15 | 16 | FFI_HEAD=' 17 | local ffi = require("ffi") 18 | local fq = ffi.load("fq") 19 | ffi.cdef [[ 20 | extern void usleep(int); 21 | 22 | typedef long time_t; 23 | 24 | typedef struct timeval { 25 | time_t tv_sec; 26 | time_t tv_usec; 27 | } timeval; 28 | 29 | int gettimeofday(struct timeval* t, void* tzp); 30 | 31 | struct ck_stack_entry { 32 | struct ck_stack_entry *next; 33 | }; 34 | typedef struct ck_stack_entry ck_stack_entry_t; 35 | 36 | // Those are defined in fqh as macros 37 | static const uint32_t FQ_PROTO_CMD_MODE = 0xcc50cafe; 38 | static const uint32_t FQ_PROTO_DATA_MODE = 0xcc50face; 39 | static const uint32_t FQ_PROTO_PEER_MODE = 0xcc50fade; 40 | static const uint32_t FQ_PROTO_READ_STAT = 0x47455420; 41 | static const uint32_t FQ_PROTO_HTTP_GET = 0x47455420; 42 | static const uint32_t FQ_PROTO_HTTP_PUT = 0x50555420; 43 | static const uint32_t FQ_PROTO_HTTP_POST = 0x504f5354; 44 | static const uint32_t FQ_PROTO_HTTP_HEAD = 0x48454144; 45 | 46 | static const uint32_t FQ_BIND_PEER = 0x00000001; 47 | static const uint32_t FQ_BIND_PERM = 0x00000110; 48 | static const uint32_t FQ_BIND_TRANS = 0x00000100; 49 | 50 | static const uint32_t FQ_PROTO_ERROR = 0xeeee; 51 | static const uint32_t FQ_PROTO_AUTH_CMD = 0xaaaa; 52 | static const uint32_t FQ_PROTO_AUTH_PLAIN = 0; 53 | static const uint32_t FQ_PROTO_AUTH_RESP = 0xaa00; 54 | static const uint32_t FQ_PROTO_HBREQ = 0x4848; 55 | static const uint32_t FQ_PROTO_HB = 0xbea7; 56 | static const uint32_t FQ_PROTO_BINDREQ = 0xb170; 57 | static const uint32_t FQ_PROTO_BIND = 0xb171; 58 | static const uint32_t FQ_PROTO_UNBINDREQ = 0x071b; 59 | static const uint32_t FQ_PROTO_UNBIND = 0x171b; 60 | static const uint32_t FQ_PROTO_STATUS = 0x57a7; 61 | static const uint32_t FQ_PROTO_STATUSREQ = 0xc7a7; 62 | static const uint32_t FQ_BIND_ILLEGAL = 0xffffffff; 63 | 64 | static const int MAX_RK_LEN = 127; 65 | static const int MAX_HOPS = 32; 66 | static const int FQ_HOOKS_V1 = 1; 67 | static const int FQ_HOOKS_V2 = 2; 68 | static const int FQ_HOOKS_V3 = 3; 69 | ' 70 | 71 | FFI_TAIL=']]' 72 | 73 | printf '%s\n' "$FFI_HEAD" > fqclient.lua 74 | cat ../fq.h | awk "$AWK" | sed 's/MAX_RK_LEN/127/' >> fqclient.lua 75 | printf '%s\n' "$FFI_TAIL" >> fqclient.lua 76 | cat fqclient.lua.tail >> fqclient.lua 77 | -------------------------------------------------------------------------------- /service-configs/50-circonus-fq.preset: -------------------------------------------------------------------------------- 1 | # Ship disabled by default 2 | disable circonus-fq.service 3 | -------------------------------------------------------------------------------- /service-configs/circonus-fq.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=FQ 3 | After=network.target 4 | 5 | [Service] 6 | User=fq 7 | WorkingDirectory=/opt/circonus/var/lib/fq 8 | EnvironmentFile=-/opt/circonus/var/lib/fq/daemon_options 9 | ExecStart=/opt/circonus/sbin/fqd -D ${DAEMON_OPTS} 10 | 11 | # Note that leaving the service as forking breaks restarts 12 | Restart=always 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /service-configs/daemon_options: -------------------------------------------------------------------------------- 1 | # Additional fqd commandline arguments 2 | DAEMON_OPTS="" 3 | -------------------------------------------------------------------------------- /test/lua-support/init.lua: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /test/run-tests.sh: -------------------------------------------------------------------------------- 1 | cd "$(dirname $0)" 2 | 3 | LD_LIBRARY_PATH="../" 4 | export LD_LIBRARY_PATH 5 | 6 | /opt/circonus/bin/mtevbusted $@ 7 | -------------------------------------------------------------------------------- /test/test_spec.lua: -------------------------------------------------------------------------------- 1 | local fqclient = require("../lua/fqclient.lua") 2 | 3 | local function mkreader(exchange, program) 4 | local key_auth = mtev.uuid() 5 | local key_bind = mtev.uuid() 6 | local key_read = mtev.uuid() 7 | local fqc_read = fqclient.new("127.0.0.1", 18765, "busted-user-1", "busted-pw") 8 | fqc_read.auth_cb = function() 9 | mtev.notify(key_auth, true) 10 | end 11 | fqc_read.bind_cb = function() 12 | mtev.notify(key_bind, true) 13 | end 14 | fqc_read:bind(exchange, program) -- need to bind before connect 15 | fqc_read:connect() 16 | -- We have to call fq_read:recv() in a loop in order for call backs to execute. 17 | mtev.coroutine_spawn(function() 18 | while true do 19 | local m = { fqc_read:recv() } 20 | if #m > 0 then 21 | mtev.log("debug", "RECV: %s\n", mtev.tojson(m):tostring()) 22 | mtev.notify(key_read, m[2]) -- just forward payload 23 | else 24 | mtev.sleep(.005) 25 | end 26 | end 27 | end) 28 | assert.truthy(mtev.waitfor(key_auth, 5)) 29 | assert.truthy(mtev.waitfor(key_bind, 5)) 30 | local reader = function(timeout) 31 | local _, m = mtev.waitfor(key_read, timeout or 5) 32 | return m 33 | end 34 | return reader 35 | end 36 | 37 | describe("fq", function() 38 | 39 | local fq, api 40 | 41 | setup(function() 42 | -- Setup fq process wrapper 43 | fq = mtev.Proc:new { 44 | path = "../fqd", 45 | argv = { 46 | "fqd", "-D", 47 | '-n', '10.254.254.1', 48 | '-c', './fqd.sqlite', 49 | '-p', '18765', 50 | '-v', 'conn,route,msg,io', 51 | }, 52 | boot_match = "Listening on port", 53 | } 54 | -- write stderr output to out.log 55 | fq:logwrite("out.log") 56 | -- Optional: Forward fqd output to error log 57 | -- fq:loglog("error") 58 | api = mtev.Api:http("127.0.0.1", '18765') 59 | end) 60 | 61 | teardown(function() 62 | fq:kill() 63 | end) 64 | 65 | it("should start", function() 66 | fq:start() 67 | assert.truthy(fq:ready()) 68 | end) 69 | 70 | it("should allow HTTP requests", function() 71 | assert.truthy(api:get("/stats.json"):check()) 72 | end) 73 | 74 | local fqc_send 75 | local exchange = "test-exchange" 76 | local program = "prefix:" 77 | local route = "test-route" 78 | local reader 79 | it("should accept connections", function() 80 | fqc_send = fqclient.new("127.0.0.1", 18765, "busted-user-2", "busted-pw") 81 | fqc_send:connect() 82 | reader = mkreader(exchange, "prefix:") 83 | end) 84 | 85 | it("should send/recv hello messages", function() 86 | local msg = "Hello!" 87 | local N = 10 88 | for i=1,N do 89 | fqc_send:send(msg, exchange, route) 90 | end 91 | for i=1,N do 92 | assert.equal(msg, reader()) 93 | end 94 | end) 95 | 96 | it("should send messages via HTTP", function() 97 | -- Submit message via HTTP 98 | -- $ curl -X POST -H "X-Fq-User: web" -H 'X-Fq-Route: user-route' \ 99 | -- -H 'X-Fq-Exchange: busted-exchange' 192.168.33.10:8765/submit \ 100 | -- --data 'Hello world!' 101 | -- {"routed":1,"dropped":0,"no_route":0,"no_exchange":0} 102 | local payload = "Some HTTP payload" 103 | r = api:post("/submit", payload, { 104 | ["X-Fq-User"] = "web", 105 | ["X-Fq-Route"] = "web-route", 106 | ["X-Fq-Exchange"] = exchange, 107 | }):check() 108 | assert.equals(r:json().routed, 1) 109 | assert.equals(payload, reader()) 110 | end) 111 | 112 | it("should send messages via fqs", function() 113 | -- quick and dirty way to spin up fqs 114 | mtev.sh(string.format( 115 | [[printf 'hello fqs' | LD_LIBRARY_PATH=../:/opt/circonus/lib ../fqs -a 127.0.0.1:18765 -x "%s" -r "%s"]], 116 | exchange, route)) 117 | assert.equals('hello fqs', reader()) 118 | end) 119 | 120 | it("should allow multiple readers", function() 121 | local reader2 = mkreader(exchange, "prefix:") 122 | local msg = "hello reader 2!" 123 | fqc_send:send(msg, exchange, route) 124 | assert.equals(msg, reader()) 125 | assert.equals(msg, reader2()) 126 | end) 127 | 128 | it("should filter prefixes", function() 129 | local reader_x = mkreader(exchange, "prefix:x") 130 | fqc_send:send("abc", exchange, "abc") 131 | fqc_send:send("xxx", exchange, "xxx") 132 | assert.equals("abc", reader()) 133 | assert.equals("xxx", reader()) 134 | assert.equals("xxx", reader_x()) 135 | end) 136 | 137 | end) 138 | -------------------------------------------------------------------------------- /web/css/theme.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 75px; 3 | padding-bottom: 30px; 4 | } 5 | 6 | .container > div.navbar-header { 7 | background: transparent url(../i/product-logo-rev2.png) no-repeat scroll left top; 8 | background-size: 50px 50px; 9 | background-position: 0 2px; 10 | padding-left: 55px; 11 | } 12 | 13 | .theme-dropdown .dropdown-menu { 14 | display: block; 15 | position: static; 16 | margin-bottom: 20px; 17 | } 18 | 19 | .theme-showcase > p > .btn { 20 | margin: 5px 0; 21 | } 22 | .label-none { 23 | background-color: #fff; 24 | } 25 | 26 | span.exchange-rate { 27 | text-align:right; 28 | display: inline-block; 29 | width: 8em; 30 | } 31 | span.exchange-value { 32 | text-align:right; 33 | float: right; 34 | display: inline-block; 35 | width: auto; 36 | } 37 | span.exchange-value:before, span.client-value:before, span.queue-value:before { 38 | content:"("; 39 | } 40 | span.exchange-value:after, span.client-value:after, span.queue-value:after { 41 | content:")"; 42 | } 43 | span.exchange-rate:after, span.client-rate:after, span.queue-rate:after { 44 | content:"/s"; 45 | } 46 | 47 | .aggregate-only { 48 | display:none; 49 | } 50 | 51 | div#exchange-5f616767726567617465 .aggregate-only { 52 | display:block; 53 | } 54 | 55 | div#queues div.panel-heading { 56 | padding-bottom: 0; 57 | } 58 | div.queue-name-title a { 59 | margin-left: 1em; 60 | } 61 | div.backlog-container { 62 | width:80px; 63 | } 64 | div.queue-name-title a:hover { 65 | text-decoration:none; 66 | } 67 | table.clients td { 68 | font-size: 12px; 69 | } 70 | table.clients th, 71 | table.clients td { 72 | width: 8%; 73 | text-align:right; 74 | } 75 | table.clients th.gap { 76 | width:1em; 77 | } 78 | table.clients th.top-span { 79 | width:auto; 80 | border-bottom:1; 81 | border-color:#999; 82 | } 83 | table.clients th.client-user, 84 | table.clients td.client-user { 85 | text-align:left; 86 | } 87 | table.clients th.client-address, 88 | table.clients td.client-address { 89 | width: 18%; 90 | text-align:center; 91 | } 92 | 93 | @media (min-width: 768px) { 94 | dl.dl-horizontal { 95 | margin-bottom:0; 96 | } 97 | .dl-horizontal dt { 98 | width:100px; 99 | } 100 | .dl-horizontal dd { 101 | margin-left: 6em; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /web/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/circonus-labs/fq/aa2bb0149df754edd6fa82e8f30ee161eaaa2954/web/favicon.png -------------------------------------------------------------------------------- /web/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/circonus-labs/fq/aa2bb0149df754edd6fa82e8f30ee161eaaa2954/web/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /web/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/circonus-labs/fq/aa2bb0149df754edd6fa82e8f30ee161eaaa2954/web/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /web/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/circonus-labs/fq/aa2bb0149df754edd6fa82e8f30ee161eaaa2954/web/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /web/i/product-logo-rev2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/circonus-labs/fq/aa2bb0149df754edd6fa82e8f30ee161eaaa2954/web/i/product-logo-rev2.png -------------------------------------------------------------------------------- /web/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Circonus Fq Operational Dashboard 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 26 | 27 | 28 | 29 | 30 | 31 |
32 |
33 |
34 | Circonus Fq 35 |
36 |
37 |
38 | 39 |
40 | 41 | 46 |
47 |
48 |
49 |
50 |

Exchanges

51 |
52 |
53 |
54 |
55 |
56 |
57 |

_aggregate

58 |
59 |
60 |
61 |
Messages In
62 |
Octets In
63 |
Routed
64 |
Dropped
65 |
Size dropped
66 |
No Route
67 |
No Exchange
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |

Queues

80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |

Routes

89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 | 97 | 98 |
99 |
100 |
101 |
102 | 119 |
120 |
121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 |
UserAddressTx (to Client)Rx (from Client)
MsgsOctetsMsgsOctetsRoutedDroppedSize DroppedNo RouteNo Exchange
141 |
142 |
143 |
144 |
145 |
146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 |









163 | 164 |
165 |

166 |
167 |
168 |
Prefix
169 |
Program
170 |
Invocations
171 |
Approximate Latency
172 |
173 |
174 |
175 |
176 | 177 |
178 |
179 | 180 |
181 |
182 |
183 |
184 |
185 | 186 |
187 |
188 | 189 |
190 |
191 |
192 | 193 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /web/js/fq.js: -------------------------------------------------------------------------------- 1 | Array.prototype.equals = function (array) { 2 | if (!array) return false; 3 | if (this.length != array.length) return false; 4 | 5 | for (var i = 0, l=this.length; i < l; i++) { 6 | if (this[i] instanceof Array && array[i] instanceof Array) { 7 | if (!this[i].equals(array[i])) return false; 8 | } 9 | else if (this[i] != array[i]) { 10 | return false; 11 | } 12 | } 13 | return true; 14 | } 15 | 16 | function pretty_number(a) { 17 | var i = 0; 18 | var unit = ''; 19 | if(a.toFixed === undefined) return a; 20 | while(i < 6 && a > 1024) { a = a/1024; i++; } 21 | switch(i) { 22 | case 1: unit = 'k'; break; 23 | case 2: unit = 'M'; break; 24 | case 3: unit = 'G'; break; 25 | case 4: unit = 'T'; break; 26 | case 5: unit = 'P'; break; 27 | case 6: unit = 'E'; break; 28 | default: 29 | } 30 | return a.toFixed(Math.min(i,3)) + unit; 31 | } 32 | 33 | function hexify(a) { 34 | return a.split(/|/) 35 | .map(function(a) { 36 | var r = a.charCodeAt(0).toString(16); 37 | if(r.length < 2) return "0" + r; 38 | return r; 39 | }).join('') 40 | } 41 | function $badge(n) { 42 | return $("").text(n); 43 | } 44 | function $label(n, type) { 45 | if(!type) type = "default"; 46 | return $("").text(n); 47 | } 48 | function alphaKeys(obj) { 49 | var keys = [] 50 | for(var key in obj) if(obj.hasOwnProperty(key)) keys.push(key); 51 | return keys.sort(); 52 | } 53 | var last_stats = {} 54 | var last_stats_time; 55 | var current_stats_time; 56 | function rate_calc(type, name, data, part) { 57 | if(!last_stats_time || 58 | !last_stats[type] || 59 | !last_stats[type][name]) return undefined; 60 | var dt = current_stats_time - last_stats_time; 61 | var dv = data[part] - last_stats[type][name][part]; 62 | return (dv/dt)*1000; /* /ms -> /s */ 63 | } 64 | 65 | function rate_queue_calc(type, name, data, i, part) { 66 | if(!last_stats_time || 67 | !last_stats[type] || 68 | !last_stats[type][name]) return undefined; 69 | if(last_stats[type][name].clients.length != data.length) return undefined; 70 | var dt = current_stats_time - last_stats_time; 71 | var dv = data[i][part] - last_stats[type][name].clients[i][part]; 72 | return (dv/dt)*1000; /* /ms -> /s */ 73 | } 74 | 75 | function update_exchange(name,detail) { 76 | var id = "exchange-" + hexify(name); 77 | var $tgt = $("div#" + id); 78 | if($tgt.length == 0) { 79 | if(name == "_aggregate") return; 80 | var $template = $("div#exchange-5f616767726567617465"); /* _aggregate */ 81 | $tgt = $template.clone(); 82 | $tgt.attr('id', id); 83 | $tgt.find('h4.panel-title').text(name); 84 | var $ip = $("#exchanges > div.row:last"); 85 | $ip.append($tgt); 86 | if($ip.find(">div").length == 3) { 87 | $("div#exchanges").append($("
")); 88 | } 89 | } 90 | ["no_exchange", "messages", "octets", "no_route", "routed", "dropped", "size_dropped"].forEach(function(attr) { 91 | var rate = rate_calc("exchanges", name, detail, attr); 92 | if(rate !== undefined) 93 | rate = parseFloat(rate).toFixed(0); 94 | else rate = ""; 95 | var value = pretty_number(parseFloat(detail[attr])); 96 | if(/NaN/.test(value)) value = ""; 97 | if(/NaN/.test(rate)) rate = ""; 98 | var ctag = attr.replace(/_/g, "-"); 99 | $tgt.find(".exchange-"+ctag).text(value); 100 | $tgt.find(".exchange-"+ctag+"-rate").text(rate); 101 | }); 102 | } 103 | 104 | function update_routes(name,detail) { 105 | if(name == "_aggregate") return; 106 | var routes = $("#routes"); 107 | var re = "route-exchange-" + hexify(name); 108 | var $panel = routes.find("#" + re); 109 | if($panel.length == 0) { 110 | $panel = $("#route-exchange-template").clone(); 111 | $panel.attr('id', re); 112 | routes.append($panel); 113 | $panel.find(".exchange-name").text(name); 114 | } else { 115 | $panel = $($panel[0]); 116 | } 117 | 118 | $panel.find(".route-row").addClass("updating"); 119 | var sortedRoutes = alphaKeys(detail); 120 | sortedRoutes.forEach(function(rid) { 121 | var r = detail[rid]; 122 | var $route = $panel.find("#route-" + rid); 123 | if($route.length == 0) { 124 | $route = $("#route-detail-template").clone(); 125 | $route.attr('id', "#route-" + rid); 126 | $panel.append($route); 127 | } else { 128 | $route = $($route[0]); 129 | } 130 | $route.find(".route-prefix").html($label(r.prefix || '[blank / no prefix]', "success")); 131 | $route.find(".route-mode").html($label(r.permanent ? "permanent" : "transient", r.permanent ? "primary" : "default")); 132 | var prog_sans_prefix = r.program.replace(/^prefix:"(?:\\.|[^"])*"\s*/, ""); 133 | $route.find(".route-program").text(prog_sans_prefix || '[ no program / match all ]'); 134 | $route.find(".route-invocations").text(pretty_number(parseInt(r.invocations))); 135 | $route.find(".route-avg-ns").text("" + r.avg_ns + "ns"); 136 | $route.removeClass("updating"); 137 | }); 138 | 139 | $panel.find(".route-row.updating").remove(); 140 | } 141 | 142 | function clear_exchanges() { 143 | $("#exchanges > div.row:not(:first-child)").remove() 144 | $("#exchanges > div.row > div.exchange-detail:not(:first-child)").remove(); 145 | $("#routes").empty(); 146 | } 147 | 148 | function mk_client(c) { 149 | return c.name; 150 | } 151 | 152 | function update_queue_row(name,detail) { 153 | var id = "queue-" + hexify(name); 154 | var $tgt = $("div#" + id); 155 | if($tgt.length == 0) { 156 | $tgt = $("#queue-template").clone(); 157 | $tgt.attr('id', id); 158 | var $collapse = $tgt.find(".panel-collapse"); 159 | $tgt.find(".panel-heading a").attr('aria-controls', id + "-clients").on('click', 160 | function() { $collapse.collapse('toggle'); } 161 | ); 162 | $collapse.attr('id', id + "-clients"); 163 | $("#queues").append($tgt); 164 | $(document).on('.data-api'); 165 | } 166 | 167 | $tgt.find(".queue-name").text(name); 168 | $tgt.find(".queue-type").empty() 169 | .append($label("type:" + detail.type, "success")); 170 | $tgt.find(".queue-exposure").empty() 171 | .append($label(detail.private?"private":"public", 172 | detail.private?"default":"primary")); 173 | $tgt.find(".queue-policy").empty() 174 | .append($label("policy:" + detail.policy, "danger")); 175 | 176 | var dropoutrate = rate_calc("queues", name, detail, "dropped_to"); 177 | if(dropoutrate !== undefined) dropoutrate = parseFloat(dropoutrate).toFixed(0); 178 | else dropoutrate = ""; 179 | var dropoutvalue = pretty_number(parseFloat(detail.dropped_to)); 180 | console.log(dropoutvalue, dropoutrate); 181 | $tgt.find(".queue-dropped-out-rate").text(dropoutrate); 182 | $tgt.find(".queue-dropped-out-value").text(dropoutvalue); 183 | 184 | var $pb = $tgt.find(".progress-bar"); 185 | var pct = 0; 186 | if(detail.backlog_limit > 0) pct = Math.floor(100 * detail.backlog / detail.backlog_limit); 187 | $pb.attr("style", "width: " + pct + "%"); 188 | $pb.attr('aria-valuemax', detail.backlog_limit); 189 | $pb.attr('aria-valuenow', detail.backlog); 190 | 191 | $pb.removeClass("progress-bar-success"); 192 | $pb.removeClass("progress-bar-warning"); 193 | $pb.addClass( pct > 75 ? "progress-bar-warning" : "progress-bar-success" ); 194 | if(detail.backlog_limit > 0) 195 | $tgt.find(".backlog").text(detail.backlog + "/" + detail.backlog_limit); 196 | else 197 | $tgt.find(".backlog").text(detail.backlog); 198 | 199 | var $clients = $tgt.find("table.clients tbody"); 200 | if($clients.find(">tr").length != detail.clients.length) { 201 | $clients.empty(); 202 | detail.clients.forEach(function() { 203 | $clients.append($("#client-template").clone().removeAttr('id')); 204 | }); 205 | } 206 | $clients = $tgt.find("table.clients tbody > tr"); 207 | for(var i=0;i<$clients.length;i++) { 208 | var $c = $($clients[i]), c = detail.clients[i]; 209 | $c.find(".client-user").text(c.user); 210 | $c.find(".client-address").text(c.remote_addr + ":" + c.remote_port); 211 | 212 | 213 | ["msgs_in", "octets_in", "msgs_out", "octets_out", 214 | "routed", "dropped", "size_dropped", "no_route", "no_exchange"].forEach(function(attr) { 215 | var rate = rate_queue_calc("queues", name, detail.clients, i, attr); 216 | if(rate !== undefined) rate = parseFloat(rate).toFixed(0); 217 | else rate = ""; 218 | var value = pretty_number(parseFloat(c[attr])); 219 | var ctag = attr.replace(/_/g, "-"); 220 | $c.find(".client-" + ctag + "-rate").text(rate); 221 | $c.find(".client-" + ctag + "-value").text(value); 222 | }); 223 | } 224 | } 225 | 226 | 227 | function refresh_stats() { 228 | $.ajax("/stats.json").done(function (x) { 229 | if(!x) return; 230 | current_stats_time = Date.now(); 231 | if(x.version) $("#fq-version").text(x.version); 232 | 233 | var sortedExchanges = alphaKeys(x.exchanges); 234 | if(!alphaKeys(last_stats.exchanges || {}).equals(sortedExchanges)) clear_exchanges(); 235 | sortedExchanges.forEach(function(exchange) { 236 | update_exchange(exchange, x.exchanges[exchange]); 237 | update_routes(exchange, x.exchanges[exchange].routes); 238 | }); 239 | 240 | var sortedQueues = alphaKeys(x.queues); 241 | if(!alphaKeys(last_stats.queues || {}).equals(sortedQueues)) 242 | $("#queues").empty(); 243 | sortedQueues.forEach(function(queue) { 244 | update_queue_row(queue, x.queues[queue]); 245 | }); 246 | 247 | last_stats_time = current_stats_time; 248 | last_stats = x; 249 | }); 250 | } 251 | 252 | setInterval(refresh_stats, 1000); 253 | refresh_stats(); 254 | --------------------------------------------------------------------------------