├── .clang-format ├── .editorconfig ├── .gitattributes ├── .gitignore ├── .travis.yml ├── AUTHORS ├── CMakeLists.txt ├── Dockerfile ├── Findcucumber.cmake ├── Findczmq.cmake ├── Findleveldb.cmake ├── Findlibzmq.cmake ├── LICENSE ├── Makefile.am ├── README.md ├── README.txt ├── api ├── dafka_consumer.api ├── dafka_consumer_msg.api ├── dafka_producer.api ├── dafka_producer_msg.api └── dafka_proto.api ├── autogen.sh ├── builds ├── check_zproject │ └── ci_build.sh ├── check_zproto │ └── ci_build.sh └── cmake │ ├── Config.cmake.in │ ├── Modules │ └── ClangFormat.cmake │ ├── ci_build.sh │ └── clang-format-check.sh.in ├── ci_build.sh ├── ci_deploy.sh ├── ci_deploy_obs.sh ├── configure.ac ├── doc ├── .gitignore ├── Makefile.am ├── asciidoc.conf ├── dafka.adoc └── mkman ├── features ├── dafka_consumer_protocol.feature └── dafka_producer_protocol.feature ├── images ├── README_1.png ├── README_2.png ├── README_3.png ├── README_4.png ├── README_5.png ├── README_6.png ├── README_7.png └── README_8.png ├── include ├── Makefile.am ├── dafka.h ├── dafka_beacon.h ├── dafka_consumer.h ├── dafka_consumer_msg.h ├── dafka_library.h ├── dafka_producer.h ├── dafka_producer_msg.h ├── dafka_proto.h ├── dafka_store.h └── dafka_tower.h ├── license.xml ├── project.xml └── src ├── .gitignore ├── .valgrind.supp ├── Makemodule-local.am ├── Makemodule.am ├── dafka_beacon.c ├── dafka_classes.h ├── dafka_console_consumer.c ├── dafka_console_producer.c ├── dafka_consumer.c ├── dafka_consumer_msg.c ├── dafka_consumer_step_defs.c ├── dafka_consumer_step_defs.h ├── dafka_consumer_step_runner.c ├── dafka_cucumber_selftest.c ├── dafka_fetch_filter.c ├── dafka_fetch_filter.h ├── dafka_head_key.c ├── dafka_head_key.h ├── dafka_msg_key.c ├── dafka_msg_key.h ├── dafka_perf_consumer.c ├── dafka_perf_store.c ├── dafka_private_selftest.c ├── dafka_producer.c ├── dafka_producer_msg.c ├── dafka_producer_step_defs.c ├── dafka_producer_step_defs.h ├── dafka_producer_step_runner.c ├── dafka_proto.bnf ├── dafka_proto.c ├── dafka_proto.xml ├── dafka_selftest.c ├── dafka_store.c ├── dafka_store_reader.c ├── dafka_store_reader.h ├── dafka_store_writer.c ├── dafka_store_writer.h ├── dafka_stored.c ├── dafka_test_peer.c ├── dafka_test_peer.h ├── dafka_tower.c ├── dafka_towerd.c ├── dafka_unacked_list.c ├── dafka_unacked_list.h ├── dafka_util.c ├── dafka_util.h ├── libdafka.pc.in └── selftest-ro └── .gitkeep /.clang-format: -------------------------------------------------------------------------------- 1 | # This is a skeleton created by zproject. 2 | # You can add hand-written code here. 3 | 4 | BasedOnStyle: LLVM 5 | IndentWidth: 4 6 | UseTab: Never 7 | BreakBeforeBraces: Custom 8 | BraceWrapping: 9 | AfterClass: true 10 | AfterControlStatement: false 11 | AfterEnum: true 12 | AfterFunction: true 13 | AfterNamespace: true 14 | AfterObjCDeclaration: true 15 | AfterStruct: true 16 | AfterUnion: true 17 | BeforeCatch: true 18 | BeforeElse: false 19 | IndentBraces: false 20 | 21 | AlignConsecutiveAssignments: false 22 | AlignConsecutiveDeclarations: false 23 | AllowShortIfStatementsOnASingleLine: false 24 | IndentCaseLabels: true 25 | BinPackArguments: true 26 | BinPackParameters: false 27 | AlignTrailingComments: true 28 | AllowShortBlocksOnASingleLine: false 29 | AllowAllParametersOfDeclarationOnNextLine: true 30 | AllowShortFunctionsOnASingleLine: InlineOnly 31 | AlwaysBreakTemplateDeclarations: false 32 | ColumnLimit: 80 33 | MaxEmptyLinesToKeep: 2 34 | KeepEmptyLinesAtTheStartOfBlocks: false 35 | ContinuationIndentWidth: 2 36 | PointerAlignment: Right 37 | ReflowComments: false 38 | SpaceBeforeAssignmentOperators: true 39 | SpaceBeforeParens: Always 40 | SpaceInEmptyParentheses: false 41 | SpacesInAngles: false 42 | SpacesInParentheses: false 43 | SpacesInSquareBrackets: false 44 | Standard: Cpp11 45 | 46 | SortIncludes: false 47 | 48 | FixNamespaceComments: false 49 | BreakBeforeBinaryOperators: NonAssignment 50 | SpaceAfterTemplateKeyword: true 51 | AlignAfterOpenBracket: Align 52 | AlignOperands: true 53 | BreakConstructorInitializers: AfterColon 54 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 55 | SpaceAfterCStyleCast: true 56 | BreakBeforeTernaryOperators: true 57 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This is a skeleton created by zproject. 2 | # You can add hand-written code here. 3 | # See http://editorconfig.org/ and https://github.com/editorconfig/ for 4 | # details about the format and links to plugins for IDEs and editors which 5 | # do not yet support this configuration out of the box - but easily can. 6 | 7 | # This is a top-level setting for project sources under this directory 8 | root = true 9 | 10 | [*] 11 | end_of_line = lf 12 | insert_final_newline = true 13 | indent_style = space 14 | indent_size = 4 15 | trim_trailing_whitespace = true 16 | charset = utf-8 17 | 18 | [*.am] 19 | indent_style = tab 20 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # This is a skeleton created by zproject. 2 | # You can add hand-written code here. 3 | # disables auto CRLF conversion for all files; create the file correctly and it will be allright 4 | * -text 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # This is a skeleton created by zproject. 2 | # You can add hand-written code here. 3 | save.xml 4 | # Object files 5 | *.o 6 | *.so 7 | *.ko 8 | *.pyc 9 | 10 | # Libraries 11 | *.lib 12 | *.a 13 | *.la 14 | *.pc 15 | *.dll.a 16 | 17 | # Shared objects (inc. Windows DLLs) 18 | *.dll 19 | *.so.* 20 | *.dylib 21 | *.lo 22 | 23 | # Executables 24 | src/dafka_selftest 25 | src/dafka_console_producer 26 | src/dafka_console_consumer 27 | src/dafka_stored 28 | src/dafka_towerd 29 | src/dafka_perf_consumer 30 | src/dafka_perf_store 31 | src/dafka_consumer_step_runner 32 | src/dafka_consumer_step_defs 33 | src/dafka_producer_step_defs 34 | src/dafka_producer_step_runner 35 | src/dafka_cucumber_selftest 36 | src/dafka_selftest 37 | *.exe 38 | *.out 39 | *.app 40 | core 41 | 42 | # Distcheck workspace and archives 43 | /dafka-*/ 44 | /dafka-*.tar.gz 45 | /dafka-*.zip 46 | 47 | # Man pages 48 | doc/*.1 49 | doc/*.3 50 | doc/*.7 51 | 52 | # autoconf files 53 | .deps 54 | .libs 55 | *.log 56 | *.trs 57 | Makefile 58 | Makefile.in 59 | !builds/openwrt/Makefile 60 | aclocal.m4 61 | autom4te.cache/ 62 | config.log 63 | config.status 64 | config/ 65 | configure 66 | configure.lineno 67 | libtool 68 | src/platform.h 69 | src/platform.h.in 70 | src/platform.h.in~ 71 | src/stamp-h1 72 | src/test-suite.log 73 | src/.dirstamp 74 | 75 | # CMake cache 76 | CMakeCache.txt 77 | CMakeFiles/ 78 | CTestTestfile.cmake 79 | cmake_install.cmake 80 | compile_commands.json 81 | DartConfiguration.tcl 82 | install_manifest.txt 83 | Testing/ 84 | 85 | # Repositories downloaded by CI integration scripts 86 | tmp-deps/ 87 | 88 | # Travis build area 89 | tmp/ 90 | 91 | # Valgrind files 92 | callgrind* 93 | vgcore* 94 | 95 | # Vagrant files 96 | .vagrant 97 | 98 | # CLion / PyCharm 99 | .idea 100 | 101 | # GYP files 102 | project.Makefile 103 | *.target.mk 104 | out/ 105 | 106 | # Android build results 107 | builds/android/prefix 108 | 109 | # Android - generated directories 110 | src/app/bin/ 111 | src/app/gen/ 112 | src/app/obj/ 113 | src/app/local.properties 114 | 115 | # Android - dependencies 116 | src/app/jni/output 117 | 118 | # Python build directory 119 | build/ 120 | 121 | # MSVC build directories 122 | obj/ 123 | bin/ 124 | *.tlog 125 | 126 | # Qt 127 | *pro.user 128 | moc_* 129 | build-* 130 | bindings/qt/selftest/selftest 131 | 132 | # Temporary files 133 | *.swp 134 | *.bak 135 | .test* 136 | 137 | # Netbeans directory 138 | nbproject/ 139 | 140 | # editor backups 141 | *~ 142 | 143 | # ccls cache 144 | .ccls-cache/ 145 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Travis CI script 2 | # This is a skeleton created by zproject. 3 | # You can add hand-written code here. 4 | 5 | language: 6 | - c 7 | 8 | cache: 9 | - ccache 10 | 11 | os: 12 | - linux 13 | - osx 14 | 15 | dist: xenial 16 | 17 | sudo: false 18 | 19 | compiler: 20 | - gcc 21 | 22 | #services: 23 | #- docker 24 | 25 | # Set CI_TIME=true to enable build-step profiling in Travis 26 | # Set CI_TRACE=true to enable shell script tracing in Travis 27 | # Set CI_CONFIG_QUIET=true to enable "configure --quiet" (only report stderr) 28 | # Set CI_REQUIRE_GOOD_GITIGNORE=false to NOT fail if "git status -s" is not clean 29 | # Set CI_REQUIRE_GOOD_CLANG_FORMAT=true to fail if "clang-format" check is not clean 30 | env: 31 | global: 32 | - CI_TIME=false 33 | - CI_TRACE=false 34 | - CI_CONFIG_QUIET=true 35 | - CI_REQUIRE_GOOD_GITIGNORE=false 36 | - CI_REQUIRE_GOOD_CLANG_FORMAT=false 37 | - CI_TEST_DISTCHECK=false 38 | # tokens to deploy releases on OBS and create/delete temporary branch on Github. 39 | # 1) Create a token on https://github.com/settings/tokens/new with "public_repo" 40 | # capability and encrypt it with travis encrypt --org -r / GH_TOKEN="" 41 | # 2) Create 2 OBS tokens with osc token --create network:messaging:zeromq:release- 42 | # encrypt them with travis encrypt --org -r / OBS__TOKEN="" 43 | # 3) Uncomment the three "secure" lines and paste the three generated hashed 44 | # strings, which include each token's name, as parameters 45 | #- secure: 46 | #- secure: 47 | #- secure: 48 | matrix: 49 | - BUILD_TYPE=default 50 | - BUILD_TYPE=default-Werror 51 | - BUILD_TYPE=cmake 52 | # - BUILD_TYPE=android 53 | # - BUILD_TYPE=check-py 54 | 55 | # Prerequisite packages provided by OS distro and used "as is" 56 | pkg_deps_prereqs_distro: &pkg_deps_prereqs_distro 57 | - libleveldb-dev 58 | 59 | # Prerequisite packages that may be built from source or used from 60 | # prebuilt packages of that source (usually not from an OS distro) 61 | pkg_deps_prereqs_source: &pkg_deps_prereqs_source 62 | - libzmq3-dev 63 | # - libczmq-dev 64 | 65 | pkg_deps_prereqs: &pkg_deps_prereqs 66 | - *pkg_deps_prereqs_source 67 | - *pkg_deps_prereqs_distro 68 | 69 | pkg_deps_doctools: &pkg_deps_doctools 70 | - asciidoc 71 | - xmlto 72 | 73 | pkg_deps_devtools: &pkg_deps_devtools 74 | - git 75 | 76 | pkg_src_zeromq_ubuntu16: &pkg_src_zeromq_ubuntu16 77 | - sourceline: 'deb http://download.opensuse.org/repositories/network:/messaging:/zeromq:/git-draft/xUbuntu_16.04/ ./' 78 | key_url: 'http://download.opensuse.org/repositories/network:/messaging:/zeromq:/git-draft/xUbuntu_16.04/Release.key' 79 | 80 | # Note: refer to ubuntu16 if you use dist==xenial 81 | # Also note that as of early 2017, either dist==trusty or services==docker 82 | # is needed for C++11 support; docker envs are usually faster to start up 83 | addons: 84 | apt: 85 | sources: *pkg_src_zeromq_ubuntu16 86 | packages: &pkg_deps_common 87 | - *pkg_deps_devtools 88 | - *pkg_deps_prereqs 89 | homebrew: 90 | packages: 91 | - leveldb 92 | - cjson 93 | 94 | matrix: 95 | include: 96 | - env: BUILD_TYPE=default-with-docs 97 | os: linux 98 | addons: 99 | apt: 100 | sources: *pkg_src_zeromq_ubuntu16 101 | packages: 102 | - *pkg_deps_common 103 | - *pkg_deps_doctools 104 | - env: BUILD_TYPE=valgrind 105 | os: linux 106 | dist: xenial 107 | addons: 108 | apt: 109 | sources: *pkg_src_zeromq_ubuntu16 110 | packages: 111 | - valgrind 112 | - *pkg_deps_common 113 | - env: BUILD_TYPE=default ADDRESS_SANITIZER=enabled 114 | os: linux 115 | dist: xenial 116 | addons: 117 | apt: 118 | sources: *pkg_src_zeromq_ubuntu16 119 | packages: 120 | - *pkg_deps_common 121 | - env: BUILD_TYPE=cmake DO_CLANG_FORMAT_CHECK=1 CLANG_FORMAT=clang-format-5.0 122 | # For non-cmake users, there is an autotools solution with a bit more overhead 123 | # to have dependencies ready and pass configure script before making this check). 124 | # Note that the autotools variant will also require dependencies preinstalled to 125 | # pass its configure script: 126 | # - env: BUILD_TYPE=clang-format-check CLANG_FORMAT=clang-format-5.0 127 | os: linux 128 | dist: xenial 129 | addons: 130 | apt: 131 | sources: 132 | - llvm-toolchain-xenial-5.0 133 | packages: 134 | - *pkg_deps_prereqs_distro 135 | - clang-5.0 136 | - clang-format-5.0 137 | #autotools# - *pkg_deps_prereqs 138 | 139 | before_install: 140 | - if [ "$TRAVIS_OS_NAME" == "osx" -a "$BUILD_TYPE" == "android" ] ; then brew install binutils ; fi 141 | - if [ "$TRAVIS_OS_NAME" == "osx" -a "$BUILD_TYPE" == "valgrind" ] ; then brew install valgrind ; fi 142 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew install leveldb cjson ; fi 143 | 144 | # Hand off to generated script for each BUILD_TYPE 145 | script: ./ci_build.sh 146 | before_deploy: . ./ci_deploy.sh && ./ci_deploy_obs.sh 147 | deploy: 148 | provider: releases 149 | api_key: 150 | # To encrypt your access token run: `travis encrypt -r user/repo` 151 | secure: 152 | file_glob: true 153 | file: ${DAFKA_DEPLOYMENT} 154 | skip_cleanup: true 155 | on: 156 | branch: master 157 | tags: true 158 | condition: $TRAVIS_OS_NAME =~ (linux) && $BUILD_TYPE =~ (default) 159 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Corporate Copyright Statements 2 | ============================== 3 | 4 | Contributors 5 | ============ 6 | 7 | Doron Somech 8 | Kevin Sapper 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | MAINTAINER dafka Developers 3 | 4 | RUN DEBIAN_FRONTEND=noninteractive apt-get update -y -q 5 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q --force-yes build-essential git-core libtool autotools-dev autoconf automake pkg-config unzip libkrb5-dev cmake 6 | RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q --force-yes \ 7 | libzmq3-dev \ 8 | libleveldb-dev \ 9 | libcjson-dev 10 | 11 | RUN useradd -d /home/zmq -m -s /bin/bash zmq 12 | RUN echo "zmq ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers 13 | 14 | WORKDIR /tmp 15 | RUN git clone --quiet https://github.com/zeromq/czmq.git czmq 16 | WORKDIR /tmp/czmq 17 | RUN ./autogen.sh 2> /dev/null 18 | RUN ./configure --quiet --without-docs 19 | RUN make 20 | RUN make install 21 | RUN ldconfig 22 | 23 | WORKDIR /tmp 24 | RUN git clone --quiet https://github.com/cucumber/gherkin-c gherkin 25 | WORKDIR /tmp/gherkin 26 | RUN ./autogen.sh 2> /dev/null 27 | RUN ./configure --quiet --without-docs 28 | RUN make 29 | RUN make install 30 | RUN ldconfig 31 | 32 | WORKDIR /tmp 33 | RUN git clone --quiet https://github.com/zeromq/dafka dafka 34 | WORKDIR /tmp/dafka 35 | RUN ./autogen.sh 2> /dev/null 36 | RUN ./configure --quiet --without-docs 37 | RUN make 38 | RUN make install 39 | RUN ldconfig 40 | 41 | 42 | USER zmq 43 | -------------------------------------------------------------------------------- /Findcucumber.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | if (NOT MSVC) 7 | include(FindPkgConfig) 8 | pkg_check_modules(PC_CUCUMBER "cucumber") 9 | if (PC_CUCUMBER_FOUND) 10 | # add CFLAGS from pkg-config file, e.g. draft api. 11 | add_definitions(${PC_CUCUMBER_CFLAGS} ${PC_CUCUMBER_CFLAGS_OTHER}) 12 | # some libraries install the headers is a subdirectory of the include dir 13 | # returned by pkg-config, so use a wildcard match to improve chances of finding 14 | # headers and SOs. 15 | set(PC_CUCUMBER_INCLUDE_HINTS ${PC_CUCUMBER_INCLUDE_DIRS} ${PC_CUCUMBER_INCLUDE_DIRS}/*) 16 | set(PC_CUCUMBER_LIBRARY_HINTS ${PC_CUCUMBER_LIBRARY_DIRS} ${PC_CUCUMBER_LIBRARY_DIRS}/*) 17 | endif(PC_CUCUMBER_FOUND) 18 | endif (NOT MSVC) 19 | 20 | find_path ( 21 | CUCUMBER_INCLUDE_DIRS 22 | NAMES cucumber_c.h 23 | HINTS ${PC_CUCUMBER_INCLUDE_HINTS} 24 | ) 25 | 26 | find_library ( 27 | CUCUMBER_LIBRARIES 28 | NAMES cucumber 29 | HINTS ${PC_CUCUMBER_LIBRARY_HINTS} 30 | ) 31 | 32 | include(FindPackageHandleStandardArgs) 33 | 34 | find_package_handle_standard_args( 35 | CUCUMBER 36 | REQUIRED_VARS CUCUMBER_LIBRARIES CUCUMBER_INCLUDE_DIRS 37 | ) 38 | mark_as_advanced( 39 | CUCUMBER_FOUND 40 | CUCUMBER_LIBRARIES CUCUMBER_INCLUDE_DIRS 41 | ) 42 | 43 | ################################################################################ 44 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 45 | # Read the zproject/README.md for information about making permanent changes. # 46 | ################################################################################ 47 | -------------------------------------------------------------------------------- /Findczmq.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | if (NOT MSVC) 7 | include(FindPkgConfig) 8 | pkg_check_modules(PC_CZMQ "libczmq") 9 | if (PC_CZMQ_FOUND) 10 | # add CFLAGS from pkg-config file, e.g. draft api. 11 | add_definitions(${PC_CZMQ_CFLAGS} ${PC_CZMQ_CFLAGS_OTHER}) 12 | # some libraries install the headers is a subdirectory of the include dir 13 | # returned by pkg-config, so use a wildcard match to improve chances of finding 14 | # headers and SOs. 15 | set(PC_CZMQ_INCLUDE_HINTS ${PC_CZMQ_INCLUDE_DIRS} ${PC_CZMQ_INCLUDE_DIRS}/*) 16 | set(PC_CZMQ_LIBRARY_HINTS ${PC_CZMQ_LIBRARY_DIRS} ${PC_CZMQ_LIBRARY_DIRS}/*) 17 | endif(PC_CZMQ_FOUND) 18 | endif (NOT MSVC) 19 | 20 | find_path ( 21 | CZMQ_INCLUDE_DIRS 22 | NAMES czmq.h 23 | HINTS ${PC_CZMQ_INCLUDE_HINTS} 24 | ) 25 | 26 | find_library ( 27 | CZMQ_LIBRARIES 28 | NAMES libczmq czmq 29 | HINTS ${PC_CZMQ_LIBRARY_HINTS} 30 | ) 31 | 32 | include(FindPackageHandleStandardArgs) 33 | 34 | find_package_handle_standard_args( 35 | CZMQ 36 | REQUIRED_VARS CZMQ_LIBRARIES CZMQ_INCLUDE_DIRS 37 | ) 38 | mark_as_advanced( 39 | CZMQ_FOUND 40 | CZMQ_LIBRARIES CZMQ_INCLUDE_DIRS 41 | ) 42 | 43 | ################################################################################ 44 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 45 | # Read the zproject/README.md for information about making permanent changes. # 46 | ################################################################################ 47 | -------------------------------------------------------------------------------- /Findleveldb.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | if (NOT MSVC) 7 | include(FindPkgConfig) 8 | pkg_check_modules(PC_LEVELDB "leveldb") 9 | if (PC_LEVELDB_FOUND) 10 | # add CFLAGS from pkg-config file, e.g. draft api. 11 | add_definitions(${PC_LEVELDB_CFLAGS} ${PC_LEVELDB_CFLAGS_OTHER}) 12 | # some libraries install the headers is a subdirectory of the include dir 13 | # returned by pkg-config, so use a wildcard match to improve chances of finding 14 | # headers and SOs. 15 | set(PC_LEVELDB_INCLUDE_HINTS ${PC_LEVELDB_INCLUDE_DIRS} ${PC_LEVELDB_INCLUDE_DIRS}/*) 16 | set(PC_LEVELDB_LIBRARY_HINTS ${PC_LEVELDB_LIBRARY_DIRS} ${PC_LEVELDB_LIBRARY_DIRS}/*) 17 | endif(PC_LEVELDB_FOUND) 18 | endif (NOT MSVC) 19 | 20 | find_path ( 21 | LEVELDB_INCLUDE_DIRS 22 | NAMES leveldb/c.h 23 | HINTS ${PC_LEVELDB_INCLUDE_HINTS} 24 | ) 25 | 26 | find_library ( 27 | LEVELDB_LIBRARIES 28 | NAMES leveldb 29 | HINTS ${PC_LEVELDB_LIBRARY_HINTS} 30 | ) 31 | 32 | include(FindPackageHandleStandardArgs) 33 | 34 | find_package_handle_standard_args( 35 | LEVELDB 36 | REQUIRED_VARS LEVELDB_LIBRARIES LEVELDB_INCLUDE_DIRS 37 | ) 38 | mark_as_advanced( 39 | LEVELDB_FOUND 40 | LEVELDB_LIBRARIES LEVELDB_INCLUDE_DIRS 41 | ) 42 | 43 | ################################################################################ 44 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 45 | # Read the zproject/README.md for information about making permanent changes. # 46 | ################################################################################ 47 | -------------------------------------------------------------------------------- /Findlibzmq.cmake: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | if (NOT MSVC) 7 | include(FindPkgConfig) 8 | pkg_check_modules(PC_LIBZMQ "libzmq") 9 | if (PC_LIBZMQ_FOUND) 10 | # add CFLAGS from pkg-config file, e.g. draft api. 11 | add_definitions(${PC_LIBZMQ_CFLAGS} ${PC_LIBZMQ_CFLAGS_OTHER}) 12 | # some libraries install the headers is a subdirectory of the include dir 13 | # returned by pkg-config, so use a wildcard match to improve chances of finding 14 | # headers and SOs. 15 | set(PC_LIBZMQ_INCLUDE_HINTS ${PC_LIBZMQ_INCLUDE_DIRS} ${PC_LIBZMQ_INCLUDE_DIRS}/*) 16 | set(PC_LIBZMQ_LIBRARY_HINTS ${PC_LIBZMQ_LIBRARY_DIRS} ${PC_LIBZMQ_LIBRARY_DIRS}/*) 17 | endif(PC_LIBZMQ_FOUND) 18 | endif (NOT MSVC) 19 | 20 | find_path ( 21 | LIBZMQ_INCLUDE_DIRS 22 | NAMES zmq.h 23 | HINTS ${PC_LIBZMQ_INCLUDE_HINTS} 24 | ) 25 | 26 | if (MSVC) 27 | # libzmq dll/lib built with MSVC is named using the Boost convention. 28 | # https://github.com/zeromq/czmq/issues/577 29 | # https://github.com/zeromq/czmq/issues/1972 30 | if (MSVC_IDE) 31 | set(MSVC_TOOLSET "-${CMAKE_VS_PLATFORM_TOOLSET}") 32 | else () 33 | set(MSVC_TOOLSET "") 34 | endif () 35 | 36 | # Retrieve ZeroMQ version number from zmq.h 37 | file(STRINGS "${LIBZMQ_INCLUDE_DIRS}/zmq.h" zmq_version_defines 38 | REGEX "#define ZMQ_VERSION_(MAJOR|MINOR|PATCH)") 39 | foreach(ver ${zmq_version_defines}) 40 | if(ver MATCHES "#define ZMQ_VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$") 41 | set(ZMQ_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "") 42 | endif() 43 | endforeach() 44 | 45 | set(_zmq_version ${ZMQ_VERSION_MAJOR}_${ZMQ_VERSION_MINOR}_${ZMQ_VERSION_PATCH}) 46 | 47 | set(_zmq_debug_names 48 | "libzmq${MSVC_TOOLSET}-mt-gd-${_zmq_version}" # Debug, BUILD_SHARED 49 | "libzmq${MSVC_TOOLSET}-mt-sgd-${_zmq_version}" # Debug, BUILD_STATIC 50 | "libzmq-mt-gd-${_zmq_version}" # Debug, BUILD_SHARED 51 | "libzmq-mt-sgd-${_zmq_version}" # Debug, BUILD_STATIC 52 | ) 53 | 54 | set(_zmq_release_names 55 | "libzmq${MSVC_TOOLSET}-mt-${_zmq_version}" # Release|RelWithDebInfo|MinSizeRel, BUILD_SHARED 56 | "libzmq${MSVC_TOOLSET}-mt-s-${_zmq_version}" # Release|RelWithDebInfo|MinSizeRel, BUILD_STATIC 57 | "libzmq-mt-${_zmq_version}" # Release|RelWithDebInfo|MinSizeRel, BUILD_SHARED 58 | "libzmq-mt-s-${_zmq_version}" # Release|RelWithDebInfo|MinSizeRel, BUILD_STATIC 59 | ) 60 | 61 | find_library (LIBZMQ_LIBRARY_DEBUG 62 | NAMES ${_zmq_debug_names} 63 | ) 64 | 65 | find_library (LIBZMQ_LIBRARY_RELEASE 66 | NAMES ${_zmq_release_names} 67 | ) 68 | 69 | include(SelectLibraryConfigurations) 70 | select_library_configurations(LIBZMQ) 71 | endif () 72 | 73 | if (NOT LIBZMQ_LIBRARIES) 74 | find_library ( 75 | LIBZMQ_LIBRARIES 76 | NAMES libzmq zmq 77 | HINTS ${PC_LIBZMQ_LIBRARY_HINTS} 78 | ) 79 | endif () 80 | 81 | include(FindPackageHandleStandardArgs) 82 | 83 | find_package_handle_standard_args( 84 | LIBZMQ 85 | REQUIRED_VARS LIBZMQ_LIBRARIES LIBZMQ_INCLUDE_DIRS 86 | ) 87 | mark_as_advanced( 88 | LIBZMQ_FOUND 89 | LIBZMQ_LIBRARIES LIBZMQ_INCLUDE_DIRS 90 | ) 91 | 92 | ################################################################################ 93 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 94 | # Read the zproject/README.md for information about making permanent changes. # 95 | ################################################################################ 96 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | ACLOCAL_AMFLAGS = -I config 7 | 8 | 9 | AM_CPPFLAGS = \ 10 | ${libzmq_CFLAGS} \ 11 | ${czmq_CFLAGS} \ 12 | ${leveldb_CFLAGS} \ 13 | ${cucumber_CFLAGS} \ 14 | -I$(srcdir)/include 15 | 16 | project_libs = ${libzmq_LIBS} ${czmq_LIBS} ${leveldb_LIBS} ${cucumber_LIBS} 17 | 18 | SUBDIRS = doc 19 | SUBDIRS += include 20 | DIST_SUBDIRS = doc 21 | DIST_SUBDIRS += include 22 | 23 | lib_LTLIBRARIES = 24 | bin_PROGRAMS = 25 | noinst_PROGRAMS = 26 | check_PROGRAMS = 27 | noinst_LTLIBRARIES = 28 | TESTS = 29 | # Prepare variables that can be populated (appended) in generated Makefiles or 30 | # manually maintained src/Makemodule-local.am 31 | EXTRA_DIST = 32 | CLEANFILES = 33 | DISTCLEANFILES = 34 | 35 | if ENABLE_DIST_CMAKEFILES 36 | EXTRA_DIST += \ 37 | Findlibzmq.cmake \ 38 | Findczmq.cmake \ 39 | Findleveldb.cmake \ 40 | Findcucumber.cmake \ 41 | builds/cmake/Modules/ClangFormat.cmake \ 42 | builds/cmake/clang-format-check.sh.in \ 43 | builds/cmake/Config.cmake.in \ 44 | CMakeLists.txt 45 | endif 46 | 47 | EXTRA_DIST += \ 48 | src/dafka_unacked_list.h \ 49 | src/dafka_fetch_filter.h \ 50 | src/dafka_msg_key.h \ 51 | src/dafka_head_key.h \ 52 | src/dafka_util.h \ 53 | src/dafka_consumer_step_defs.h \ 54 | src/dafka_producer_step_defs.h \ 55 | src/dafka_store_reader.h \ 56 | src/dafka_store_writer.h \ 57 | src/dafka_test_peer.h \ 58 | LICENSE \ 59 | README.txt \ 60 | README.md \ 61 | src/dafka_classes.h 62 | 63 | # NOTE: this "include" syntax is not a "make" but an "autotools" keyword, 64 | # see https://www.gnu.org/software/automake/manual/html_node/Include.html 65 | include $(srcdir)/src/Makemodule.am 66 | include $(srcdir)/src/Makemodule-local.am # Optional project-local hook 67 | 68 | ################################################################################ 69 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 70 | # Read the zproject/README.md for information about making permanent changes. # 71 | ################################################################################ 72 | -------------------------------------------------------------------------------- /api/dafka_consumer.api: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Creates a new dafka consumer client that runs in its own background thread. 5 | 6 | The args parameter consists of configuration and record sink. 7 | 8 | If a record sink is provided this socket will be used the send the consumer 9 | messages to. 10 | 11 | The configuration argument takes settings for both the consumer and the 12 | beacon, see below. 13 | 14 | Consumer configuration: 15 | * consumer/offset/reset = earliest|latest (default: latest) 16 | * consumer/high_watermark (default: 1.000.000) 17 | * consumer/verbose = 0|1 (default: 0 -> false) 18 | 19 | Beacon configuration: 20 | * beacon/interval (default: 1000) in ms 21 | * beacon/verbose = 0|1 (default: 0 -> false) 22 | * beacon/sub_address (default: tcp://127.0.0.1:5556) 23 | * beacon/pub_address (default: tcp://127.0.0.1:5557) 24 | 25 | 26 | 27 | 28 | Destroys an instance of dafka consumer client by gracefully stopping its 29 | background thread. 30 | 31 | 32 | 33 | Subscribe to a given topic. 34 | 35 | 36 | 37 | 38 | 39 | Unsubscribe from a topic currently subscribed to. 40 | 41 | 42 | 43 | 44 | 45 | Returns the address of the consumer instance. 46 | 47 | 48 | 49 | 50 | Get the current subscription as list of strings. 51 | 52 | 53 | 54 | 55 | Returns the internal record source socket. 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /api/dafka_consumer_msg.api: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Return the subject of the message. 10 | 11 | 12 | 13 | 14 | Return the sender address of the message. 15 | 16 | 17 | 18 | 19 | Return the content of the message. 20 | Content buffer is belong to the message. 21 | 22 | 23 | 24 | 25 | Return the size of the content 26 | 27 | 28 | 29 | 30 | Return the content, user takes ownership of the frame returned. 31 | 32 | 33 | 34 | 35 | Receive a message from a consumer actor. 36 | Return 0 on success and -1 on error. 37 | 38 | 39 | 40 | 41 | 42 | Receive a message from a custom socket. 43 | Return 0 on success and -1 on error. 44 | 45 | 46 | 47 | 48 | 49 | Return frame data copied into freshly allocated string 50 | Caller must free string when finished with it. 51 | 52 | 53 | 54 | 55 | Return TRUE if content is equal to string, excluding terminator 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /api/dafka_producer.api: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /api/dafka_producer_msg.api: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Return the content of the message. 10 | Content buffer is belong to the message and user should not try to free it. 11 | 12 | 13 | 14 | 15 | Return the size of the content 16 | 17 | 18 | 19 | 20 | Return the content, user takes ownership of the buffer returned. 21 | 22 | 23 | 24 | 25 | Create a new content buffer at the specific size. 26 | Return 0 on success and -1 if not enough memory. 27 | 28 | 29 | 30 | 31 | 32 | Set the content with a frame. Takes ownership on the content. 33 | 34 | 35 | 36 | 37 | Set the content from string. 38 | Return 0 on success and -1 if not enough memory. 39 | 40 | 41 | 42 | 43 | 44 | Set the content from string. 45 | Return 0 on success and -1 if not enough memory. 46 | 47 | 48 | 49 | 50 | 51 | 52 | Send a message to producer. 53 | Content will ownership will be moved to a background thread and will be set to NULL. 54 | Return 0 on success and -1 on error. 55 | 56 | 57 | 58 | 59 | 60 | Return frame data copied into freshly allocated string 61 | Caller must free string when finished with it. 62 | 63 | 64 | 65 | 66 | Return TRUE if content is equal to string, excluding terminator 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /api/dafka_proto.api: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | Create a new empty dafka_proto 47 | 48 | 49 | 50 | Create a new dafka_proto from zpl/zconfig_t * 51 | 52 | 53 | 54 | 55 | Destroy a dafka_proto instance 56 | 57 | 58 | 59 | Create a deep copy of a dafka_proto instance 60 | 61 | 62 | 63 | 64 | Receive a dafka_proto from the socket. Returns 0 if OK, -1 if 65 | there was an error. Blocks if there is no message waiting. 66 | 67 | 68 | 69 | 70 | 71 | Send the dafka_proto to the output socket, does not destroy it 72 | 73 | 74 | 75 | 76 | 77 | Encode the first frame of dafka_proto. Does not destroy it. Returns the frame if OK, else NULL. 78 | 79 | 80 | 81 | 82 | Print contents of message to stdout 83 | 84 | 85 | 86 | Export class as zconfig_t*. Caller is responsibe for destroying the instance 87 | 88 | 89 | 90 | 91 | 92 | Get the message routing id, as a frame 93 | 94 | 95 | 96 | 97 | Set the message routing id from a frame 98 | 99 | 100 | 101 | 102 | Get the topic of the message for publishing over pub/sub 103 | 104 | 105 | 106 | 107 | Set the topic of the message for publishing over pub/sub 108 | 109 | 110 | 111 | 112 | Subscribe a socket to a specific message id and a topic. 113 | 114 | 115 | 116 | 117 | 118 | 119 | Unsubscribe a socket form a specific message id and a topic. 120 | 121 | 122 | 123 | 124 | 125 | 126 | Get whether the last subscription received on XPUB socket was of type subscribe. 127 | 128 | 129 | 130 | 131 | Get the dafka_proto message id 132 | 133 | 134 | 135 | 136 | Set the dafka_proto message id 137 | 138 | 139 | 140 | 141 | Get the dafka_proto message id as printable text 142 | 143 | 144 | 145 | 146 | Get the address field 147 | 148 | 149 | 150 | Set the address field 151 | 152 | 153 | 154 | Get the subject field 155 | 156 | 157 | 158 | Set the subject field 159 | 160 | 161 | 162 | Get the sequence field 163 | 164 | 165 | 166 | Set the sequence field 167 | 168 | 169 | 170 | Get a copy of the content field 171 | 172 | 173 | 174 | Get the content field and transfer ownership to caller 175 | 176 | 177 | 178 | Set the content field, transferring ownership from caller 179 | 180 | 181 | 182 | Get the count field 183 | 184 | 185 | 186 | Set the count field 187 | 188 | 189 | 190 | Get the subjects field 191 | 192 | 193 | 194 | Get the subjects field and transfer ownership to caller 195 | 196 | 197 | 198 | Set the subjects field 199 | 200 | 201 | 202 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | ################################################################################ 3 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 4 | # Read the zproject/README.md for information about making permanent changes. # 5 | ################################################################################ 6 | 7 | # Script to generate all required files from fresh git checkout. 8 | 9 | if [ ! -f src/Makemodule-local.am ] ; then 10 | echo "autogen.sh: generating a dummy src/Makemodule-local.am to fulfill dependencies." 1>&2 11 | touch src/Makemodule-local.am 12 | fi 13 | 14 | # Debian and Ubuntu do not shipt libtool anymore, but OSX does not ship libtoolize. 15 | command -v libtoolize >/dev/null 2>&1 16 | if [ $? -ne 0 ]; then 17 | command -v libtool >/dev/null 2>&1 18 | if [ $? -ne 0 ]; then 19 | echo "autogen.sh: error: could not find libtool. libtool is required to run autogen.sh." 1>&2 20 | exit 1 21 | fi 22 | fi 23 | 24 | command -v autoreconf >/dev/null 2>&1 25 | if [ $? -ne 0 ]; then 26 | echo "autogen.sh: error: could not find autoreconf. autoconf and automake are required to run autogen.sh." 1>&2 27 | exit 1 28 | fi 29 | 30 | command -v pkg-config >/dev/null 2>&1 31 | if [ $? -ne 0 ]; then 32 | echo "autogen.sh: error: could not find pkg-config. pkg-config is required to run autogen.sh." 1>&2 33 | exit 1 34 | fi 35 | 36 | mkdir -p ./config 37 | if [ $? -ne 0 ]; then 38 | echo "autogen.sh: error: could not create directory: ./config." 1>&2 39 | exit 1 40 | fi 41 | 42 | autoreconf --install --force --verbose -I config 43 | status=$? 44 | if [ $status -ne 0 ]; then 45 | echo "autogen.sh: error: autoreconf exited with status $status" 1>&2 46 | exit 1 47 | fi 48 | 49 | ################################################################################ 50 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 51 | # Read the zproject/README.md for information about making permanent changes. # 52 | ################################################################################ 53 | -------------------------------------------------------------------------------- /builds/check_zproject/ci_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # NOTE: This script is not standalone, it is included from project root 5 | # ci_build.sh script, which sets some envvars (like REPO_DIR below). 6 | [ -n "${REPO_DIR-}" ] || exit 1 7 | 8 | # Verify all required dependencies with repos can be checked out 9 | # Note: certain zproject scripts that deal with deeper dependencies expect that 10 | # such checkouts are directly in the same parent directory as "this" project. 11 | cd "$REPO_DIR/.." 12 | git clone --quiet --depth 1 https://github.com/zeromq/libzmq.git libzmq 13 | git clone --quiet --depth 1 https://github.com/zeromq/czmq.git czmq 14 | cd - 15 | 16 | if ! ((command -v dpkg >/dev/null 2>&1 && dpkg -s zproject >/dev/null 2>&1) || \ 17 | (command -v brew >/dev/null 2>&1 && brew ls --versions zproject >/dev/null 2>&1)) \ 18 | ; then 19 | cd "$REPO_DIR/.." 20 | git clone --quiet --depth 1 https://github.com/zeromq/zproject zproject 21 | cd zproject 22 | PATH="`pwd`:$PATH" 23 | fi 24 | 25 | if ! ((command -v dpkg >/dev/null 2>&1 && dpkg -s generator-scripting-language >/dev/null 2>&1) || \ 26 | (command -v brew >/dev/null 2>&1 && brew ls --versions gsl >/dev/null 2>&1)) \ 27 | ; then 28 | cd "$REPO_DIR/.." 29 | git clone https://github.com/zeromq/gsl.git gsl 30 | cd gsl/src 31 | make 32 | PATH="`pwd`:$PATH" 33 | fi 34 | export PATH 35 | 36 | # Verify that zproject template is up-to-date with files it can overwrite 37 | # As we will overwrite this script file make sure bash loads the 38 | # next lines into memory before executing 39 | # http://stackoverflow.com/questions/21096478/overwrite-executing-bash-script-files 40 | { 41 | cd "$REPO_DIR" 42 | gsl project.xml 43 | 44 | # keep an eye on git version used by CI 45 | git --version 46 | if [[ $(git --no-pager diff -w) ]]; then 47 | git --no-pager diff -w 48 | echo "There are diffs between current code and code generated by zproject!" 49 | exit 1 50 | fi 51 | if [[ $(git status -s) ]]; then 52 | git status -s 53 | echo "zproject generated new files!" 54 | exit 1 55 | fi 56 | exit 0 57 | } 58 | -------------------------------------------------------------------------------- /builds/check_zproto/ci_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | 4 | # NOTE: This script is not standalone, it is included from project root 5 | # ci_build.sh script, which sets some envvars (like REPO_DIR below). 6 | [ -n "${REPO_DIR-}" ] || exit 1 7 | 8 | docker run -e GSL_BUILD_DIR=/code/src -v "$REPO_DIR":/code zeromqorg/zproto -zproject:1 -q dafka_proto.xml 9 | 10 | # keep an eye on git version used by CI 11 | git --version 12 | if [[ $(git --no-pager diff -w api/*) ]]; then 13 | git --no-pager diff -w api/* 14 | echo "There are diffs between current code and code generated by zproto!" 15 | exit 1 16 | fi 17 | if [[ $(git status -s api) ]]; then 18 | git status -s api 19 | echo "zproto generated new files!" 20 | exit 1 21 | fi 22 | -------------------------------------------------------------------------------- /builds/cmake/Config.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") 4 | check_required_components("@PROJECT_NAME@") 5 | -------------------------------------------------------------------------------- /builds/cmake/Modules/ClangFormat.cmake: -------------------------------------------------------------------------------- 1 | # additional target to perform clang-format run, requires clang-format 2 | 3 | # get all project files 4 | file(GLOB_RECURSE ALL_SOURCE_FILES 5 | RELATIVE ${CMAKE_CURRENT_BINARY_DIR} 6 | ${PROJECT_SOURCE_DIR}/src/*.c ${PROJECT_SOURCE_DIR}/src/*.cc ${PROJECT_SOURCE_DIR}/src/*.cpp 7 | ${PROJECT_SOURCE_DIR}/src/*.h ${PROJECT_SOURCE_DIR}/src/*.hpp 8 | ${PROJECT_SOURCE_DIR}/tests/*.c ${PROJECT_SOURCE_DIR}/tests/*.cc ${PROJECT_SOURCE_DIR}/tests/*.cpp 9 | ${PROJECT_SOURCE_DIR}/tests/*.h ${PROJECT_SOURCE_DIR}/tests/*.hpp 10 | ${PROJECT_SOURCE_DIR}/perf/*.c ${PROJECT_SOURCE_DIR}/perf/*.cc ${PROJECT_SOURCE_DIR}/perf/*.cpp 11 | ${PROJECT_SOURCE_DIR}/perf/*.h ${PROJECT_SOURCE_DIR}/perf/*.hpp 12 | ${PROJECT_SOURCE_DIR}/tools/*.c ${PROJECT_SOURCE_DIR}/tools/*.cc ${PROJECT_SOURCE_DIR}/tools/*.cpp 13 | ${PROJECT_SOURCE_DIR}/tools/*.h ${PROJECT_SOURCE_DIR}/tools/*.hpp 14 | ${PROJECT_SOURCE_DIR}/include/*.h ${PROJECT_SOURCE_DIR}/include/*.hpp 15 | ) 16 | 17 | if("${CLANG_FORMAT}" STREQUAL "") 18 | set(CLANG_FORMAT "clang-format") 19 | endif() 20 | 21 | if (NOT TARGET clang-format) 22 | add_custom_target( 23 | clang-format 24 | COMMAND ${CLANG_FORMAT} -style=file -i ${ALL_SOURCE_FILES} 25 | ) 26 | endif() 27 | 28 | function(JOIN VALUES GLUE OUTPUT) 29 | string (REPLACE ";" "${GLUE}" _TMP_STR "${VALUES}") 30 | set (${OUTPUT} "${_TMP_STR}" PARENT_SCOPE) 31 | endfunction() 32 | 33 | configure_file(builds/cmake/clang-format-check.sh.in clang-format-check.sh @ONLY) 34 | 35 | if (NOT TARGET clang-format-check) 36 | add_custom_target( 37 | clang-format-check 38 | COMMAND chmod +x clang-format-check.sh 39 | COMMAND ./clang-format-check.sh 40 | COMMENT "Checking correct formatting according to .clang-format file using ${CLANG_FORMAT}" 41 | ) 42 | endif() 43 | 44 | if (NOT TARGET clang-format-check-CI) 45 | add_custom_target( 46 | clang-format-check-CI 47 | COMMAND chmod +x clang-format-check.sh 48 | COMMAND ./clang-format-check.sh --CI 49 | COMMENT "Checking correct formatting according to .clang-format file using ${CLANG_FORMAT}" 50 | ) 51 | endif() 52 | 53 | 54 | if (NOT TARGET clang-format-diff) 55 | add_custom_target( 56 | clang-format-diff 57 | COMMAND ${CLANG_FORMAT} -style=file -i ${ALL_SOURCE_FILES} 58 | COMMAND git diff ${ALL_SOURCE_FILES} 59 | COMMENT "Formatting with clang-format (using ${CLANG_FORMAT}) and showing differences with latest commit" 60 | ) 61 | endif() 62 | 63 | -------------------------------------------------------------------------------- /builds/cmake/clang-format-check.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Use clang-format tool to verify this codebase conforms to our style standards 4 | 5 | FAILED=0 6 | IFS=";" 7 | FILES="@ALL_SOURCE_FILES@" 8 | IDS=$(echo -en "\n\b") 9 | [ -n "$MAKE" ] || MAKE=make 10 | for FILE in $FILES 11 | do 12 | @CLANG_FORMAT@ -style=file -output-replacements-xml "$FILE" | grep "/dev/null && \ 13 | { 14 | echo "$FILE is not correctly formatted" >&2 15 | FAILED=1 16 | } 17 | done 18 | if [ "$FAILED" -eq "1" ] ; then 19 | if [ "$1" = "--CI" ] ; then 20 | echo "Style mismatches were found by clang-format; detailing below:" >&2 21 | ${MAKE} clang-format-diff 22 | if test x"${CI_REQUIRE_GOOD_CLANG_FORMAT}" = xtrue ; then 23 | echo "FAILED : Style checks have failed and CI_REQUIRE_GOOD_CLANG_FORMAT==true" >&2 24 | exit 1 25 | fi 26 | echo "WARNING : Style checks have failed, but the result is not-fatal because CI_REQUIRE_GOOD_CLANG_FORMAT!=true" >&2 27 | exit 0 28 | fi 29 | exit 1 30 | fi 31 | -------------------------------------------------------------------------------- /ci_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################################################ 4 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 5 | # READ THE ZPROJECT/README.MD FOR INFORMATION ABOUT MAKING PERMANENT CHANGES. # 6 | ################################################################################ 7 | 8 | set -x 9 | set -e 10 | 11 | if [ "$BUILD_TYPE" == "default" ]; then 12 | # Tell travis to deploy all files in dist 13 | mkdir dist 14 | export DAFKA_DEPLOYMENT=dist/* 15 | # Move archives to dist 16 | mv *.tar.gz dist 17 | mv *.zip dist 18 | # Generate hash sums 19 | cd dist 20 | md5sum *.zip *.tar.gz > MD5SUMS 21 | sha1sum *.zip *.tar.gz > SHA1SUMS 22 | cd - 23 | elif [ "$BUILD_TYPE" == "bindings" ] && [ "$BINDING" == "jni" ] && [ -z "$BINDING_OPTS" ]; then 24 | ( cd bindings/jni && TERM=dumb ./gradlew clean bintrayUpload -PisRelease -PbuildPrefix=/tmp/jni_build ) 25 | elif [ "$BUILD_TYPE" == "bindings" ] && [ "$BINDING" == "jni" ] && [ "$BINDING_OPTS" == "android" ]; then 26 | export DAFKA_DEPLOYMENT=bindings/jni/dafka-jni/android/dafka-android-*.jar 27 | else 28 | export DAFKA_DEPLOYMENT="" 29 | fi 30 | -------------------------------------------------------------------------------- /ci_deploy_obs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################################################ 4 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 5 | # READ THE ZPROJECT/README.MD FOR INFORMATION ABOUT MAKING PERMANENT CHANGES. # 6 | ################################################################################ 7 | 8 | # do NOT set -x or it will log the secret tokens! 9 | set -e 10 | 11 | if [ "$BUILD_TYPE" == "default" ] && [ -n "${OBS_STABLE_TOKEN}" -o -n "${OBS_DRAFT_TOKEN}" ]; then 12 | # Trigger source run on new tag on OBS. The latest tag will be fetched. 13 | if [ -n "${OBS_STABLE_TOKEN}" ]; then 14 | curl -H "Authorization: Token ${OBS_STABLE_TOKEN}" -X POST https://api.opensuse.org/trigger/runservice 15 | fi 16 | if [ -n "${OBS_DRAFT_TOKEN}" ]; then 17 | curl -H "Authorization: Token ${OBS_DRAFT_TOKEN}" -X POST https://api.opensuse.org/trigger/runservice 18 | fi 19 | fi 20 | -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | # Ignore the generated manpages and intermediate files 7 | *.1 8 | *.3 9 | *.7 10 | *.xml 11 | *.xml7 12 | 13 | # Ignore the source doc texts generated from program sources 14 | dafka_consumer_msg.txt 15 | dafka_consumer_msg.doc 16 | dafka_producer_msg.txt 17 | dafka_producer_msg.doc 18 | dafka_producer.txt 19 | dafka_producer.doc 20 | dafka_consumer.txt 21 | dafka_consumer.doc 22 | dafka_proto.txt 23 | dafka_proto.doc 24 | dafka_beacon.txt 25 | dafka_beacon.doc 26 | dafka_tower.txt 27 | dafka_tower.doc 28 | dafka_store.txt 29 | dafka_store.doc 30 | dafka_console_producer.txt 31 | dafka_console_producer.doc 32 | dafka_console_consumer.txt 33 | dafka_console_consumer.doc 34 | dafka_stored.txt 35 | dafka_stored.doc 36 | dafka_towerd.txt 37 | dafka_towerd.doc 38 | 39 | # Make sure to track the manually maintained project description 40 | !*.adoc 41 | 42 | ################################################################################ 43 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 44 | # Read the zproject/README.md for information about making permanent changes. # 45 | ################################################################################ 46 | -------------------------------------------------------------------------------- /doc/Makefile.am: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | # Default target 6 | all-local: doc 7 | 8 | # Public programs ("main" tags in project.xml), auto-regenerated: 9 | MAN1 = dafka_console_producer.1 dafka_console_consumer.1 dafka_stored.1 dafka_towerd.1 10 | # Public classes ("class" tags in project.xml), auto-regenerated: 11 | MAN3 = dafka_consumer_msg.3 dafka_producer_msg.3 dafka_producer.3 dafka_consumer.3 dafka_proto.3 dafka_beacon.3 dafka_tower.3 dafka_store.3 12 | # Project overview, written by a human after initial skeleton: 13 | # NOTE: stub doc/dafka.adoc is generated by GSL from project.xml 14 | # and then comitted to SCM and maintained manually to describe the 15 | # project (section 7 = Overview, conventions, and miscellaneous). 16 | MAN7 = dafka.7 17 | MAN_DOC = $(MAN1) $(MAN3) $(MAN7) 18 | 19 | # Assumption: the single .7 page only covers the project and is maintained 20 | # manually. The SCM-tracked text source file name uses an .adoc extension 21 | # so as not to conflict with generated .txt files (in cases when a "class" 22 | # or a "main" name is same as overall project name). 23 | MAN_TXT = $(MAN7:%.7=%.adoc) 24 | 25 | EXTRA_DIST = asciidoc.conf mkman 26 | 27 | if INSTALL_MAN 28 | dist_man_MANS = $(MAN_DOC) 29 | endif 30 | 31 | if BUILD_DOC 32 | MAN_TXT += $(MAN1:%.1=%.txt) 33 | MAN_TXT += $(MAN3:%.3=%.txt) 34 | 35 | DISTCLEANFILES = $(MAN_DOC) 36 | 37 | dist-hook : $(MAN_DOC) 38 | 39 | SUFFIXES=.txt .adoc .xml .xml7 .1 .3 .7 40 | 41 | .txt.xml: 42 | asciidoc -d manpage -b docbook -f $(srcdir)/asciidoc.conf \ 43 | -adafka_version=@PACKAGE_VERSION@ -o$@ $< 44 | .xml.1: 45 | xmlto -o $(@D) man $< 46 | .xml.3: 47 | xmlto -o $(@D) man $< 48 | 49 | # Special handling for project overview whose basename may collide 50 | # with a main or class name 51 | .adoc.xml7: 52 | asciidoc -d manpage -b docbook -f $(srcdir)/asciidoc.conf \ 53 | -adafka_version=@PACKAGE_VERSION@ -o$@ $< 54 | .xml7.7: 55 | xmlto -o $(@D) man $< 56 | 57 | # List of *.txt and *.doc files generated during build from comments 58 | # in project program source files and further processed into manpages. 59 | GENERATED_DOCS = 60 | 61 | # No-op, docs and texts are generated by mkman in one shot - just 62 | # make a dependency that can not parallelize and break stuff. 63 | # Also, to be validly processed, the dependency must have SOME payload 64 | .txt.doc: 65 | @true 66 | 67 | GENERATED_DOCS += dafka_consumer_msg.txt dafka_consumer_msg.doc 68 | dafka_consumer_msg.txt: $(top_srcdir)/src/dafka_consumer_msg.c 69 | "$(srcdir)/mkman" "dafka_consumer_msg" "$(builddir)/dafka_consumer_msg.txt" "$(srcdir)/.." 70 | 71 | GENERATED_DOCS += dafka_producer_msg.txt dafka_producer_msg.doc 72 | dafka_producer_msg.txt: $(top_srcdir)/src/dafka_producer_msg.c 73 | "$(srcdir)/mkman" "dafka_producer_msg" "$(builddir)/dafka_producer_msg.txt" "$(srcdir)/.." 74 | 75 | GENERATED_DOCS += dafka_producer.txt dafka_producer.doc 76 | dafka_producer.txt: $(top_srcdir)/src/dafka_producer.c 77 | "$(srcdir)/mkman" "dafka_producer" "$(builddir)/dafka_producer.txt" "$(srcdir)/.." 78 | 79 | GENERATED_DOCS += dafka_consumer.txt dafka_consumer.doc 80 | dafka_consumer.txt: $(top_srcdir)/src/dafka_consumer.c 81 | "$(srcdir)/mkman" "dafka_consumer" "$(builddir)/dafka_consumer.txt" "$(srcdir)/.." 82 | 83 | GENERATED_DOCS += dafka_proto.txt dafka_proto.doc 84 | dafka_proto.txt: $(top_srcdir)/src/dafka_proto.c 85 | "$(srcdir)/mkman" "dafka_proto" "$(builddir)/dafka_proto.txt" "$(srcdir)/.." 86 | 87 | GENERATED_DOCS += dafka_beacon.txt dafka_beacon.doc 88 | dafka_beacon.txt: $(top_srcdir)/src/dafka_beacon.c 89 | "$(srcdir)/mkman" "dafka_beacon" "$(builddir)/dafka_beacon.txt" "$(srcdir)/.." 90 | 91 | GENERATED_DOCS += dafka_tower.txt dafka_tower.doc 92 | dafka_tower.txt: $(top_srcdir)/src/dafka_tower.c 93 | "$(srcdir)/mkman" "dafka_tower" "$(builddir)/dafka_tower.txt" "$(srcdir)/.." 94 | 95 | GENERATED_DOCS += dafka_store.txt dafka_store.doc 96 | dafka_store.txt: $(top_srcdir)/src/dafka_store.c 97 | "$(srcdir)/mkman" "dafka_store" "$(builddir)/dafka_store.txt" "$(srcdir)/.." 98 | 99 | ### Note: for mains, we keep the source name rather than flattened name:c 100 | ### so that the manpages for binary programs match their name, at expense 101 | ### of perhaps being built in a subdirectory under doc/. 102 | GENERATED_DOCS += dafka_console_producer.txt dafka_console_producer.doc 103 | dafka_console_producer.txt: $(top_srcdir)/src/dafka_console_producer.c 104 | mkdir -p "$(builddir)/$(@D)" 105 | "$(srcdir)/mkman" "dafka_console_producer" "$(builddir)/dafka_console_producer.txt" "$(srcdir)/.." 106 | 107 | GENERATED_DOCS += dafka_console_consumer.txt dafka_console_consumer.doc 108 | dafka_console_consumer.txt: $(top_srcdir)/src/dafka_console_consumer.c 109 | mkdir -p "$(builddir)/$(@D)" 110 | "$(srcdir)/mkman" "dafka_console_consumer" "$(builddir)/dafka_console_consumer.txt" "$(srcdir)/.." 111 | 112 | GENERATED_DOCS += dafka_stored.txt dafka_stored.doc 113 | dafka_stored.txt: $(top_srcdir)/src/dafka_stored.c 114 | mkdir -p "$(builddir)/$(@D)" 115 | "$(srcdir)/mkman" "dafka_stored" "$(builddir)/dafka_stored.txt" "$(srcdir)/.." 116 | 117 | GENERATED_DOCS += dafka_towerd.txt dafka_towerd.doc 118 | dafka_towerd.txt: $(top_srcdir)/src/dafka_towerd.c 119 | mkdir -p "$(builddir)/$(@D)" 120 | "$(srcdir)/mkman" "dafka_towerd" "$(builddir)/dafka_towerd.txt" "$(srcdir)/.." 121 | 122 | 123 | clean-local: 124 | rm -f *.1 *.3 *.7 $(GENERATED_DOCS) 125 | 126 | doc: $(GENERATED_DOCS) 127 | 128 | else 129 | doc: 130 | @echo "SKIPPING documentation generation and formatting (BUILD_DOC was not required and/or tools are missing)" >&2 131 | endif 132 | 133 | EXTRA_DIST += $(MAN_TXT) 134 | ################################################################################ 135 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 136 | # Read the zproject/README.md for information about making permanent changes. # 137 | ################################################################################ 138 | -------------------------------------------------------------------------------- /doc/asciidoc.conf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | [paradef-default] 6 | literal-style=template="literalparagraph" 7 | 8 | [macros] 9 | (?su)[\\]?(?Plinkdafka):(?P\S*?)\[(?P.*?)\]= 10 | 11 | ifdef::backend-docbook[] 12 | [linkdafka-inlinemacro] 13 | {0%{target}} 14 | {0#} 15 | {0#{target}{0}} 16 | {0#} 17 | endif::backend-docbook[] 18 | 19 | ifdef::backend-xhtml11[] 20 | [linkdafka-inlinemacro] 21 | {target}{0?({0})} 22 | endif::backend-xhtml11[] 23 | 24 | ifdef::doctype-manpage[] 25 | ifdef::backend-docbook[] 26 | [header] 27 | template::[header-declarations] 28 | 29 | 30 | {mantitle} 31 | {manvolnum} 32 | Dafka 33 | {dafka_version} 34 | Dafka Manual 35 | 36 | 37 | {manname} 38 | {manpurpose} 39 | 40 | [footer] 41 | 42 | AUTHORS 43 | The dafka manual was written by the authors in the AUTHORS file. 44 | 45 | 46 | RESOURCES 47 | Main web site: 48 | Report bugs to the email <zeromq-dev@lists.zeromq.org> 49 | 50 | 51 | COPYRIGHT 52 | 53 | Copyright (c) the Contributors as noted in the AUTHORS file. This 54 | file is part of DAFKA, a decentralized distributed streaming 55 | platform: http://zeromq.org. 56 | 57 | This Source Code Form is subject to the terms of the Mozilla Public 58 | License, v. 2.0. If a copy of the MPL was not distributed with this 59 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 60 | LICENSE included with the dafka distribution. 61 | 62 | 63 | endif::backend-docbook[] 64 | endif::doctype-manpage[] 65 | -------------------------------------------------------------------------------- /doc/dafka.adoc: -------------------------------------------------------------------------------- 1 | Name(7) 2 | ======= 3 | 4 | 5 | NAME 6 | ---- 7 | dafka - Overview of ZeroMQ messaging middleware 8 | 9 | 10 | SYNOPSIS 11 | -------- 12 | 13 | Project dafka aims to ... (short marketing pitch) 14 | 15 | It delivers several programs with their respective man pages: 16 | dafka_console_producer.1 dafka_console_consumer.1 dafka_stored.1 dafka_towerd.1 17 | and public classes in a shared library: 18 | dafka_consumer_msg.3 dafka_producer_msg.3 dafka_producer.3 dafka_consumer.3 dafka_proto.3 dafka_beacon.3 dafka_tower.3 dafka_store.3 19 | 20 | Generally you can compile and link against it like this: 21 | ---- 22 | #include 23 | 24 | cc ['flags'] 'files' -ldafka ['libraries'] 25 | ---- 26 | 27 | 28 | DESCRIPTION 29 | ----------- 30 | 31 | This is a skeleton document created by zproject, which will not be 32 | regenerated automatically (by Makefiles nor project.xml re-parsing). 33 | You should add hand-written text here to detail whatever applies to 34 | Linux standard manpage Section 7 (note that other OSes may follow 35 | a different standard with similar concepts, and extend the recipes 36 | to package this document into a different section number): 37 | 38 | ---- 39 | 7 Overview, conventions, and miscellaneous : 40 | Overviews or descriptions of various topics, conventions 41 | and protocols, character set standards, the standard 42 | filesystem layout, and miscellaneous other things. 43 | ---- 44 | 45 | Classes 46 | ~~~~~~~ 47 | 48 | Something for developers to consider. Note there are separate man 49 | pages generated for public classes during build with contents taken 50 | from source code comments. 51 | 52 | -------------------------------------------------------------------------------- /doc/mkman: -------------------------------------------------------------------------------- 1 | #! /usr/bin/perl 2 | # 3 | # mkman - Generates man pages from C source and header files. 4 | # 5 | # Syntax: './mkman srcname [outp rootsrcdir]' 6 | # Must be executed in doc/ subdirectory under build workspace root. 7 | # srcname - basename of .c or .cc source file located directly in the 8 | # $rootsrcdir/src/ directory. For classes (MAN3) also implies 9 | # a header (.h) file located directly in $rootsrcdir/include/. 10 | # outp - (optional) filename without extension (will be chopped if 11 | # provided) of the resulting TXT and DOC files; may be a 12 | # slash-separated path e.g. for out-of-tree builds (distcheck). 13 | # Defaults to: "$srcname" 14 | # rootsrcdir - (optional) location of project source root (may differ 15 | # from build root) where to look for input source-code files. 16 | # Defaults to: ".." (good for in-tree builds) 17 | # 18 | # Copyright (c) 1996-2016 iMatix Corporation 19 | # Copyright (c) 2016-2017 zproject community 20 | # 21 | # This is free software; you can redistribute it and/or modify it under the 22 | # terms of the GNU General Public License as published by the Free Software 23 | # Foundation; either version 3 of the License, or (at your option) any later 24 | # version. 25 | # 26 | # This software is distributed in the hope that it will be useful, but 27 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABIL- 28 | # ITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 29 | # License for more details. 30 | # 31 | # You should have received a copy of the GNU General Public License along 32 | # with this program. If not, see . 33 | # 34 | use File::Basename; 35 | use Cwd; 36 | 37 | sub pull { 38 | local ($_) = @_; 39 | if (/^(.*)(@[a-zA-Z0-9]+)(,(\w*)\s*)?/) { 40 | $file = $1; 41 | $tag = $2; 42 | $opts = $4; 43 | $text = ""; 44 | $ext = (fileparse("$file", qr/[^.]*/))[2]; 45 | die "Can't read '$file': $!" 46 | unless open (SOURCE, $file); 47 | 48 | while () { 49 | if (/$tag/) { 50 | while () { 51 | last if /\@discuss/ || /\@end/ || /\@ignore/ || /\@header/; 52 | $_ = " $_" if ($opts eq "code"); 53 | s/^ // if ($opts eq "left"); 54 | $_ = " $_" if ($opts eq "test"); 55 | s/^ / / if ($opts eq "test"); 56 | $text .= $_; 57 | } 58 | } 59 | } 60 | close (SOURCE); 61 | # Add code fences for markdown highlighting 62 | $text = "```$ext\n$text```\n" if (length $text) and ($opts eq "code" or $opts eq "test"); 63 | 64 | $text = "Please add '$tag' section in '$file'.\n" unless $text; 65 | return $text; 66 | } 67 | else { 68 | print "E: bad pull request: $_\n"; 69 | } 70 | } 71 | 72 | sub generate_manpage { 73 | local ($name, $outp, $rootsrcdir) = @_; 74 | $name = $1 if $name =~ /(\w+)\.\w+/; # Chop off extensions 75 | $outp = $1 if $outp =~ /^(.+)\.[^\.\/]+$/; 76 | $outp_basename = basename ($outp); 77 | $outp_basename = $2 if $outp =~ /^(.*\/)?(\w+)$/; 78 | 79 | if ($ENV{"V"} == "1") { 80 | printf "D: generate_manpage() got name='$name' outp='$outp' outp_basename='$outp_basename' rootsrcdir='$rootsrcdir'\n"; 81 | } 82 | 83 | # Check if we're making the man page for a main program, or a class 84 | $cat = 0; # Unknown category 85 | $cat_text = ""; 86 | die "Can't open '" . cwd() . "/Makefile'" 87 | unless open (MAKEFILE, "Makefile"); 88 | while () { 89 | if (/MAN1.*$outp_basename\.1/) { 90 | $source = "$rootsrcdir/src/$name.c"; 91 | $header = "$rootsrcdir/src/$name.c"; 92 | $cat = 1; 93 | $cat_text = "Program for "; 94 | last; 95 | } 96 | elsif (/MAN3.*$outp_basename\.3/) { 97 | $source = "$rootsrcdir/src/$name.c"; 98 | $header = "$rootsrcdir/include/$name.h"; 99 | $cat = 3; 100 | $cat_text = "Class for "; 101 | last; 102 | } 103 | } 104 | close MAKEFILE; 105 | 106 | die "The Makefile defined no manpage category for $outp_basename.1 or $name.3" 107 | unless ($source ne ""); 108 | 109 | # Look for class title in 2nd line of source 110 | # If there's no class file, leave hand-written man page alone 111 | if (! open (SOURCE, $source)) { 112 | # Support mixed-source projects (named C in config, 113 | # but having both .c and .cc files in reality) 114 | printf "Can't open C '$source', retry for C++\n"; 115 | if ($header eq $source) { 116 | $header .= "c"; 117 | } 118 | $source .= "c"; # Retry for a *.cc filename 119 | } 120 | die "Can't open '$source'" 121 | unless open (SOURCE, $source); 122 | # NOTE: This duplication of lines is needed for proper work: 123 | $_ = ; 124 | $_ = ; 125 | $title = "no title found"; 126 | $title = $1 if (/ \w+ - (.*)/); 127 | close (SOURCE); 128 | 129 | printf " MKMAN[txt]\t$source "; 130 | if ($source ne $header) { 131 | printf "+ $header "; 132 | } 133 | printf "\t=> $outp.txt\n"; 134 | # Open output file 135 | die "Can't create '$outp.txt': $!" 136 | unless open (OUTPUT, ">$outp.txt"); 137 | 138 | $underline = "=" x (length ($name) + 3); 139 | 140 | $template = <<"END"; 141 | $name($cat) 142 | $underline 143 | 144 | NAME 145 | ---- 146 | $outp_basename - $cat_text$title 147 | 148 | SYNOPSIS 149 | -------- 150 | ---- 151 | pull $header\@interface 152 | pull $source\@interface,left 153 | ---- 154 | 155 | DESCRIPTION 156 | ----------- 157 | 158 | pull $source\@header,left 159 | 160 | pull $source\@discuss,left 161 | 162 | EXAMPLE 163 | ------- 164 | .From $name\_test method 165 | ---- 166 | pull $source\@selftest,left 167 | ---- 168 | END 169 | 170 | # Now process template 171 | for (split /^/, $template) { 172 | if (/^pull (.*)$/) { 173 | print OUTPUT pull ($1); 174 | } 175 | else { 176 | print OUTPUT $_; 177 | } 178 | } 179 | close OUTPUT; 180 | 181 | # Generate a simple text documentation for README.txt 182 | printf " MKMAN[doc]\t$source "; 183 | if ($source ne $header) { 184 | printf "+ $header "; 185 | } 186 | printf "\t=> $outp.doc\n"; 187 | 188 | die "Can't create '$outp.doc': $!" 189 | unless open (OUTPUT, ">$outp.doc"); 190 | 191 | print OUTPUT "#### $outp_basename - $title\n\n"; 192 | print OUTPUT pull ("$source\@header,left"); 193 | print OUTPUT "\n"; 194 | print OUTPUT pull ("$source\@discuss,left"); 195 | print OUTPUT "\nThis is the class interface:\n\n"; 196 | print OUTPUT pull ("$header\@interface,code"); 197 | print OUTPUT pull ("$source\@interface,left"); 198 | print OUTPUT "\nThis is the class self test code:\n\n"; 199 | print OUTPUT pull ("$source\@selftest,test"); 200 | print OUTPUT "\n"; 201 | close OUTPUT; 202 | } 203 | 204 | $name = shift (@ARGV); 205 | $outp = shift (@ARGV); 206 | if (!$outp) { 207 | $outp=$name; 208 | } 209 | $rootsrcdir = shift (@ARGV); 210 | if (!$rootsrcdir) { 211 | $rootsrcdir=".."; 212 | } 213 | 214 | if ($ENV{"V"} == "1") { 215 | printf "D: got name='$name' outp='$outp' rootsrcdir='$rootsrcdir' from the start\n"; 216 | } 217 | generate_manpage ($name, $outp, $rootsrcdir); 218 | -------------------------------------------------------------------------------- /features/dafka_consumer_protocol.feature: -------------------------------------------------------------------------------- 1 | Feature: Dafka consumer protocol 2 | 3 | Scenario: STORE-HELLO -> CONSUMER-HELLO without subscription 4 | Given a dafka consumer with offset reset earliest 5 | And no subscriptions 6 | When a STORE-HELLO command is send by a store 7 | Then the consumer responds with CONSUMER-HELLO containing 0 topics 8 | 9 | Scenario: STORE-HELLO -> CONSUMER-HELLO with subscription 10 | Given a dafka consumer with offset reset earliest 11 | And a subscription to topic hello 12 | When a STORE-HELLO command is send by a store 13 | Then the consumer responds with CONSUMER-HELLO containing 1 topic 14 | 15 | Scenario: Consumer requests partition heads when joining 16 | Given a dafka consumer with offset reset earliest 17 | When the consumer subscribes to topic hello 18 | Then the consumer will send a GET_HEADS message for topic hello 19 | 20 | Scenario: First record for topic with offset reset earliest 21 | Given a dafka consumer with offset reset earliest 22 | And a subscription to topic hello 23 | When a RECORD message with sequence 1 and content 'CONTENT' is send on topic hello 24 | Then the consumer will send a FETCH message for topic hello with sequence 0 25 | When a RECORD message with sequence 0 and content 'CONTENT' is send on topic hello 26 | Then a consumer_msg is send to the user with topic hello and content 'CONTENT' 27 | 28 | Scenario: First record for topic with offset reset latest 29 | Given a dafka consumer with offset reset latest 30 | And a subscription to topic hello 31 | When a RECORD message with sequence 2 and content 'CONTENT' is send on topic hello 32 | Then a consumer_msg is send to the user with topic hello and content 'CONTENT' 33 | -------------------------------------------------------------------------------- /features/dafka_producer_protocol.feature: -------------------------------------------------------------------------------- 1 | Feature: Dafka producer protocol 2 | 3 | Scenario: Consumer requests missed non-acked message from producer 4 | Given a dafka producer for topic hello 5 | And the producer sent a RECORD message with content 'CONTENT' 6 | When the producer receives a FETCH message with sequence 0 7 | Then the producer will send a DIRECT_RECORD message with content 'CONTENT' 8 | 9 | Scenario: Producer sends HEAD messages at interval after first message was produced 10 | Given a dafka producer for topic hello 11 | When the producer sends a RECORD message with content 'CONTENT' 12 | Then the producer will send HEAD messages at regular intervals 13 | -------------------------------------------------------------------------------- /images/README_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_1.png -------------------------------------------------------------------------------- /images/README_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_2.png -------------------------------------------------------------------------------- /images/README_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_3.png -------------------------------------------------------------------------------- /images/README_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_4.png -------------------------------------------------------------------------------- /images/README_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_5.png -------------------------------------------------------------------------------- /images/README_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_6.png -------------------------------------------------------------------------------- /images/README_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_7.png -------------------------------------------------------------------------------- /images/README_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/images/README_8.png -------------------------------------------------------------------------------- /include/Makefile.am: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | include_HEADERS = \ 6 | dafka.h \ 7 | dafka_consumer_msg.h \ 8 | dafka_producer_msg.h \ 9 | dafka_producer.h \ 10 | dafka_consumer.h \ 11 | dafka_proto.h \ 12 | dafka_beacon.h \ 13 | dafka_tower.h \ 14 | dafka_store.h \ 15 | dafka_library.h 16 | 17 | 18 | ################################################################################ 19 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 20 | # Read the zproject/README.md for information about making permanent changes. # 21 | ################################################################################ 22 | -------------------------------------------------------------------------------- /include/dafka.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka - ZeroMQ messaging middleware 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_H_H_INCLUDED 15 | #define DAFKA_H_H_INCLUDED 16 | 17 | // Include the project library file 18 | #include "dafka_library.h" 19 | 20 | // Add your own public definitions here, if you need them 21 | 22 | typedef struct { 23 | const char *topic; 24 | zconfig_t *config; 25 | } dafka_producer_args_t; 26 | 27 | typedef struct { 28 | const char *log_prefix; 29 | zconfig_t *config; 30 | } dafka_beacon_args_t; 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /include/dafka_beacon.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_beacon - 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_BEACON_H_INCLUDED 15 | #define DAFKA_BEACON_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | 22 | // @interface 23 | // Create new dafka_beacon actor instance. 24 | // @TODO: Describe the purpose of this actor! 25 | // 26 | // zactor_t *dafka_beacon = zactor_new (dafka_beacon, NULL); 27 | // 28 | // Destroy dafka_beacon instance. 29 | // 30 | // zactor_destroy (&dafka_beacon); 31 | // 32 | // Enable verbose logging of commands and activity: 33 | // 34 | // zstr_send (dafka_beacon, "VERBOSE"); 35 | // 36 | // Start dafka_beacon actor. 37 | // 38 | // zstr_sendx (dafka_beacon, "START", NULL); 39 | // 40 | // Stop dafka_beacon actor. 41 | // 42 | // zstr_sendx (dafka_beacon, "STOP", NULL); 43 | // 44 | // This is the dafka_beacon constructor as a zactor_fn; 45 | DAFKA_EXPORT void 46 | dafka_beacon_actor (zsock_t *pipe, void *args); 47 | 48 | DAFKA_EXPORT void 49 | dafka_beacon_recv (zactor_t *self, zsock_t *sub, bool verbose, const char *log_prefix); 50 | 51 | // Self test of this actor 52 | DAFKA_EXPORT void 53 | dafka_beacon_test (bool verbose); 54 | // @end 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /include/dafka_consumer.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_consumer - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_CONSUMER_H_INCLUDED 15 | #define DAFKA_CONSUMER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT 22 | // @warning Please edit the model at "api/dafka_consumer.api" to make changes. 23 | // @interface 24 | // This is a stable class, and may not change except for emergencies. It 25 | // is provided in stable builds. 26 | // Creates a new dafka consumer client that runs in its own background thread. 27 | // 28 | // The args parameter consists of configuration and record sink. 29 | // 30 | // If a record sink is provided this socket will be used the send the consumer 31 | // messages to. 32 | // 33 | // The configuration argument takes settings for both the consumer and the 34 | // beacon, see below. 35 | // 36 | // Consumer configuration: 37 | // * consumer/offset/reset = earliest|latest (default: latest) 38 | // * consumer/high_watermark (default: 1.000.000) 39 | // * consumer/verbose = 0|1 (default: 0 -> false) 40 | // 41 | // Beacon configuration: 42 | // * beacon/interval (default: 1000) in ms 43 | // * beacon/verbose = 0|1 (default: 0 -> false) 44 | // * beacon/sub_address (default: tcp://127.0.0.1:5556) 45 | // * beacon/pub_address (default: tcp://127.0.0.1:5557) 46 | DAFKA_EXPORT dafka_consumer_t * 47 | dafka_consumer_new (dafka_consumer_args_t *args); 48 | 49 | // Destroys an instance of dafka consumer client by gracefully stopping its 50 | // background thread. 51 | DAFKA_EXPORT void 52 | dafka_consumer_destroy (dafka_consumer_t **self_p); 53 | 54 | // Subscribe to a given topic. 55 | DAFKA_EXPORT int 56 | dafka_consumer_subscribe (dafka_consumer_t *self, const char *subject); 57 | 58 | // Unsubscribe from a topic currently subscribed to. 59 | DAFKA_EXPORT int 60 | dafka_consumer_unsubscribe (dafka_consumer_t *self, const char *subject); 61 | 62 | // Returns the address of the consumer instance. 63 | DAFKA_EXPORT const char * 64 | dafka_consumer_address (dafka_consumer_t *self); 65 | 66 | // Get the current subscription as list of strings. 67 | DAFKA_EXPORT zlist_t * 68 | dafka_consumer_subscription (dafka_consumer_t *self); 69 | 70 | // Returns the internal record source socket. 71 | DAFKA_EXPORT zsock_t * 72 | dafka_consumer_record_source (dafka_consumer_t *self); 73 | 74 | // Self test of this class. 75 | DAFKA_EXPORT void 76 | dafka_consumer_test (bool verbose); 77 | 78 | // @end 79 | 80 | #ifdef __cplusplus 81 | } 82 | #endif 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /include/dafka_consumer_msg.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_consumer_record - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_CONSUMER_RECORD_H_INCLUDED 15 | #define DAFKA_CONSUMER_RECORD_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT 22 | // @warning Please edit the model at "api/dafka_consumer_msg.api" to make changes. 23 | // @interface 24 | // This is a stable class, and may not change except for emergencies. It 25 | // is provided in stable builds. 26 | // Create a new dafka_consumer_msg. 27 | DAFKA_EXPORT dafka_consumer_msg_t * 28 | dafka_consumer_msg_new (void); 29 | 30 | // Destroy the dafka_consumer_msg. 31 | DAFKA_EXPORT void 32 | dafka_consumer_msg_destroy (dafka_consumer_msg_t **self_p); 33 | 34 | // Return the subject of the message. 35 | DAFKA_EXPORT const char * 36 | dafka_consumer_msg_subject (dafka_consumer_msg_t *self); 37 | 38 | // Return the sender address of the message. 39 | DAFKA_EXPORT const char * 40 | dafka_consumer_msg_address (dafka_consumer_msg_t *self); 41 | 42 | // Return the content of the message. 43 | // Content buffer is belong to the message. 44 | DAFKA_EXPORT const byte * 45 | dafka_consumer_msg_content (dafka_consumer_msg_t *self); 46 | 47 | // Return the size of the content 48 | DAFKA_EXPORT size_t 49 | dafka_consumer_msg_content_size (dafka_consumer_msg_t *self); 50 | 51 | // Return the content, user takes ownership of the frame returned. 52 | // Caller owns return value and must destroy it when done. 53 | DAFKA_EXPORT zframe_t * 54 | dafka_consumer_msg_get_content (dafka_consumer_msg_t *self); 55 | 56 | // Receive a message from a consumer actor. 57 | // Return 0 on success and -1 on error. 58 | DAFKA_EXPORT int 59 | dafka_consumer_msg_recv (dafka_consumer_msg_t *self, dafka_consumer_t *consumer); 60 | 61 | // Receive a message from a custom socket. 62 | // Return 0 on success and -1 on error. 63 | DAFKA_EXPORT int 64 | dafka_consumer_msg_recv_from_socket (dafka_consumer_msg_t *self, zsock_t *socket); 65 | 66 | // Return frame data copied into freshly allocated string 67 | // Caller must free string when finished with it. 68 | // Caller owns return value and must destroy it when done. 69 | DAFKA_EXPORT char * 70 | dafka_consumer_msg_strdup (dafka_consumer_msg_t *self); 71 | 72 | // Return TRUE if content is equal to string, excluding terminator 73 | DAFKA_EXPORT bool 74 | dafka_consumer_msg_streq (dafka_consumer_msg_t *self, const char *string); 75 | 76 | // Self test of this class. 77 | DAFKA_EXPORT void 78 | dafka_consumer_msg_test (bool verbose); 79 | 80 | // @end 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /include/dafka_library.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka - generated layer of public API 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | 12 | ################################################################################ 13 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 14 | # Read the zproject/README.md for information about making permanent changes. # 15 | ################################################################################ 16 | ========================================================================= 17 | */ 18 | 19 | #ifndef DAFKA_LIBRARY_H_INCLUDED 20 | #define DAFKA_LIBRARY_H_INCLUDED 21 | 22 | // Set up environment for the application 23 | 24 | // External dependencies 25 | #include 26 | 27 | // DAFKA version macros for compile-time API detection 28 | #define DAFKA_VERSION_MAJOR 0 29 | #define DAFKA_VERSION_MINOR 1 30 | #define DAFKA_VERSION_PATCH 0 31 | 32 | #define DAFKA_MAKE_VERSION(major, minor, patch) \ 33 | ((major) * 10000 + (minor) * 100 + (patch)) 34 | #define DAFKA_VERSION \ 35 | DAFKA_MAKE_VERSION(DAFKA_VERSION_MAJOR, DAFKA_VERSION_MINOR, DAFKA_VERSION_PATCH) 36 | 37 | #if defined (__WINDOWS__) 38 | # if defined DAFKA_STATIC 39 | # define DAFKA_EXPORT 40 | # elif defined DAFKA_INTERNAL_BUILD 41 | # if defined DLL_EXPORT 42 | # define DAFKA_EXPORT __declspec(dllexport) 43 | # else 44 | # define DAFKA_EXPORT 45 | # endif 46 | # elif defined DAFKA_EXPORTS 47 | # define DAFKA_EXPORT __declspec(dllexport) 48 | # else 49 | # define DAFKA_EXPORT __declspec(dllimport) 50 | # endif 51 | # define DAFKA_PRIVATE 52 | #elif defined (__CYGWIN__) 53 | # define DAFKA_EXPORT 54 | # define DAFKA_PRIVATE 55 | #else 56 | # if (defined __GNUC__ && __GNUC__ >= 4) || defined __INTEL_COMPILER 57 | # define DAFKA_PRIVATE __attribute__ ((visibility ("hidden"))) 58 | # define DAFKA_EXPORT __attribute__ ((visibility ("default"))) 59 | # else 60 | # define DAFKA_PRIVATE 61 | # define DAFKA_EXPORT 62 | # endif 63 | #endif 64 | 65 | // Opaque class structures to allow forward references 66 | // These classes are stable or legacy and built in all releases 67 | typedef struct _dafka_consumer_msg_t dafka_consumer_msg_t; 68 | #define DAFKA_CONSUMER_MSG_T_DEFINED 69 | typedef struct _dafka_producer_msg_t dafka_producer_msg_t; 70 | #define DAFKA_PRODUCER_MSG_T_DEFINED 71 | typedef struct _dafka_producer_t dafka_producer_t; 72 | #define DAFKA_PRODUCER_T_DEFINED 73 | typedef struct _dafka_consumer_t dafka_consumer_t; 74 | #define DAFKA_CONSUMER_T_DEFINED 75 | typedef struct _dafka_proto_t dafka_proto_t; 76 | #define DAFKA_PROTO_T_DEFINED 77 | typedef struct _dafka_beacon_t dafka_beacon_t; 78 | #define DAFKA_BEACON_T_DEFINED 79 | typedef struct _dafka_tower_t dafka_tower_t; 80 | #define DAFKA_TOWER_T_DEFINED 81 | typedef struct _dafka_store_t dafka_store_t; 82 | #define DAFKA_STORE_T_DEFINED 83 | 84 | typedef struct { 85 | zconfig_t *config; 86 | zsock_t *record_sink; 87 | } dafka_consumer_args_t; 88 | 89 | // Public classes, each with its own header file 90 | #include "dafka_consumer_msg.h" 91 | #include "dafka_producer_msg.h" 92 | #include "dafka_producer.h" 93 | #include "dafka_consumer.h" 94 | #include "dafka_proto.h" 95 | #include "dafka_beacon.h" 96 | #include "dafka_tower.h" 97 | #include "dafka_store.h" 98 | 99 | #ifdef DAFKA_BUILD_DRAFT_API 100 | 101 | #ifdef __cplusplus 102 | extern "C" { 103 | #endif 104 | 105 | // Self test for private classes 106 | DAFKA_EXPORT void 107 | dafka_private_selftest (bool verbose, const char *subtest); 108 | 109 | #ifdef __cplusplus 110 | } 111 | #endif 112 | #endif // DAFKA_BUILD_DRAFT_API 113 | 114 | #endif 115 | /* 116 | ################################################################################ 117 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 118 | # Read the zproject/README.md for information about making permanent changes. # 119 | ################################################################################ 120 | */ 121 | -------------------------------------------------------------------------------- /include/dafka_producer.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_PRODUCER_H_INCLUDED 15 | #define DAFKA_PRODUCER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT 22 | // @warning Please edit the model at "api/dafka_producer.api" to make changes. 23 | // @interface 24 | // This is a stable class, and may not change except for emergencies. It 25 | // is provided in stable builds. 26 | // 27 | DAFKA_EXPORT void 28 | dafka_producer (zsock_t *pipe, void *args); 29 | 30 | // 31 | DAFKA_EXPORT const char * 32 | dafka_producer_address (zactor_t *self); 33 | 34 | // Self test of this class. 35 | DAFKA_EXPORT void 36 | dafka_producer_test (bool verbose); 37 | 38 | // @end 39 | 40 | #ifdef __cplusplus 41 | } 42 | #endif 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /include/dafka_producer_msg.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer_msg - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_PRODUCER_MSG_H_INCLUDED 15 | #define DAFKA_PRODUCER_MSG_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT 22 | // @warning Please edit the model at "api/dafka_producer_msg.api" to make changes. 23 | // @interface 24 | // This is a stable class, and may not change except for emergencies. It 25 | // is provided in stable builds. 26 | // Create a new dafka_producer_msg. 27 | DAFKA_EXPORT dafka_producer_msg_t * 28 | dafka_producer_msg_new (void); 29 | 30 | // Destroy the dafka_producer_msg. 31 | DAFKA_EXPORT void 32 | dafka_producer_msg_destroy (dafka_producer_msg_t **self_p); 33 | 34 | // Return the content of the message. 35 | // Content buffer is belong to the message and user should not try to free it. 36 | DAFKA_EXPORT const byte * 37 | dafka_producer_msg_content (dafka_producer_msg_t *self); 38 | 39 | // Return the size of the content 40 | DAFKA_EXPORT size_t 41 | dafka_producer_msg_content_size (dafka_producer_msg_t *self); 42 | 43 | // Return the content, user takes ownership of the buffer returned. 44 | // Caller owns return value and must destroy it when done. 45 | DAFKA_EXPORT zframe_t * 46 | dafka_producer_msg_get_content (dafka_producer_msg_t *self); 47 | 48 | // Create a new content buffer at the specific size. 49 | // Return 0 on success and -1 if not enough memory. 50 | DAFKA_EXPORT int 51 | dafka_producer_msg_init_content (dafka_producer_msg_t *self, size_t size); 52 | 53 | // Set the content with a frame. Takes ownership on the content. 54 | DAFKA_EXPORT void 55 | dafka_producer_msg_set_content (dafka_producer_msg_t *self, zframe_t **content); 56 | 57 | // Set the content from string. 58 | // Return 0 on success and -1 if not enough memory. 59 | DAFKA_EXPORT int 60 | dafka_producer_msg_set_content_str (dafka_producer_msg_t *self, const char *content); 61 | 62 | // Set the content from string. 63 | // Return 0 on success and -1 if not enough memory. 64 | DAFKA_EXPORT int 65 | dafka_producer_msg_set_content_buffer (dafka_producer_msg_t *self, const byte *content, size_t content_size); 66 | 67 | // Send a message to producer. 68 | // Content will ownership will be moved to a background thread and will be set to NULL. 69 | // Return 0 on success and -1 on error. 70 | DAFKA_EXPORT int 71 | dafka_producer_msg_send (dafka_producer_msg_t *self, zactor_t *producer); 72 | 73 | // Return frame data copied into freshly allocated string 74 | // Caller must free string when finished with it. 75 | // Caller owns return value and must destroy it when done. 76 | DAFKA_EXPORT char * 77 | dafka_producer_msg_strdup (dafka_producer_msg_t *self); 78 | 79 | // Return TRUE if content is equal to string, excluding terminator 80 | DAFKA_EXPORT bool 81 | dafka_producer_msg_streq (dafka_producer_msg_t *self, const char *string); 82 | 83 | // Self test of this class. 84 | DAFKA_EXPORT void 85 | dafka_producer_msg_test (bool verbose); 86 | 87 | // @end 88 | 89 | #ifdef __cplusplus 90 | } 91 | #endif 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /include/dafka_proto.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_proto - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_PROTO_H_INCLUDED 15 | #define DAFKA_PROTO_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @warning THE FOLLOWING @INTERFACE BLOCK IS AUTO-GENERATED BY ZPROJECT 22 | // @warning Please edit the model at "api/dafka_proto.api" to make changes. 23 | // @interface 24 | // This is a stable class, and may not change except for emergencies. It 25 | // is provided in stable builds. 26 | #define DAFKA_PROTO_RECORD 'M' // 27 | #define DAFKA_PROTO_DIRECT_RECORD 'D' // 28 | #define DAFKA_PROTO_FETCH 'F' // 29 | #define DAFKA_PROTO_ACK 'K' // 30 | #define DAFKA_PROTO_HEAD 'H' // 31 | #define DAFKA_PROTO_DIRECT_HEAD 'E' // 32 | #define DAFKA_PROTO_GET_HEADS 'G' // 33 | #define DAFKA_PROTO_CONSUMER_HELLO 'W' // 34 | #define DAFKA_PROTO_STORE_HELLO 'L' // 35 | 36 | // Create a new empty dafka_proto 37 | DAFKA_EXPORT dafka_proto_t * 38 | dafka_proto_new (void); 39 | 40 | // Create a new dafka_proto from zpl/zconfig_t * 41 | DAFKA_EXPORT dafka_proto_t * 42 | dafka_proto_new_zpl (zconfig_t *config); 43 | 44 | // Destroy a dafka_proto instance 45 | DAFKA_EXPORT void 46 | dafka_proto_destroy (dafka_proto_t **self_p); 47 | 48 | // Create a deep copy of a dafka_proto instance 49 | // Caller owns return value and must destroy it when done. 50 | DAFKA_EXPORT dafka_proto_t * 51 | dafka_proto_dup (dafka_proto_t *self); 52 | 53 | // Receive a dafka_proto from the socket. Returns 0 if OK, -1 if 54 | // there was an error. Blocks if there is no message waiting. 55 | DAFKA_EXPORT int 56 | dafka_proto_recv (dafka_proto_t *self, zsock_t *input); 57 | 58 | // Send the dafka_proto to the output socket, does not destroy it 59 | DAFKA_EXPORT int 60 | dafka_proto_send (dafka_proto_t *self, zsock_t *output); 61 | 62 | // Encode the first frame of dafka_proto. Does not destroy it. Returns the frame if OK, else NULL. 63 | DAFKA_EXPORT zframe_t * 64 | dafka_proto_encode (dafka_proto_t *self); 65 | 66 | // Print contents of message to stdout 67 | DAFKA_EXPORT void 68 | dafka_proto_print (dafka_proto_t *self); 69 | 70 | // Export class as zconfig_t*. Caller is responsibe for destroying the instance 71 | // Caller owns return value and must destroy it when done. 72 | DAFKA_EXPORT zconfig_t * 73 | dafka_proto_zpl (dafka_proto_t *self, zconfig_t *parent); 74 | 75 | // Get the message routing id, as a frame 76 | DAFKA_EXPORT zframe_t * 77 | dafka_proto_routing_id (dafka_proto_t *self); 78 | 79 | // Set the message routing id from a frame 80 | DAFKA_EXPORT void 81 | dafka_proto_set_routing_id (dafka_proto_t *self, zframe_t *routing_id); 82 | 83 | // Get the topic of the message for publishing over pub/sub 84 | DAFKA_EXPORT const char * 85 | dafka_proto_topic (dafka_proto_t *self); 86 | 87 | // Set the topic of the message for publishing over pub/sub 88 | DAFKA_EXPORT void 89 | dafka_proto_set_topic (dafka_proto_t *self, const char *topic); 90 | 91 | // Subscribe a socket to a specific message id and a topic. 92 | DAFKA_EXPORT void 93 | dafka_proto_subscribe (zsock_t *sub, char id, const char *topic); 94 | 95 | // Unsubscribe a socket form a specific message id and a topic. 96 | DAFKA_EXPORT void 97 | dafka_proto_unsubscribe (zsock_t *sub, char id, const char *topic); 98 | 99 | // Get whether the last subscription received on XPUB socket was of type subscribe. 100 | DAFKA_EXPORT bool 101 | dafka_proto_is_subscribe (dafka_proto_t *self); 102 | 103 | // Get the dafka_proto message id 104 | DAFKA_EXPORT char 105 | dafka_proto_id (dafka_proto_t *self); 106 | 107 | // Set the dafka_proto message id 108 | DAFKA_EXPORT void 109 | dafka_proto_set_id (dafka_proto_t *self, char id); 110 | 111 | // Get the dafka_proto message id as printable text 112 | DAFKA_EXPORT const char * 113 | dafka_proto_command (dafka_proto_t *self); 114 | 115 | // Get the address field 116 | DAFKA_EXPORT const char * 117 | dafka_proto_address (dafka_proto_t *self); 118 | 119 | // Set the address field 120 | DAFKA_EXPORT void 121 | dafka_proto_set_address (dafka_proto_t *self, const char *address); 122 | 123 | // Get the subject field 124 | DAFKA_EXPORT const char * 125 | dafka_proto_subject (dafka_proto_t *self); 126 | 127 | // Set the subject field 128 | DAFKA_EXPORT void 129 | dafka_proto_set_subject (dafka_proto_t *self, const char *subject); 130 | 131 | // Get the sequence field 132 | DAFKA_EXPORT uint64_t 133 | dafka_proto_sequence (dafka_proto_t *self); 134 | 135 | // Set the sequence field 136 | DAFKA_EXPORT void 137 | dafka_proto_set_sequence (dafka_proto_t *self, uint64_t sequence); 138 | 139 | // Get a copy of the content field 140 | DAFKA_EXPORT zmq_msg_t * 141 | dafka_proto_content (dafka_proto_t *self); 142 | 143 | // Get the content field and transfer ownership to caller 144 | DAFKA_EXPORT void 145 | dafka_proto_get_content (dafka_proto_t *self, zmq_msg_t *content); 146 | 147 | // Set the content field, transferring ownership from caller 148 | DAFKA_EXPORT void 149 | dafka_proto_set_content (dafka_proto_t *self, zmq_msg_t *content); 150 | 151 | // Get the count field 152 | DAFKA_EXPORT uint32_t 153 | dafka_proto_count (dafka_proto_t *self); 154 | 155 | // Set the count field 156 | DAFKA_EXPORT void 157 | dafka_proto_set_count (dafka_proto_t *self, uint32_t count); 158 | 159 | // Get the subjects field 160 | DAFKA_EXPORT zlist_t * 161 | dafka_proto_subjects (dafka_proto_t *self); 162 | 163 | // Get the subjects field and transfer ownership to caller 164 | DAFKA_EXPORT zlist_t * 165 | dafka_proto_get_subjects (dafka_proto_t *self); 166 | 167 | // Set the subjects field 168 | DAFKA_EXPORT void 169 | dafka_proto_set_subjects (dafka_proto_t *self, zlist_t **subjects_p); 170 | 171 | // Self test of this class. 172 | DAFKA_EXPORT void 173 | dafka_proto_test (bool verbose); 174 | 175 | // @end 176 | 177 | #ifdef __cplusplus 178 | } 179 | #endif 180 | 181 | #endif 182 | -------------------------------------------------------------------------------- /include/dafka_store.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_store - 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_STORE_H_INCLUDED 15 | #define DAFKA_STORE_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | 22 | // @interface 23 | // Create new dafka_store actor instance. 24 | // @TODO: Describe the purpose of this actor! 25 | // 26 | // zactor_t *dafka_store = zactor_new (dafka_store, NULL); 27 | // 28 | // Destroy dafka_store instance. 29 | // 30 | // zactor_destroy (&dafka_store); 31 | // 32 | // Start dafka_store actor. 33 | // 34 | // zstr_sendx (dafka_store, "START", NULL); 35 | // 36 | // Stop dafka_store actor. 37 | // 38 | // zstr_sendx (dafka_store, "STOP", NULL); 39 | // 40 | // This is the dafka_store constructor as a zactor_fn; 41 | DAFKA_EXPORT void 42 | dafka_store_actor (zsock_t *pipe, void *args); 43 | 44 | // Self test of this actor 45 | DAFKA_EXPORT void 46 | dafka_store_test (bool verbose); 47 | // @end 48 | 49 | #ifdef __cplusplus 50 | } 51 | #endif 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /include/dafka_tower.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_tower - 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_TOWER_H_INCLUDED 15 | #define DAFKA_TOWER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | 22 | // @interface 23 | // Create new dafka_tower actor instance. 24 | // @TODO: Describe the purpose of this actor! 25 | // 26 | // zactor_t *dafka_tower = zactor_new (dafka_tower, NULL); 27 | // 28 | // Destroy dafka_tower instance. 29 | // 30 | // zactor_destroy (&dafka_tower); 31 | // 32 | // Start dafka_tower actor. 33 | // 34 | // zstr_sendx (dafka_tower, "START", NULL); 35 | // 36 | // Stop dafka_tower actor. 37 | // 38 | // zstr_sendx (dafka_tower, "STOP", NULL); 39 | // 40 | // This is the dafka_tower constructor as a zactor_fn; 41 | DAFKA_EXPORT void 42 | dafka_tower_actor (zsock_t *pipe, void *args); 43 | 44 | // Self test of this actor 45 | DAFKA_EXPORT void 46 | dafka_tower_test (bool verbose); 47 | // @end 48 | 49 | #ifdef __cplusplus 50 | } 51 | #endif 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /license.xml: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) the Contributors as noted in the AUTHORS file. This 3 | file is part of DAFKA, a decentralized distributed streaming 4 | platform: http://zeromq.org. 5 | 6 | This Source Code Form is subject to the terms of the Mozilla Public 7 | License, v. 2.0. If a copy of the MPL was not distributed with this 8 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | -------------------------------------------------------------------------------- /project.xml: -------------------------------------------------------------------------------- 1 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 |
51 |
52 |
53 |
54 | 55 |
56 |
57 | 58 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | # Ignore files generated from .in templates 7 | # + pkgconfig: 8 | libdafka.pc 9 | # + main->extra sources (if any): 10 | 11 | # + config files for mains (if any): 12 | 13 | # location designated for writing self-tests: 14 | selftest-rw/ 15 | 16 | ################################################################################ 17 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 18 | # Read the zproject/README.md for information about making permanent changes. # 19 | ################################################################################ 20 | -------------------------------------------------------------------------------- /src/.valgrind.supp: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Memcheck:Param 4 | socketcall.sendto(msg) 5 | fun:send 6 | ... 7 | } 8 | { 9 | 10 | Memcheck:Param 11 | socketcall.send(msg) 12 | fun:send 13 | ... 14 | } 15 | { 16 | 17 | Memcheck:Free 18 | fun:free 19 | ... 20 | fun:__libc_freeres 21 | ... 22 | } 23 | { 24 | 25 | Memcheck:Leak 26 | match-leak-kinds: reachable 27 | fun:malloc 28 | fun:pool 29 | fun:__static_initialization_and_destruction_0 30 | fun:_GLOBAL__sub_I_eh_alloc.cc 31 | ... 32 | } 33 | { 34 | 35 | Memcheck:Leak 36 | match-leak-kinds: reachable 37 | fun:malloc 38 | ... 39 | fun:_gcry_mpi_init 40 | fun:global_init 41 | ... 42 | } 43 | { 44 | 45 | Memcheck:Leak 46 | match-leak-kinds: reachable 47 | fun:malloc 48 | ... 49 | fun:call_init.part.0 50 | fun:_dl_init 51 | ... 52 | } 53 | -------------------------------------------------------------------------------- /src/Makemodule-local.am: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/src/Makemodule-local.am -------------------------------------------------------------------------------- /src/dafka_classes.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_classes - private header file 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ################################################################################ 12 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 13 | # Read the zproject/README.md for information about making permanent changes. # 14 | ################################################################################ 15 | ========================================================================= 16 | */ 17 | 18 | #ifndef DAFKA_CLASSES_H_INCLUDED 19 | #define DAFKA_CLASSES_H_INCLUDED 20 | 21 | // Platform definitions, must come first 22 | #include "platform.h" 23 | 24 | // Asserts check the invariants of methods. If they're not 25 | // fulfilled the program should fail fast. Therefore enforce them! 26 | #ifdef NDEBUG 27 | #undef NDEBUG 28 | #include 29 | #define NDEBUG 30 | #else 31 | #include 32 | #endif 33 | 34 | // External API 35 | #include "../include/dafka.h" 36 | 37 | // Private external dependencies 38 | 39 | // Opaque class structures to allow forward references 40 | #ifndef DAFKA_UNACKED_LIST_T_DEFINED 41 | typedef struct _dafka_unacked_list_t dafka_unacked_list_t; 42 | #define DAFKA_UNACKED_LIST_T_DEFINED 43 | #endif 44 | #ifndef DAFKA_FETCH_FILTER_T_DEFINED 45 | typedef struct _dafka_fetch_filter_t dafka_fetch_filter_t; 46 | #define DAFKA_FETCH_FILTER_T_DEFINED 47 | #endif 48 | #ifndef DAFKA_MSG_KEY_T_DEFINED 49 | typedef struct _dafka_msg_key_t dafka_msg_key_t; 50 | #define DAFKA_MSG_KEY_T_DEFINED 51 | #endif 52 | #ifndef DAFKA_HEAD_KEY_T_DEFINED 53 | typedef struct _dafka_head_key_t dafka_head_key_t; 54 | #define DAFKA_HEAD_KEY_T_DEFINED 55 | #endif 56 | #ifndef DAFKA_UTIL_T_DEFINED 57 | typedef struct _dafka_util_t dafka_util_t; 58 | #define DAFKA_UTIL_T_DEFINED 59 | #endif 60 | #ifndef DAFKA_CONSUMER_STEP_DEFS_T_DEFINED 61 | typedef struct _dafka_consumer_step_defs_t dafka_consumer_step_defs_t; 62 | #define DAFKA_CONSUMER_STEP_DEFS_T_DEFINED 63 | #endif 64 | #ifndef DAFKA_PRODUCER_STEP_DEFS_T_DEFINED 65 | typedef struct _dafka_producer_step_defs_t dafka_producer_step_defs_t; 66 | #define DAFKA_PRODUCER_STEP_DEFS_T_DEFINED 67 | #endif 68 | #ifndef DAFKA_STORE_READER_T_DEFINED 69 | typedef struct _dafka_store_reader_t dafka_store_reader_t; 70 | #define DAFKA_STORE_READER_T_DEFINED 71 | #endif 72 | #ifndef DAFKA_STORE_WRITER_T_DEFINED 73 | typedef struct _dafka_store_writer_t dafka_store_writer_t; 74 | #define DAFKA_STORE_WRITER_T_DEFINED 75 | #endif 76 | #ifndef DAFKA_TEST_PEER_T_DEFINED 77 | typedef struct _dafka_test_peer_t dafka_test_peer_t; 78 | #define DAFKA_TEST_PEER_T_DEFINED 79 | #endif 80 | 81 | // Extra headers 82 | 83 | // Internal API 84 | 85 | #include "dafka_unacked_list.h" 86 | #include "dafka_fetch_filter.h" 87 | #include "dafka_msg_key.h" 88 | #include "dafka_head_key.h" 89 | #include "dafka_util.h" 90 | #include "dafka_consumer_step_defs.h" 91 | #include "dafka_producer_step_defs.h" 92 | #include "dafka_store_reader.h" 93 | #include "dafka_store_writer.h" 94 | #include "dafka_test_peer.h" 95 | 96 | // *** To avoid double-definitions, only define if building without draft *** 97 | #ifndef DAFKA_BUILD_DRAFT_API 98 | 99 | // *** Draft method, defined for internal use only *** 100 | // Self test of this class. 101 | DAFKA_PRIVATE void 102 | dafka_unacked_list_test (bool verbose); 103 | 104 | // *** Draft method, defined for internal use only *** 105 | // Self test of this class. 106 | DAFKA_PRIVATE void 107 | dafka_util_test (bool verbose); 108 | 109 | // *** Draft method, defined for internal use only *** 110 | // Self test of this class. 111 | DAFKA_PRIVATE void 112 | dafka_consumer_step_defs_test (bool verbose); 113 | 114 | // *** Draft method, defined for internal use only *** 115 | // Self test of this class. 116 | DAFKA_PRIVATE void 117 | dafka_producer_step_defs_test (bool verbose); 118 | 119 | // Self test for private classes 120 | DAFKA_PRIVATE void 121 | dafka_private_selftest (bool verbose, const char *subtest); 122 | 123 | #endif // DAFKA_BUILD_DRAFT_API 124 | 125 | #endif 126 | -------------------------------------------------------------------------------- /src/dafka_console_consumer.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_console_consumer - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_console_consumer - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | zsys_set_linger ((size_t ) -1); 26 | 27 | zargs_t *args = zargs_new (argc, argv); 28 | 29 | if (zargs_hasx (args, "--help", "-h", NULL) || zargs_arguments (args) != 1) { 30 | puts ("Usage: dafka_console_consumer topic [--verbose] [--from-beginning] [-c config] [--pub tower-pub-address] [--sub tower-sub-address]"); 31 | return 0; 32 | } 33 | 34 | zconfig_t *config; 35 | 36 | if (zargs_has (args, "-c")) 37 | config = zconfig_load (zargs_get (args, "-c")); 38 | else 39 | config = zconfig_new ("root", NULL); 40 | 41 | if (zargs_has (args, "--verbose")) { 42 | zconfig_put (config, "beacon/verbose", "1"); 43 | zconfig_put (config, "consumer/verbose", "1"); 44 | } 45 | 46 | if (zargs_has (args, "--from-beginning")) 47 | zconfig_put (config, "consumer/offset/reset", "earliest"); 48 | 49 | if (zargs_has (args, "--pub")) 50 | zconfig_put (config, "beacon/pub_address", zargs_get (args, "--pub")); 51 | 52 | if (zargs_has (args, "--sub")) 53 | zconfig_put (config, "beacon/sub_address", zargs_get (args, "--sub")); 54 | 55 | const char *topic = zargs_first (args); 56 | 57 | dafka_consumer_args_t consumer_args = { .config = config }; 58 | dafka_consumer_t *consumer = dafka_consumer_new (&consumer_args); 59 | assert (consumer); 60 | 61 | // Give time until connected to pubs and stores 62 | zclock_sleep (1000); 63 | 64 | int rc = dafka_consumer_subscribe (consumer, topic); 65 | assert (rc == 0); 66 | 67 | dafka_consumer_msg_t *msg = dafka_consumer_msg_new (); 68 | while (true) { 69 | rc = dafka_consumer_msg_recv (msg, consumer); 70 | if (rc == -1) 71 | break; // Interrupted 72 | 73 | char *content_str = dafka_consumer_msg_strdup (msg); 74 | printf ("%s %s %s\n", dafka_consumer_msg_subject (msg), 75 | dafka_consumer_msg_address (msg), 76 | content_str); 77 | zstr_free (&content_str); 78 | } 79 | 80 | dafka_consumer_msg_destroy (&msg); 81 | dafka_consumer_destroy (&consumer); 82 | zconfig_destroy (&config); 83 | zargs_destroy (&args); 84 | 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /src/dafka_console_producer.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dakfa_console_producer - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dakfa_console_producer - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | zsys_set_linger ((size_t ) -1); 26 | 27 | zargs_t *args = zargs_new (argc, argv); 28 | 29 | if (zargs_hasx (args, "--help", "-h", NULL) || zargs_arguments (args) != 1) { 30 | puts ("Usage: dafka_console_producer [--verbose] [-c config] [--pub tower-pub-address] [--sub tower-sub-address] topic"); 31 | return 0; 32 | } 33 | 34 | zconfig_t *config; 35 | 36 | if (zargs_has (args, "-c")) 37 | config = zconfig_load (zargs_get (args, "-c")); 38 | else 39 | config = zconfig_new ("root", NULL); 40 | 41 | if (zargs_has (args, "--verbose")) { 42 | zconfig_put (config, "beacon/verbose", "1"); 43 | zconfig_put (config, "producer/verbose", "1"); 44 | } 45 | 46 | if (zargs_has (args, "--pub")) 47 | zconfig_put (config, "beacon/pub_address", zargs_get (args, "--pub")); 48 | 49 | if (zargs_has (args, "--sub")) 50 | zconfig_put (config, "beacon/sub_address", zargs_get (args, "--sub")); 51 | 52 | const char *topic = zargs_first (args); 53 | 54 | dafka_producer_args_t producer_args = { topic, config}; 55 | zactor_t *producer = zactor_new (dafka_producer, &producer_args); 56 | 57 | char *content = NULL; 58 | size_t size = 0; 59 | 60 | dafka_producer_msg_t *msg = dafka_producer_msg_new (); 61 | 62 | while (true) { 63 | int64_t content_size = getline (&content, &size, stdin); 64 | if (content_size == -1) 65 | break; 66 | 67 | while (content[content_size - 1] == '\n' || content[content_size - 1] == '\r' || content[content_size - 1] == EOF) 68 | content_size--; 69 | 70 | dafka_producer_msg_set_content_buffer (msg, (const byte *) content, (size_t) content_size); 71 | dafka_producer_msg_send (msg, producer); 72 | } 73 | 74 | zstr_free (&content); 75 | dafka_producer_msg_destroy (&msg); 76 | zactor_destroy (&producer); 77 | zargs_destroy (&args); 78 | zconfig_destroy (&config); 79 | 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /src/dafka_consumer_msg.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_consumer_msg - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_consumer_msg - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | // Structure of our class 24 | 25 | struct _dafka_consumer_msg_t { 26 | char subject[256]; 27 | char address[256]; 28 | zframe_t *content; 29 | }; 30 | 31 | 32 | // -------------------------------------------------------------------------- 33 | // Create a new dafka_consumer_msg 34 | 35 | dafka_consumer_msg_t * 36 | dafka_consumer_msg_new (void) 37 | { 38 | dafka_consumer_msg_t *self = (dafka_consumer_msg_t *) zmalloc (sizeof (dafka_consumer_msg_t)); 39 | assert (self); 40 | return self; 41 | } 42 | 43 | 44 | // -------------------------------------------------------------------------- 45 | // Destroy the dafka_consumer_msg 46 | 47 | void 48 | dafka_consumer_msg_destroy (dafka_consumer_msg_t **self_p) 49 | { 50 | assert (self_p); 51 | if (*self_p) { 52 | dafka_consumer_msg_t *self = *self_p; 53 | zframe_destroy (&self->content); 54 | 55 | // Free object itself 56 | free (self); 57 | *self_p = NULL; 58 | } 59 | } 60 | 61 | // Return the subject of the msg. 62 | const char * 63 | dafka_consumer_msg_subject (dafka_consumer_msg_t *self) { 64 | assert (self); 65 | 66 | return self->subject; 67 | } 68 | 69 | // Return the sender address of the msg. 70 | const char * 71 | dafka_consumer_msg_address (dafka_consumer_msg_t *self) { 72 | assert (self); 73 | 74 | return self->address; 75 | } 76 | 77 | // Return the content of the msg. 78 | // Content buffer is belong to the msg. 79 | const byte * 80 | dafka_consumer_msg_content (dafka_consumer_msg_t *self) { 81 | assert (self); 82 | 83 | if (self->content == NULL) 84 | return NULL; 85 | 86 | return zframe_data (self->content); 87 | } 88 | 89 | // Return the size of the content 90 | DAFKA_EXPORT size_t 91 | dafka_consumer_msg_content_size (dafka_consumer_msg_t *self) { 92 | assert (self); 93 | 94 | if (self->content == NULL) 95 | return 0; 96 | 97 | return zframe_size (self->content); 98 | } 99 | 100 | // Return the content, user takes ownership on the frame returned. 101 | // Caller owns return value and must destroy it when done. 102 | zframe_t * 103 | dafka_consumer_msg_get_content (dafka_consumer_msg_t *self) { 104 | assert (self); 105 | 106 | zframe_t *content = self->content; 107 | self->content = NULL; 108 | 109 | return content; 110 | } 111 | 112 | // Receive a msg from a consumer actor. 113 | // Return 0 on success and -1 on error. 114 | DAFKA_EXPORT int 115 | dafka_consumer_msg_recv (dafka_consumer_msg_t *self, dafka_consumer_t *consumer) { 116 | assert (self); 117 | assert (consumer); 118 | zsock_t *record_source = dafka_consumer_record_source (consumer); 119 | 120 | return dafka_consumer_msg_recv_from_socket (self, record_source); 121 | } 122 | 123 | DAFKA_EXPORT int 124 | dafka_consumer_msg_recv_from_socket (dafka_consumer_msg_t *self, zsock_t *socket) { 125 | assert (self); 126 | assert (socket); 127 | 128 | zmq_msg_t msg; 129 | zmq_msg_init (&msg); 130 | 131 | int count = zmq_msg_recv (&msg, zsock_resolve (socket), 0); 132 | if (count < 0) 133 | return -1; 134 | 135 | memcpy (self->subject, zmq_msg_data (&msg), count); 136 | self->subject[count] = '\0'; 137 | 138 | count = zmq_msg_recv (&msg, zsock_resolve (socket), 0); 139 | memcpy (self->address, zmq_msg_data (&msg), count); 140 | self->address[count] = '\0'; 141 | 142 | zframe_destroy (&self->content); 143 | self->content = zframe_recv (socket); 144 | 145 | return 0; 146 | 147 | } 148 | 149 | 150 | // Return frame data copied into freshly allocated string 151 | // Caller must free string when finished with it. 152 | // Caller owns return value and must destroy it when done. 153 | char * 154 | dafka_consumer_msg_strdup (dafka_consumer_msg_t *self) { 155 | assert (self); 156 | 157 | if (self->content == NULL) 158 | return NULL; 159 | 160 | return zframe_strdup (self->content); 161 | } 162 | 163 | // Return TRUE if content is equal to string, excluding terminator 164 | bool 165 | dafka_consumer_msg_streq (dafka_consumer_msg_t *self, const char *string) { 166 | assert (self); 167 | assert (string); 168 | 169 | if (self->content == NULL) 170 | return false; 171 | 172 | return zframe_streq (self->content, string); 173 | } 174 | 175 | 176 | 177 | // -------------------------------------------------------------------------- 178 | // Self test of this class 179 | 180 | // If your selftest reads SCMed fixture data, please keep it in 181 | // src/selftest-ro; if your test creates filesystem objects, please 182 | // do so under src/selftest-rw. 183 | // The following pattern is suggested for C selftest code: 184 | // char *filename = NULL; 185 | // filename = zsys_sprintf ("%s/%s", SELFTEST_DIR_RO, "mytemplate.file"); 186 | // assert (filename); 187 | // ... use the "filename" for I/O ... 188 | // zstr_free (&filename); 189 | // This way the same "filename" variable can be reused for many subtests. 190 | #define SELFTEST_DIR_RO "src/selftest-ro" 191 | #define SELFTEST_DIR_RW "src/selftest-rw" 192 | 193 | void 194 | dafka_consumer_msg_test (bool verbose) 195 | { 196 | printf (" * dafka_consumer_msg: "); 197 | 198 | // @selftest 199 | // Simple create/destroy test 200 | dafka_consumer_msg_t *self = dafka_consumer_msg_new (); 201 | assert (self); 202 | dafka_consumer_msg_destroy (&self); 203 | // @end 204 | printf ("OK\n"); 205 | } 206 | -------------------------------------------------------------------------------- /src/dafka_consumer_step_defs.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_consumer_step_defs - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of DAFKA the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_CONSUMER_STEP_DEFS_H_INCLUDED 15 | #define DAFKA_CONSUMER_STEP_DEFS_H_INCLUDED 16 | 17 | #if defined (HAVE_CUCUMBER) 18 | #include 19 | 20 | typedef struct _dafka_consumer_state dafka_consumer_state_t; 21 | 22 | DAFKA_EXPORT dafka_consumer_state_t * 23 | dafka_consumer_state_new (bool verbose); 24 | 25 | DAFKA_EXPORT void 26 | dafka_consumer_state_destroy (dafka_consumer_state_t **self_p); 27 | 28 | DAFKA_EXPORT void 29 | register_dafka_consumer_step_defs (cucumber_t *cucumber); 30 | 31 | #endif 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/dafka_consumer_step_runner.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_consumer_step_runner - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_consumer_step_runner - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | bool verbose = false; 26 | int argn; 27 | for (argn = 1; argn < argc; argn++) { 28 | if (streq (argv [argn], "--help") 29 | || streq (argv [argn], "-h")) { 30 | puts ("dafka_consumer_step_runner [options] ..."); 31 | puts (" --verbose / -v verbose test output"); 32 | puts (" --help / -h this information"); 33 | return 0; 34 | } 35 | else 36 | if (streq (argv [argn], "--verbose") 37 | || streq (argv [argn], "-v")) 38 | verbose = true; 39 | else { 40 | printf ("Unknown option: %s\n", argv [argn]); 41 | return 1; 42 | } 43 | } 44 | // Insert main code here 45 | if (verbose) 46 | zsys_info ("dafka_consumer_step_runner - "); 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /src/dafka_cucumber_selftest.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_cucumber_selftest.c - run cucumber selftests 3 | 4 | Runs cucumber steps servers and cucumber feature runner in one process 5 | 6 | ------------------------------------------------------------------------- 7 | Copyright (c) the Contributors as noted in the AUTHORS file. This 8 | file is part of DAFKA, a decentralized distributed streaming 9 | platform: http://zeromq.org. 10 | 11 | This Source Code Form is subject to the terms of the Mozilla Public 12 | License, v. 2.0. If a copy of the MPL was not distributed with this 13 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | 15 | ################################################################################ 16 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 17 | # Read the zproject/README.md for information about making permanent changes. # 18 | ################################################################################ 19 | ========================================================================= 20 | */ 21 | 22 | #include "dafka_classes.h" 23 | #if defined (HAVE_CUCUMBER) 24 | #include 25 | 26 | int main (int argc, char *argv []) 27 | { 28 | zargs_t *args = zargs_new (argc, argv); 29 | zsock_t *client = zsock_new_router ("@tcp://127.0.0.1:8888"); 30 | assert (client); 31 | 32 | zlist_t *step_runners = zlist_new (); 33 | CREATE_STEP_RUNNER_ACTOR(dafka_consumer, dafka_consumer_state_new, dafka_consumer_state_destroy) 34 | zlist_append (step_runners, dafka_consumer_steps_runner); 35 | 36 | CREATE_STEP_RUNNER_ACTOR(dafka_producer, dafka_producer_state_new, dafka_producer_state_destroy) 37 | zlist_append (step_runners, dafka_producer_steps_runner); 38 | 39 | zlist_t *step_runner_identities = zlist_new (); 40 | while (zlist_size (step_runner_identities) < zlist_size (step_runners)) { 41 | zframe_t *identity; 42 | char *command; 43 | zsock_recv (client, "fs", &identity, &command); 44 | if (streq (command, "HELLO")) { 45 | zlist_append (step_runner_identities, identity); 46 | } 47 | zstr_free (&command); 48 | } 49 | 50 | const char *filename = zargs_first (args); 51 | cucumber_feature_runner_t *feature_runner = cucumber_feature_runner_new (filename); 52 | bool rc = cucumber_feature_runner_run (feature_runner, client, step_runner_identities); 53 | 54 | zactor_t *step_runner = (zactor_t *) zlist_first (step_runners); 55 | while (step_runner != NULL) { 56 | zstr_send (step_runner, "$TERM"); 57 | zactor_destroy (&step_runner); 58 | step_runner = (zactor_t *) zlist_next (step_runners); 59 | } 60 | 61 | zargs_destroy (&args); 62 | zlist_destroy (&step_runners);; 63 | zframe_t *identity = (zframe_t *) zlist_first (step_runner_identities); 64 | while (identity) { 65 | zframe_destroy (&identity); 66 | identity = (zframe_t *) zlist_next (step_runner_identities); 67 | } 68 | zlist_destroy (&step_runner_identities); 69 | zsock_destroy (&client); 70 | cucumber_feature_runner_destroy (&feature_runner); 71 | return rc ? 0 : 1; 72 | } 73 | /* 74 | ################################################################################ 75 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 76 | # Read the zproject/README.md for information about making permanent changes. # 77 | ################################################################################ 78 | */ 79 | #endif 80 | -------------------------------------------------------------------------------- /src/dafka_fetch_filter.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_fetch_filter - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_fetch_filter - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | // Structure of our class 24 | 25 | typedef struct { 26 | char subject[256]; 27 | char address[256]; 28 | uint64_t sequence_hash; 29 | uint64_t time_hash; 30 | } fetch_request_t; 31 | 32 | struct _dafka_fetch_filter_t { 33 | zsock_t *publisher; 34 | bool verbose; 35 | int filter_size; 36 | fetch_request_t *cache; 37 | dafka_proto_t *msg; 38 | }; 39 | 40 | 41 | // -------------------------------------------------------------------------- 42 | // Create a new dafka_fetch_filter 43 | 44 | dafka_fetch_filter_t * 45 | dafka_fetch_filter_new (zsock_t *publisher, const char *address, bool verbose) 46 | { 47 | dafka_fetch_filter_t *self = (dafka_fetch_filter_t *) zmalloc (sizeof (dafka_fetch_filter_t)); 48 | assert (self); 49 | 50 | self->publisher = publisher; 51 | self->filter_size = 10000; 52 | self->cache = (fetch_request_t *) zmalloc (sizeof (fetch_request_t) * self->filter_size); 53 | self->msg = dafka_proto_new (); 54 | self->verbose = verbose; 55 | 56 | dafka_proto_set_id (self->msg, DAFKA_PROTO_FETCH); 57 | dafka_proto_set_address (self->msg, address); 58 | 59 | 60 | // Initialize class properties here 61 | return self; 62 | } 63 | 64 | static uint64_t 65 | hash_string (const char *s) 66 | { 67 | uint64_t hash = 0; 68 | while (*s != '\0') 69 | hash = 33 * hash ^ *s++; 70 | 71 | return hash; 72 | } 73 | 74 | void 75 | dafka_fetch_filter_send (dafka_fetch_filter_t *self, const char *subject, const char *address, uint64_t sequence) 76 | { 77 | int64_t time = zclock_mono (); 78 | int max_count = 100000; 79 | 80 | uint64_t sequence_hash = sequence / max_count; 81 | uint64_t time_hash = (time / 1000); 82 | int count = (sequence_hash + 1) * max_count - sequence; 83 | 84 | uint64_t hash = hash_string(subject); 85 | hash = 33 * hash ^ hash_string(address); 86 | hash = 33 * hash ^ sequence_hash; 87 | hash = 33 * hash ^ time_hash; 88 | int index = hash % self->filter_size; 89 | 90 | fetch_request_t *request = &self->cache[index]; 91 | 92 | if (sequence_hash == request->sequence_hash && 93 | time_hash == request->time_hash && 94 | streq (subject, request->subject) && 95 | streq (address, request->address)) { 96 | if (self->verbose) 97 | zsys_debug ("Filter: Filtered fetch request %s %s %" PRIu64, subject, address, sequence); 98 | } else { 99 | if (self->verbose) 100 | zsys_debug ("Filter: Sending fetch request %s %s %" PRIu64, subject, address, sequence); 101 | 102 | strncpy (request->subject, subject, 255); 103 | strncpy (request->address, address, 255); 104 | request->sequence_hash = sequence_hash; 105 | request->time_hash = time_hash; 106 | 107 | dafka_proto_set_topic (self->msg, address); 108 | dafka_proto_set_subject (self->msg, subject); 109 | dafka_proto_set_sequence (self->msg, sequence); 110 | dafka_proto_set_count (self->msg, count); 111 | 112 | dafka_proto_send (self->msg, self->publisher); 113 | } 114 | } 115 | 116 | // -------------------------------------------------------------------------- 117 | // Destroy the dafka_fetch_filter 118 | 119 | void 120 | dafka_fetch_filter_destroy (dafka_fetch_filter_t **self_p) 121 | { 122 | assert (self_p); 123 | if (*self_p) { 124 | dafka_fetch_filter_t *self = *self_p; 125 | // Free class properties here 126 | free (self->cache); 127 | self->cache = NULL; 128 | dafka_proto_destroy (&self->msg); 129 | // Free object itself 130 | free (self); 131 | *self_p = NULL; 132 | } 133 | } -------------------------------------------------------------------------------- /src/dafka_fetch_filter.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_fetch_filter - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_FETCH_FILTER_H_INCLUDED 15 | #define DAFKA_FETCH_FILTER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @interface 22 | // Create a new dafka_fetch_filter 23 | DAFKA_PRIVATE dafka_fetch_filter_t * 24 | dafka_fetch_filter_new (zsock_t *publisher, const char *address, bool verbose); 25 | 26 | DAFKA_PRIVATE void dafka_fetch_filter_send (dafka_fetch_filter_t *self, const char *subject, const char *address, uint64_t sequence); 27 | 28 | // Destroy the dafka_fetch_filter 29 | DAFKA_PRIVATE void 30 | dafka_fetch_filter_destroy (dafka_fetch_filter_t **self_p); 31 | 32 | 33 | // @end 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/dafka_head_key.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_head_key - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_head_key - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | #define MAX_HEAD_KEY_SIZE 1 + 256 + 256 24 | 25 | // Structure of our class 26 | 27 | struct _dafka_head_key_t { 28 | byte buffer[MAX_HEAD_KEY_SIZE]; 29 | size_t buffer_size; 30 | const char *subject; 31 | const char *address; 32 | 33 | uint32_t hash; 34 | }; 35 | 36 | 37 | // -------------------------------------------------------------------------- 38 | // Create a new dafka_head_key 39 | 40 | dafka_head_key_t * 41 | dafka_head_key_new (void) 42 | { 43 | dafka_head_key_t *self = (dafka_head_key_t *) zmalloc (sizeof (dafka_head_key_t)); 44 | assert (self); 45 | // Initialize class properties here 46 | return self; 47 | } 48 | 49 | 50 | // -------------------------------------------------------------------------- 51 | // Destroy the dafka_head_key 52 | 53 | void 54 | dafka_head_key_destroy (dafka_head_key_t **self_p) 55 | { 56 | assert (self_p); 57 | if (*self_p) { 58 | dafka_head_key_t *self = *self_p; 59 | // Free class properties here 60 | // Free object itself 61 | free (self); 62 | *self_p = NULL; 63 | } 64 | } 65 | 66 | void 67 | dafka_head_key_set (dafka_head_key_t *self, const char* subject, const char* address) { 68 | assert (self); 69 | 70 | size_t subject_len = strlen (subject); 71 | size_t address_len = strlen (address); 72 | 73 | byte *needle = self->buffer; 74 | *needle = 'H'; 75 | needle++; 76 | 77 | memcpy (needle, subject, subject_len + 1); 78 | self->subject = (const char *) needle; 79 | needle += subject_len + 1; 80 | 81 | memcpy (needle, address, address_len + 1); 82 | self->address = (const char *) needle; 83 | needle += address_len + 1; 84 | 85 | self->buffer_size = needle - self->buffer; 86 | 87 | self->hash = 0; 88 | const byte *pointer = (const byte *) self->buffer; 89 | while (pointer != needle) 90 | self->hash = 33 * self->hash ^ *pointer++; 91 | } 92 | 93 | const char * 94 | dafka_head_key_subject (dafka_head_key_t *self) { 95 | assert (self); 96 | return self->subject; 97 | } 98 | 99 | const char * 100 | dafka_head_key_address (dafka_head_key_t *self) { 101 | assert (self); 102 | return self->address; 103 | } 104 | 105 | dafka_head_key_t * 106 | dafka_head_key_dup (dafka_head_key_t *self) { 107 | assert (self); 108 | 109 | dafka_head_key_t *copy = dafka_head_key_new (); 110 | dafka_head_key_decode (copy, self->buffer, self->buffer_size); 111 | 112 | return copy; 113 | } 114 | 115 | uint32_t 116 | dafka_head_key_hash (dafka_head_key_t *self) { 117 | assert (self); 118 | return self->hash; 119 | } 120 | 121 | // Encode head key as bytes for leveldb 122 | const char * 123 | dafka_head_key_encode (dafka_head_key_t *self, size_t *size_p) { 124 | assert (self); 125 | 126 | *size_p = self->buffer_size; 127 | 128 | return (const char *) self->buffer; 129 | } 130 | 131 | int 132 | dafka_head_key_decode (dafka_head_key_t *self, const byte* buffer, size_t size) { 133 | assert (self); 134 | 135 | if (size < 1 + 2) 136 | return -1; 137 | 138 | if (*buffer != 'H') 139 | return -1; 140 | 141 | memcpy (self->buffer, buffer, size); 142 | byte* needle = self->buffer; 143 | needle++; 144 | 145 | self->subject = (const char *) needle; 146 | needle += strlen (self->subject) + 1; 147 | 148 | self->address = (const char *) needle; 149 | needle += strlen (self->address) + 1; 150 | 151 | self->buffer_size = needle - self->buffer; 152 | assert (self->buffer_size == size); 153 | 154 | self->hash = 0; 155 | return 0; 156 | } 157 | 158 | int 159 | dafka_head_key_cmp (const dafka_head_key_t *self, const dafka_head_key_t *other) { 160 | const size_t min_size = (self->buffer_size < other->buffer_size) ? self->buffer_size : other->buffer_size; 161 | int r = memcmp(self->buffer, other->buffer, min_size); 162 | if (r == 0) { 163 | if (self->buffer_size < other->buffer_size) 164 | r = -1; 165 | else 166 | if (self->buffer_size > other->buffer_size) 167 | r = 1; 168 | } 169 | return r; 170 | } 171 | 172 | int 173 | dafka_head_key_iter (dafka_head_key_t *self, leveldb_iterator_t *iter) { 174 | assert (self); 175 | assert (iter); 176 | 177 | size_t size; 178 | const char* bytes = leveldb_iter_key (iter, &size); 179 | 180 | return dafka_head_key_decode (self, (const byte *) bytes, size); 181 | } 182 | 183 | void 184 | dafka_head_key_iter_seek (dafka_head_key_t *self, leveldb_iterator_t *iter) { 185 | assert (self); 186 | assert (iter); 187 | 188 | leveldb_iter_seek (iter, (const char *) self->buffer, self->buffer_size); 189 | } 190 | 191 | 192 | // Set hashx key functions, comparator, hasher, dup and destroy 193 | void 194 | dafka_head_key_hashx_set (zhashx_t *hashx) { 195 | zhashx_set_key_destructor (hashx, (zhashx_destructor_fn *) dafka_head_key_destroy); 196 | zhashx_set_key_comparator (hashx, (zhashx_comparator_fn *) dafka_head_key_cmp); 197 | zhashx_set_key_hasher (hashx, (zhashx_hash_fn *) dafka_head_key_hash); 198 | zhashx_set_key_duplicator (hashx, (zhashx_duplicator_fn *) dafka_head_key_dup); 199 | } 200 | -------------------------------------------------------------------------------- /src/dafka_head_key.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_head_key - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_HEAD_KEY_H_INCLUDED 15 | #define DAFKA_HEAD_KEY_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @interface 22 | // Create a new dafka_head_key 23 | DAFKA_PRIVATE dafka_head_key_t * 24 | dafka_head_key_new (void); 25 | 26 | // Destroy the dafka_head_key 27 | DAFKA_PRIVATE void 28 | dafka_head_key_destroy (dafka_head_key_t **self_p); 29 | 30 | DAFKA_PRIVATE void 31 | dafka_head_key_set (dafka_head_key_t *self, const char* subject, const char* address); 32 | 33 | DAFKA_PRIVATE const char * 34 | dafka_head_key_subject (dafka_head_key_t *self); 35 | 36 | DAFKA_PRIVATE const char * 37 | dafka_head_key_address (dafka_head_key_t *self); 38 | 39 | DAFKA_PRIVATE dafka_head_key_t * 40 | dafka_head_key_dup (dafka_head_key_t *self); 41 | 42 | DAFKA_PRIVATE uint32_t 43 | dafka_head_key_hash (dafka_head_key_t *self); 44 | 45 | // Encode head key as bytes for leveldb 46 | DAFKA_PRIVATE const char * 47 | dafka_head_key_encode (dafka_head_key_t *self, size_t *size_p); 48 | 49 | DAFKA_PRIVATE int 50 | dafka_head_key_decode (dafka_head_key_t *self, const byte* buffer, size_t size); 51 | 52 | DAFKA_PRIVATE int 53 | dafka_head_key_cmp (const dafka_head_key_t *self, const dafka_head_key_t *other); 54 | 55 | DAFKA_PRIVATE int 56 | dafka_head_key_iter (dafka_head_key_t *self, leveldb_iterator_t *iter); 57 | 58 | DAFKA_PRIVATE void 59 | dafka_head_key_iter_seek (dafka_head_key_t *self, leveldb_iterator_t *iter); 60 | 61 | 62 | // Set hashx key functions, comparator, hasher, dup and destroy 63 | DAFKA_PRIVATE void 64 | dafka_head_key_hashx_set (zhashx_t *hashx); 65 | 66 | // @end 67 | 68 | #ifdef __cplusplus 69 | } 70 | #endif 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/dafka_msg_key.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_msg_key - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_msg_key - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | #define MAX_MSG_KEY_SIZE 1 + 256 + 256 + 8 24 | 25 | // Structure of our class 26 | 27 | struct _dafka_msg_key_t { 28 | byte buffer[MAX_MSG_KEY_SIZE]; 29 | size_t buffer_size; 30 | const char *subject; 31 | const char *address; 32 | uint64_t sequence; 33 | 34 | uint32_t hash; 35 | }; 36 | 37 | 38 | // -------------------------------------------------------------------------- 39 | // Create a new dafka_msg_key 40 | 41 | dafka_msg_key_t * 42 | dafka_msg_key_new (void) 43 | { 44 | dafka_msg_key_t *self = (dafka_msg_key_t *) zmalloc (sizeof (dafka_msg_key_t)); 45 | assert (self); 46 | // Initialize class properties here 47 | return self; 48 | } 49 | 50 | 51 | // -------------------------------------------------------------------------- 52 | // Destroy the dafka_msg_key 53 | 54 | void 55 | dafka_msg_key_destroy (dafka_msg_key_t **self_p) 56 | { 57 | assert (self_p); 58 | if (*self_p) { 59 | dafka_msg_key_t *self = *self_p; 60 | // Free class properties here 61 | // Free object itself 62 | free (self); 63 | *self_p = NULL; 64 | } 65 | } 66 | 67 | void 68 | dafka_msg_key_set (dafka_msg_key_t *self, const char* subject, const char* address, uint64_t sequence) { 69 | assert (self); 70 | 71 | size_t subject_len = strlen (subject); 72 | size_t address_len = strlen (address); 73 | 74 | byte *needle = self->buffer; 75 | *needle = 'M'; 76 | needle++; 77 | 78 | memcpy (needle, subject, subject_len + 1); 79 | self->subject = (const char *) needle; 80 | needle += subject_len + 1; 81 | 82 | memcpy (needle, address, address_len + 1); 83 | self->address = (const char *) needle; 84 | needle += address_len + 1; 85 | 86 | needle += uint64_put_be (needle, sequence); 87 | self->sequence = sequence; 88 | 89 | self->buffer_size = needle - self->buffer; 90 | self->hash = 0; 91 | const byte *pointer = (const byte *) self->buffer; 92 | while (pointer != needle) 93 | self->hash = 33 * self->hash ^ *pointer++; 94 | } 95 | 96 | const char * 97 | dafka_msg_key_subject (dafka_msg_key_t *self) { 98 | assert (self); 99 | return self->subject; 100 | } 101 | 102 | const char * 103 | dafka_msg_key_address (dafka_msg_key_t *self) { 104 | assert (self); 105 | return self->address; 106 | } 107 | 108 | uint64_t 109 | dafka_msg_key_sequence (dafka_msg_key_t *self) { 110 | assert (self); 111 | return self->sequence; 112 | } 113 | 114 | dafka_msg_key_t * 115 | dafka_msg_key_dup (dafka_msg_key_t *self) { 116 | assert (self); 117 | 118 | dafka_msg_key_t *copy = dafka_msg_key_new (); 119 | dafka_msg_key_decode (copy, self->buffer, self->buffer_size); 120 | 121 | return copy; 122 | } 123 | 124 | uint32_t 125 | dafka_msg_key_hash (dafka_msg_key_t *self) { 126 | assert (self); 127 | 128 | if (self->hash == 0) { 129 | 130 | } 131 | 132 | return self->hash; 133 | } 134 | 135 | const char * 136 | dafka_msg_key_encode (dafka_msg_key_t *self, size_t *size_p) { 137 | assert (self); 138 | 139 | *size_p = self->buffer_size; 140 | 141 | return (const char *) self->buffer; 142 | } 143 | 144 | int 145 | dafka_msg_key_decode (dafka_msg_key_t *self, const byte* buffer, size_t size) { 146 | assert (self); 147 | 148 | if (size < 1 + 2 + 8) 149 | return -1; 150 | 151 | if (*buffer != 'M') 152 | return -1; 153 | 154 | memcpy (self->buffer, buffer, size); 155 | byte* needle = self->buffer; 156 | needle++; 157 | 158 | self->subject = (const char *) needle; 159 | needle += strlen (self->subject) + 1; 160 | 161 | self->address = (const char *) needle; 162 | needle += strlen (self->address) + 1; 163 | 164 | needle += uint64_get_be (needle, &self->sequence); 165 | 166 | self->buffer_size = needle - self->buffer; 167 | assert (self->buffer_size == size); 168 | 169 | self->hash = 0; 170 | return 0; 171 | } 172 | 173 | 174 | int 175 | dafka_msg_key_cmp (const dafka_msg_key_t *self, const dafka_msg_key_t *other) { 176 | const size_t min_size = (self->buffer_size < other->buffer_size) ? self->buffer_size : other->buffer_size; 177 | int r = memcmp(self->buffer, other->buffer, min_size); 178 | if (r == 0) { 179 | if (self->buffer_size < other->buffer_size) 180 | r = -1; 181 | else 182 | if (self->buffer_size > other->buffer_size) 183 | r = 1; 184 | } 185 | return r; 186 | } 187 | 188 | int 189 | dafka_msg_key_iter (dafka_msg_key_t *self, leveldb_iterator_t *iter) { 190 | assert (self); 191 | assert (iter); 192 | 193 | size_t size; 194 | const char* bytes = leveldb_iter_key (iter, &size); 195 | 196 | return dafka_msg_key_decode (self, (const byte *) bytes, size); 197 | } 198 | 199 | void 200 | dafka_msg_key_iter_seek (dafka_msg_key_t *self, leveldb_iterator_t *iter) { 201 | assert (self); 202 | assert (iter); 203 | 204 | leveldb_iter_seek (iter, (const char *) self->buffer, self->buffer_size); 205 | } 206 | 207 | // Set hashx key functions, comparator, hasher, dup and destroy 208 | void 209 | dafka_msg_key_hashx_set (zhashx_t *hashx) { 210 | zhashx_set_key_destructor (hashx, (zhashx_destructor_fn *) dafka_msg_key_destroy); 211 | zhashx_set_key_comparator (hashx, (zhashx_comparator_fn *) dafka_msg_key_cmp); 212 | zhashx_set_key_hasher (hashx, (zhashx_hash_fn *) dafka_msg_key_hash); 213 | zhashx_set_key_duplicator (hashx, (zhashx_duplicator_fn *) dafka_msg_key_dup); 214 | } 215 | -------------------------------------------------------------------------------- /src/dafka_msg_key.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_msg_key - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_MSG_KEY_H_INCLUDED 15 | #define DAFKA_MSG_KEY_H_INCLUDED 16 | 17 | #include 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | // @interface 24 | // Create a new dafka_msg_key 25 | DAFKA_PRIVATE dafka_msg_key_t * 26 | dafka_msg_key_new (void); 27 | 28 | // Destroy the dafka_msg_key 29 | DAFKA_PRIVATE void 30 | dafka_msg_key_destroy (dafka_msg_key_t **self_p); 31 | 32 | DAFKA_PRIVATE void 33 | dafka_msg_key_set (dafka_msg_key_t *self, const char* subject, const char* address, uint64_t sequence); 34 | 35 | DAFKA_PRIVATE const char * 36 | dafka_msg_key_subject (dafka_msg_key_t *self); 37 | 38 | DAFKA_PRIVATE const char * 39 | dafka_msg_key_address (dafka_msg_key_t *self); 40 | 41 | DAFKA_PRIVATE uint64_t 42 | dafka_msg_key_sequence (dafka_msg_key_t *self); 43 | 44 | DAFKA_PRIVATE dafka_msg_key_t * 45 | dafka_msg_key_dup (dafka_msg_key_t *self); 46 | 47 | DAFKA_PRIVATE uint32_t 48 | dafka_msg_key_hash (dafka_msg_key_t *self); 49 | 50 | DAFKA_PRIVATE const char * 51 | dafka_msg_key_encode (dafka_msg_key_t *self, size_t *size_p); 52 | 53 | DAFKA_PRIVATE int 54 | dafka_msg_key_decode (dafka_msg_key_t *self, const byte* buffer, size_t size); 55 | 56 | DAFKA_PRIVATE int 57 | dafka_msg_key_cmp (const dafka_msg_key_t *self, const dafka_msg_key_t *other); 58 | 59 | DAFKA_PRIVATE int 60 | dafka_msg_key_iter (dafka_msg_key_t *self, leveldb_iterator_t *iter); 61 | 62 | DAFKA_PRIVATE void 63 | dafka_msg_key_iter_seek (dafka_msg_key_t *self, leveldb_iterator_t *iter); 64 | 65 | // Set hashx key functions, comparator, hasher, dup and destroy 66 | DAFKA_PRIVATE void 67 | dafka_msg_key_hashx_set (zhashx_t *hashx); 68 | 69 | 70 | // @end 71 | 72 | #ifdef __cplusplus 73 | } 74 | #endif 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /src/dafka_perf_consumer.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_perf_consumer - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_perf_consumer - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | zsys_set_pipehwm (1000000); 26 | 27 | zargs_t *args = zargs_new (argc, argv); 28 | 29 | if (zargs_hasx (args, "--help", "-h", NULL) || zargs_arguments (args) != 2) { 30 | puts ("Usage: dafka_perf_consumer [--verbose] [-c config] [--pub tower-pub-address] [--sub tower-sub-address] count size"); 31 | return 0; 32 | } 33 | 34 | zconfig_t *config; 35 | 36 | if (zargs_has (args, "-c")) 37 | config = zconfig_load (zargs_get (args, "-c")); 38 | else 39 | config = zconfig_new ("root", NULL); 40 | 41 | assert (config); 42 | 43 | bool verbose = zargs_has (args, "--verbose"); 44 | 45 | if (verbose) { 46 | zconfig_put (config, "beacon/verbose", "1"); 47 | zconfig_put (config, "consumer/verbose", "1"); 48 | } 49 | 50 | if (zargs_has (args, "--pub")) 51 | zconfig_put (config, "beacon/pub_address", zargs_get (args, "--pub")); 52 | 53 | if (zargs_has (args, "--sub")) 54 | zconfig_put (config, "beacon/sub_address", zargs_get (args, "--sub")); 55 | 56 | int count = atoi (zargs_first (args)); 57 | int size = atoi (zargs_next (args)); 58 | 59 | // Creating the consumer 60 | dafka_consumer_args_t consumer_args = { .config = config }; 61 | dafka_consumer_t *consumer = dafka_consumer_new (&consumer_args); 62 | dafka_consumer_subscribe (consumer, "$STORE_PERF"); 63 | 64 | dafka_consumer_msg_t *msg = dafka_consumer_msg_new (); 65 | dafka_consumer_msg_recv (msg, consumer); 66 | printf ("Received first\n"); 67 | int left = count - 1; 68 | 69 | void *watch = zmq_stopwatch_start (); 70 | 71 | while (left > 0) { 72 | dafka_consumer_msg_recv (msg, consumer); 73 | left--; 74 | } 75 | 76 | unsigned long elapsed = zmq_stopwatch_stop (watch); 77 | if (elapsed == 0) 78 | elapsed = 1; 79 | 80 | double throughput = ((double) count / (double) elapsed * 1000000); 81 | double megabits = ((double) throughput * size * 8) / 1000000; 82 | 83 | printf ("message size: %d [B]\n", size); 84 | printf ("message count: %d\n", count); 85 | printf ("mean throughput: %d [msg/s]\n", (int) throughput); 86 | printf ("mean throughput: %.3f [Mb/s]\n", megabits); 87 | 88 | dafka_consumer_msg_destroy (&msg); 89 | dafka_consumer_destroy (&consumer); 90 | zargs_destroy (&args); 91 | zconfig_destroy (&config); 92 | 93 | return 0; 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/dafka_perf_store.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_perf_store - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_perf_store - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | zsys_set_pipehwm (1000000); 26 | 27 | zargs_t *args = zargs_new (argc, argv); 28 | 29 | if (zargs_hasx (args, "--help", "-h", NULL) || zargs_arguments (args) != 2) { 30 | puts ("Usage: dafka_perf_store [--verbose] [-c config] [--pub tower-pub-address] [--sub tower-sub-address] count size"); 31 | return 0; 32 | } 33 | 34 | zconfig_t *config; 35 | 36 | if (zargs_has (args, "-c")) 37 | config = zconfig_load (zargs_get (args, "-c")); 38 | else 39 | config = zconfig_new ("root", NULL); 40 | 41 | bool verbose = zargs_has (args, "--verbose"); 42 | 43 | if (verbose) { 44 | zconfig_put (config, "beacon/verbose", "1"); 45 | zconfig_put (config, "producer/verbose", "1"); 46 | } 47 | 48 | if (zargs_has (args, "--pub")) 49 | zconfig_put (config, "beacon/pub_address", zargs_get (args, "--pub")); 50 | 51 | if (zargs_has (args, "--sub")) 52 | zconfig_put (config, "beacon/sub_address", zargs_get (args, "--sub")); 53 | 54 | int count = atoi (zargs_first (args)); 55 | int size = atoi (zargs_next (args)); 56 | 57 | // Creating the producer 58 | dafka_producer_args_t producer_args = {"$STORE_PERF", config}; 59 | zactor_t *producer = zactor_new (dafka_producer, &producer_args); 60 | 61 | // Give everyone time to connect 62 | zclock_sleep (1000); 63 | 64 | // Sending all the messages now 65 | dafka_producer_msg_t *msg = dafka_producer_msg_new (); 66 | 67 | void *watch = zmq_stopwatch_start (); 68 | for (int i = 0; i < count; ++i) { 69 | dafka_producer_msg_init_content (msg, (size_t) size); 70 | dafka_producer_msg_send (msg, producer); 71 | } 72 | 73 | printf ("Done publishing\n"); 74 | 75 | // Now waiting for the subscriber confirmation 76 | zactor_destroy (&producer); 77 | 78 | unsigned long elapsed = zmq_stopwatch_stop (watch); 79 | if (elapsed == 0) 80 | elapsed = 1; 81 | 82 | double throughput = ((double) count / (double) elapsed * 1000000); 83 | double megabits = ((double) throughput * size * 8) / 1000000; 84 | 85 | printf ("message size: %d [B]\n", size); 86 | printf ("message count: %d\n", count); 87 | printf ("mean throughput: %d [msg/s]\n", (int) throughput); 88 | printf ("mean throughput: %.3f [Mb/s]\n", megabits); 89 | 90 | dafka_producer_msg_destroy (&msg); 91 | zargs_destroy (&args); 92 | zconfig_destroy (&config); 93 | } 94 | -------------------------------------------------------------------------------- /src/dafka_private_selftest.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_private_selftest.c - run private classes selftests 3 | 4 | Runs all private classes selftests. 5 | 6 | ------------------------------------------------------------------------- 7 | Copyright (c) the Contributors as noted in the AUTHORS file. This 8 | file is part of DAFKA, a decentralized distributed streaming 9 | platform: http://zeromq.org. 10 | 11 | This Source Code Form is subject to the terms of the Mozilla Public 12 | License, v. 2.0. If a copy of the MPL was not distributed with this 13 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | 15 | ################################################################################ 16 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 17 | # Read the zproject/README.md for information about making permanent changes. # 18 | ################################################################################ 19 | ========================================================================= 20 | */ 21 | 22 | #include "dafka_classes.h" 23 | 24 | 25 | // ------------------------------------------------------------------------- 26 | // Run all private classes tests. 27 | // 28 | 29 | void 30 | dafka_private_selftest (bool verbose, const char *subtest) 31 | { 32 | // Tests for stable private classes: 33 | if (streq (subtest, "$ALL") || streq (subtest, "dafka_unacked_list_test")) 34 | dafka_unacked_list_test (verbose); 35 | if (streq (subtest, "$ALL") || streq (subtest, "dafka_util_test")) 36 | dafka_util_test (verbose); 37 | if (streq (subtest, "$ALL") || streq (subtest, "dafka_test_peer_test")) 38 | dafka_test_peer_test (verbose); 39 | } 40 | /* 41 | ################################################################################ 42 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 43 | # Read the zproject/README.md for information about making permanent changes. # 44 | ################################################################################ 45 | */ 46 | -------------------------------------------------------------------------------- /src/dafka_producer_msg.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer_msg - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_producer_msg - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | // Structure of our class 24 | 25 | struct _dafka_producer_msg_t { 26 | zframe_t *content; 27 | }; 28 | 29 | 30 | // -------------------------------------------------------------------------- 31 | // Create a new dafka_producer_msg 32 | 33 | dafka_producer_msg_t * 34 | dafka_producer_msg_new (void) 35 | { 36 | dafka_producer_msg_t *self = (dafka_producer_msg_t *) zmalloc (sizeof (dafka_producer_msg_t)); 37 | assert (self); 38 | return self; 39 | } 40 | 41 | 42 | // -------------------------------------------------------------------------- 43 | // Destroy the dafka_producer_msg 44 | 45 | void 46 | dafka_producer_msg_destroy (dafka_producer_msg_t **self_p) 47 | { 48 | assert (self_p); 49 | if (*self_p) { 50 | dafka_producer_msg_t *self = *self_p; 51 | zframe_destroy (&self->content); 52 | 53 | // Free object itself 54 | free (self); 55 | *self_p = NULL; 56 | } 57 | } 58 | 59 | 60 | // Return the content of the record. 61 | // Content buffer is belong to the record. 62 | const byte * 63 | dafka_producer_msg_content (dafka_producer_msg_t *self) { 64 | assert (self); 65 | 66 | if (self->content == NULL) 67 | return NULL; 68 | 69 | return zframe_data (self->content); 70 | } 71 | 72 | // Return the size of the content 73 | size_t 74 | dafka_producer_msg_content_size (dafka_producer_msg_t *self) { 75 | assert (self); 76 | 77 | if (self->content == NULL) 78 | return 0; 79 | 80 | return zframe_size (self->content); 81 | } 82 | 83 | // Return the content, user takes ownership of the frame returned. 84 | // Caller owns return value and must destroy it when done. 85 | zframe_t * 86 | dafka_producer_msg_get_content (dafka_producer_msg_t *self) { 87 | assert (self); 88 | 89 | zframe_t *content = self->content; 90 | self->content = NULL; 91 | 92 | return content; 93 | } 94 | 95 | // Create a new content buffer at the specific size. 96 | int 97 | dafka_producer_msg_init_content (dafka_producer_msg_t *self, size_t size) { 98 | assert (self); 99 | zframe_destroy (&self->content); 100 | 101 | self->content = zframe_new (NULL, size); 102 | 103 | return self->content ? 0 : 1; 104 | } 105 | 106 | // Set the content with a frame. Takes ownership on the content. 107 | DAFKA_EXPORT void 108 | dafka_producer_msg_set_content (dafka_producer_msg_t *self, zframe_t **content) { 109 | assert (self); 110 | assert (content); 111 | zframe_destroy (&self->content); 112 | 113 | self->content = *content; 114 | *content = NULL; 115 | } 116 | 117 | // Set the content from string. 118 | int 119 | dafka_producer_msg_set_content_str (dafka_producer_msg_t *self, const char *content) { 120 | assert (self); 121 | assert (content); 122 | 123 | zframe_destroy (&self->content); 124 | 125 | size_t size = strlen (content); 126 | self->content = zframe_new (content, size); 127 | 128 | return self->content ? 0 : 1; 129 | } 130 | 131 | // Set the content from string. 132 | // Return 0 on success and -1 if not enough memory. 133 | int 134 | dafka_producer_msg_set_content_buffer (dafka_producer_msg_t *self, const byte *content, size_t content_size) { 135 | assert (self); 136 | assert (content); 137 | 138 | zframe_destroy (&self->content); 139 | self->content = zframe_new (content, content_size); 140 | 141 | return self->content ? 0 : 1; 142 | } 143 | 144 | 145 | // Send a record to producer. 146 | // Content will ownership will be moved to a background thread and will be set to NULL. 147 | // Return 0 on success and -1 on error. 148 | int 149 | dafka_producer_msg_send (dafka_producer_msg_t *self, zactor_t *producer) { 150 | assert (self); 151 | assert (self->content); 152 | 153 | int rc = zstr_sendm (producer, "P"); 154 | if (rc == -1) 155 | return -1; 156 | 157 | zframe_send (&self->content, producer, 0); 158 | 159 | return 0; 160 | } 161 | 162 | // Return frame data copied into freshly allocated string 163 | // Caller must free string when finished with it. 164 | // Caller owns return value and must destroy it when done. 165 | char * 166 | dafka_producer_msg_strdup (dafka_producer_msg_t *self) { 167 | assert (self); 168 | 169 | if (self->content == NULL) 170 | return NULL; 171 | 172 | return zframe_strdup (self->content); 173 | } 174 | 175 | // Return TRUE if content is equal to string, excluding terminator 176 | bool 177 | dafka_producer_msg_streq (dafka_producer_msg_t *self, const char *string) { 178 | assert (self); 179 | assert (string); 180 | 181 | if (self->content == NULL) 182 | return false; 183 | 184 | return zframe_streq (self->content, string); 185 | } 186 | 187 | 188 | // -------------------------------------------------------------------------- 189 | // Self test of this class 190 | 191 | // If your selftest reads SCMed fixture data, please keep it in 192 | // src/selftest-ro; if your test creates filesystem objects, please 193 | // do so under src/selftest-rw. 194 | // The following pattern is suggested for C selftest code: 195 | // char *filename = NULL; 196 | // filename = zsys_sprintf ("%s/%s", SELFTEST_DIR_RO, "mytemplate.file"); 197 | // assert (filename); 198 | // ... use the "filename" for I/O ... 199 | // zstr_free (&filename); 200 | // This way the same "filename" variable can be reused for many subtests. 201 | #define SELFTEST_DIR_RO "src/selftest-ro" 202 | #define SELFTEST_DIR_RW "src/selftest-rw" 203 | 204 | void 205 | dafka_producer_msg_test (bool verbose) 206 | { 207 | printf (" * dafka_producer_msg: "); 208 | 209 | // @selftest 210 | // Simple create/destroy test 211 | dafka_producer_msg_t *self = dafka_producer_msg_new (); 212 | assert (self); 213 | dafka_producer_msg_destroy (&self); 214 | // @end 215 | printf ("OK\n"); 216 | } 217 | -------------------------------------------------------------------------------- /src/dafka_producer_step_defs.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer_step_defs - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | */ 12 | 13 | #include "dafka_classes.h" 14 | #if defined (HAVE_CUCUMBER) 15 | #include 16 | 17 | #define SELFTEST_DIR_RW "src/selftest-rw" 18 | 19 | typedef struct _dafka_producer_state { 20 | zconfig_t *config; 21 | zactor_t *tower; 22 | zactor_t *test_peer; 23 | zactor_t *producer; 24 | char *topic; 25 | } dafka_producer_state_t; 26 | 27 | dafka_producer_state_t * 28 | dafka_producer_state_new (bool verbose) 29 | { 30 | dafka_producer_state_t *self = (dafka_producer_state_t *) zmalloc (sizeof (dafka_producer_state_t)); 31 | if (zsys_file_exists (SELFTEST_DIR_RW "/storedb")) { 32 | zdir_t *store_dir = zdir_new (SELFTEST_DIR_RW "/storedb", NULL); 33 | zdir_remove (store_dir, true); 34 | zdir_destroy (&store_dir); 35 | } 36 | 37 | self->config = zconfig_new ("root", NULL); 38 | zconfig_put (self->config, "test/verbose", verbose ? "1" : "0"); 39 | zconfig_put (self->config, "beacon/interval", "50"); 40 | zconfig_put (self->config, "beacon/verbose", verbose ? "1" : "0"); 41 | zconfig_put (self->config, "beacon/sub_address", "inproc://producer-tower-sub"); 42 | zconfig_put (self->config, "beacon/pub_address", "inproc://producer-tower-pub"); 43 | zconfig_put (self->config, "tower/verbose", verbose ? "1" : "0"); 44 | zconfig_put (self->config, "tower/sub_address", "inproc://producer-tower-sub"); 45 | zconfig_put (self->config, "tower/pub_address", "inproc://producer-tower-pub"); 46 | zconfig_put (self->config, "producer/verbose", verbose ? "1" : "0"); 47 | zconfig_put (self->config, "producer/terminate_unacked", "1"); 48 | zconfig_put (self->config, "producer/head_interval", "100"); 49 | zconfig_put (self->config, "consumer/verbose", verbose ? "1" : "0"); 50 | zconfig_put (self->config, "store/verbose", verbose ? "1" : "0"); 51 | zconfig_put (self->config, "store/db", SELFTEST_DIR_RW "/storedb"); 52 | 53 | zconfig_put (self->config, "consumer/offset/reset", "earliest"); 54 | 55 | self->tower = zactor_new (dafka_tower_actor, self->config); 56 | assert (self->tower); 57 | 58 | self->test_peer = zactor_new (dafka_test_peer, self->config); 59 | assert (self->test_peer); 60 | 61 | return self; 62 | } 63 | 64 | void 65 | dafka_producer_state_destroy (dafka_producer_state_t **self_p) 66 | { 67 | assert (self_p); 68 | if (*self_p) { 69 | dafka_producer_state_t *self = *self_p; 70 | // Free class properties 71 | zactor_destroy (&self->tower); 72 | zactor_destroy (&self->test_peer); 73 | zactor_destroy (&self->producer); 74 | zconfig_destroy (&self->config); 75 | zstr_free (&self->topic); 76 | // Free object itself 77 | free (self); 78 | *self_p = NULL; 79 | } 80 | } 81 | 82 | void 83 | given_a_dafka_producer_for_topic (cucumber_step_def_t *self, void *state_p) 84 | { 85 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 86 | const char *topic; 87 | FETCH_PARAMS (&topic); 88 | 89 | dafka_producer_args_t args = {topic, state->config}; 90 | state->producer = zactor_new (dafka_producer, &args); 91 | assert (state->producer); 92 | zclock_sleep (250); // Make sure producer is connected to test_peer before continuing 93 | state->topic = strdup (topic); 94 | } 95 | 96 | void 97 | given_a_dafka_producer_sent_a_record_message (cucumber_step_def_t *self, void *state_p) 98 | { 99 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 100 | const char *content; 101 | FETCH_PARAMS (&content); 102 | 103 | dafka_producer_msg_t *msg = dafka_producer_msg_new (); 104 | dafka_producer_msg_set_content_str (msg, content); 105 | dafka_producer_msg_send (msg, state->producer); 106 | dafka_producer_msg_destroy (&msg); 107 | 108 | dafka_proto_t *received_msg = dafka_test_peer_recv (state->test_peer); 109 | assert_that_char (dafka_proto_id (received_msg), char_equals, DAFKA_PROTO_RECORD); 110 | dafka_proto_destroy (&received_msg); 111 | } 112 | 113 | void 114 | when_a_dafka_producer_sends_a_record_message (cucumber_step_def_t *self, void *state_p) 115 | { 116 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 117 | const char *content; 118 | FETCH_PARAMS (&content); 119 | 120 | dafka_producer_msg_t *msg = dafka_producer_msg_new (); 121 | dafka_producer_msg_set_content_str (msg, content); 122 | dafka_producer_msg_send (msg, state->producer); 123 | dafka_producer_msg_destroy (&msg); 124 | } 125 | 126 | void 127 | when_the_producer_recieves_a_fetch_message (cucumber_step_def_t *self, void *state_p) 128 | { 129 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 130 | const char *sequence; 131 | FETCH_PARAMS (&sequence); 132 | 133 | dafka_test_peer_send_fetch (state->test_peer, 134 | dafka_producer_address (state->producer), 135 | state->topic, 136 | atoi (sequence), 137 | 1); 138 | } 139 | 140 | void 141 | then_the_producer_will_send_a_direct_record_message (cucumber_step_def_t *self, void *state_p) 142 | { 143 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 144 | const char *content; 145 | FETCH_PARAMS (&content); 146 | 147 | dafka_proto_t *msg = NULL; 148 | do { 149 | if (msg) 150 | dafka_proto_destroy (&msg); 151 | 152 | msg = dafka_test_peer_recv (state->test_peer); 153 | } while (dafka_proto_id (msg) == DAFKA_PROTO_HEAD); 154 | 155 | assert_that_char (dafka_proto_id (msg), char_equals, DAFKA_PROTO_DIRECT_RECORD); 156 | assert_that_str (dafka_proto_topic (msg), streq, dafka_test_peer_address (state->test_peer)); 157 | assert_that_str (dafka_proto_subject (msg), streq, state->topic); 158 | zmq_msg_t *content_msg = dafka_proto_content (msg); 159 | char *data = (char *) zmq_msg_data (content_msg); 160 | assert_that_str (data, streq, content); 161 | 162 | dafka_proto_destroy (&msg); 163 | } 164 | 165 | void 166 | then_a_producer_will_send_head_messages_at_regular_intervals (cucumber_step_def_t *self, void *state_p) 167 | { 168 | dafka_producer_state_t *state = (dafka_producer_state_t *) state_p; 169 | 170 | for (int index = 0; index < 5; index++) { 171 | dafka_proto_t *msg = dafka_test_peer_recv (state->test_peer); 172 | 173 | if (dafka_proto_id (msg) != DAFKA_PROTO_RECORD) { 174 | assert_that_char (dafka_proto_id (msg), char_equals, DAFKA_PROTO_HEAD); 175 | } 176 | 177 | dafka_proto_destroy (&msg); 178 | } 179 | } 180 | 181 | STEP_DEFS(dafka_producer, dafka_producer_state_new, dafka_producer_state_destroy) { 182 | GIVEN("a dafka producer for topic (\\w+)", given_a_dafka_producer_for_topic) 183 | GIVEN("the producer sent a RECORD message with content '(\\w+)'", 184 | given_a_dafka_producer_sent_a_record_message) 185 | 186 | WHEN("the producer receives a FETCH message with sequence (\\d+)", 187 | when_the_producer_recieves_a_fetch_message) 188 | WHEN("the producer sends a RECORD message with content '(\\w+)'", 189 | when_a_dafka_producer_sends_a_record_message) 190 | 191 | THEN("the producer will send a DIRECT_RECORD message with content '(\\w+)'", 192 | then_the_producer_will_send_a_direct_record_message) 193 | THEN("the producer will send HEAD messages at regular intervals", 194 | then_a_producer_will_send_head_messages_at_regular_intervals) 195 | } 196 | #endif 197 | -------------------------------------------------------------------------------- /src/dafka_producer_step_defs.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer_step_defs - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_PRODUCER_STEP_DEFS_H_INCLUDED 15 | #define DAFKA_PRODUCER_STEP_DEFS_H_INCLUDED 16 | 17 | #if defined (HAVE_CUCUMBER) 18 | #include 19 | 20 | typedef struct _dafka_producer_state dafka_producer_state_t; 21 | 22 | DAFKA_EXPORT dafka_producer_state_t * 23 | dafka_producer_state_new (bool verbose); 24 | 25 | DAFKA_EXPORT void 26 | dafka_producer_state_destroy (dafka_producer_state_t **self_p); 27 | 28 | DAFKA_EXPORT void 29 | register_dafka_producer_step_defs (cucumber_t *cucumber); 30 | #endif 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /src/dafka_producer_step_runner.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_producer_step_runner - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. This 5 | file is part of DAFKA, a decentralized distributed streaming 6 | platform: http://zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_producer_step_runner - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | int main (int argc, char *argv []) 24 | { 25 | bool verbose = false; 26 | int argn; 27 | for (argn = 1; argn < argc; argn++) { 28 | if (streq (argv [argn], "--help") 29 | || streq (argv [argn], "-h")) { 30 | puts ("dafka_producer_step_runner [options] ..."); 31 | puts (" --verbose / -v verbose test output"); 32 | puts (" --help / -h this information"); 33 | return 0; 34 | } 35 | else 36 | if (streq (argv [argn], "--verbose") 37 | || streq (argv [argn], "-v")) 38 | verbose = true; 39 | else { 40 | printf ("Unknown option: %s\n", argv [argn]); 41 | return 1; 42 | } 43 | } 44 | // Insert main code here 45 | if (verbose) 46 | zsys_info ("dafka_producer_step_runner - "); 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /src/dafka_proto.bnf: -------------------------------------------------------------------------------- 1 | The following ABNF grammar defines the dafka_proto: 2 | 3 | DAFKA = join-consumer / publish / offsets 4 | 5 | join-consumer = S:STORE-HELLO C:CONSUMER-HELLO *( S:DIRECT-HEAD [ consumer-fetch ] ) 6 | 7 | consumer-fetch = C:FETCH 1*( P:DIRECT-RECORD / S:DIRECT-RECORD ) 8 | 9 | publish = P:RECORD [ consumer-fetch / store-fetch S:ACK ] 10 | 11 | store-fetch = S:FETCH 1*( ( P:DIRECT-RECORD / S:DIRECT-RECORD ) [ S:ACK ] ) 12 | 13 | offsets = P:HEAD [ consumer-fetch / store-fetch ] 14 | 15 | ; Record from producer to consumers and stores. Topic is the name of the 16 | ; topic. Subject is the name of the topic. Address is the address of the 17 | ; producer (partition). 18 | 19 | RECORD = signature %d'm' version address subject sequence content 20 | signature = %xAA %xA5 ; two octets 21 | version = number-1 ; Version = 1 22 | address = string ; 23 | subject = string ; 24 | sequence = number-8 ; 25 | content = frame ; 26 | 27 | ; Direct record from a producer or a store to a consumer. Topic is the 28 | ; address of the requestor. Subject is the name of the topic. Address is 29 | ; the address of the producer (partition). 30 | 31 | DIRECT-RECORD = signature %d'd' version address subject sequence content 32 | version = number-1 ; Version = 1 33 | address = string ; 34 | subject = string ; 35 | sequence = number-8 ; 36 | content = frame ; 37 | 38 | ; Consumer or store publish this message when a record is missing. 39 | ; Either producer or a store can answer. Topic is the address of the 40 | ; producer (partition). Subject is the name of the topic. Address is the 41 | ; address of this message's sender. Count is the number of messages to 42 | ; fetch starting with the record identified by sequence. 43 | 44 | FETCH = signature %d'f' version address subject sequence count 45 | version = number-1 ; Version = 1 46 | address = string ; 47 | subject = string ; 48 | sequence = number-8 ; 49 | count = number-4 ; 50 | 51 | ; Ack from a stores to a producer. Topic is the address of the producer 52 | ; (partition). Subject is the name of the topic. 53 | 54 | ACK = signature %d'k' version subject sequence 55 | version = number-1 ; Version = 1 56 | subject = string ; 57 | sequence = number-8 ; 58 | 59 | ; Head from producer to consumers and stores. Topic is the name of the 60 | ; topic. Subject is the name of the topic. Address is the address of the 61 | ; producer (partition). 62 | 63 | HEAD = signature %d'h' version address subject sequence 64 | version = number-1 ; Version = 1 65 | address = string ; 66 | subject = string ; 67 | sequence = number-8 ; 68 | 69 | ; Head from producer or store to a consumers. Topic is the name of the 70 | ; topic. Subject is the name of the topic. Address is the address of the 71 | ; producer (partition). 72 | 73 | DIRECT-HEAD = signature %d'e' version address subject sequence 74 | version = number-1 ; Version = 1 75 | address = string ; 76 | subject = string ; 77 | sequence = number-8 ; 78 | 79 | ; Get heads from stores send by a consumer. Topic is the name of the 80 | ; topic. Address is the address of the consumer. 81 | 82 | GET-HEADS = signature %d'g' version address 83 | version = number-1 ; Version = 1 84 | address = string ; 85 | 86 | ; Hello message from a consumer to a store. Topic is the store's 87 | ; address. Address is the address of the consumer. Subjects is a list of 88 | ; all topic the consumer is subscribed to. 89 | 90 | CONSUMER-HELLO = signature %d'w' version address subjects 91 | version = number-1 ; Version = 1 92 | address = string ; 93 | subjects = strings ; 94 | 95 | ; Hello message from a store to a consumer. Topic is the consumer's 96 | ; address. Address is the address of the store. 97 | 98 | STORE-HELLO = signature %d'l' version address 99 | version = number-1 ; Version = 1 100 | address = string ; 101 | 102 | ; A list of string values 103 | strings = strings-count *strings-value 104 | strings-count = number-4 105 | strings-value = longstr 106 | 107 | ; A frame is zero or more octets encoded as a ZeroMQ frame 108 | frame = *OCTET 109 | 110 | ; Strings are always length + text contents 111 | string = number-1 *VCHAR 112 | longstr = number-4 *VCHAR 113 | 114 | ; Numbers are unsigned integers in network byte order 115 | number-1 = 1OCTET 116 | number-4 = 4OCTET 117 | number-8 = 8OCTET 118 | -------------------------------------------------------------------------------- /src/dafka_proto.xml: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | DAFKA = join-consumer / publish / offsets 15 | 16 | join-consumer = S:STORE-HELLO C:CONSUMER-HELLO *( S:DIRECT-HEAD [ consumer-fetch ] ) 17 | 18 | consumer-fetch = C:FETCH 1*( P:DIRECT-RECORD / S:DIRECT-RECORD ) 19 | 20 | publish = P:RECORD [ consumer-fetch / store-fetch S:ACK ] 21 | 22 | store-fetch = S:FETCH 1*( ( P:DIRECT-RECORD / S:DIRECT-RECORD ) [ S:ACK ] ) 23 | 24 | offsets = P:HEAD [ consumer-fetch / store-fetch ] 25 | 26 | 27 | 28 |
29 | Version = 1 30 |
31 | 32 | 33 | Record from producer to consumers and stores. 34 | 35 | Topic is the name of the topic. 36 | Subject is the name of the topic. 37 | Address is the address of the producer (partition). 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | Direct record from a producer or a store to a consumer. 47 | 48 | Topic is the address of the requestor. 49 | Subject is the name of the topic. 50 | Address is the address of the producer (partition). 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | Consumer or store publish this message when a record is missing. 60 | Either producer or a store can answer. 61 | 62 | Topic is the address of the producer (partition). 63 | Subject is the name of the topic. 64 | Address is the address of this message's sender. 65 | 66 | Count is the number of messages to fetch starting with the record 67 | identified by sequence. 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | Ack from a stores to a producer. 77 | 78 | Topic is the address of the producer (partition). 79 | Subject is the name of the topic. 80 | 81 | 82 | 83 | 84 | 85 | 86 | Head from producer to consumers and stores. 87 | 88 | Topic is the name of the topic. 89 | Subject is the name of the topic. 90 | Address is the address of the producer (partition). 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | Head from producer or store to a consumers. 99 | 100 | Topic is the name of the topic. 101 | Subject is the name of the topic. 102 | Address is the address of the producer (partition). 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | Get heads from stores send by a consumer. 111 | 112 | Topic is the name of the topic. 113 | Address is the address of the consumer. 114 | 115 | 116 | 117 | 118 | 119 | Hello message from a consumer to a store. 120 | 121 | Topic is the store's address. 122 | Address is the address of the consumer. 123 | Subjects is a list of all topic the consumer is subscribed to. 124 | 125 | 126 | 127 | 128 | 129 | 130 | Hello message from a store to a consumer. 131 | 132 | Topic is the consumer's address. 133 | Address is the address of the store. 134 | 135 | 136 | 137 | 138 | 139 | -------------------------------------------------------------------------------- /src/dafka_selftest.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_selftest.c - run selftests 3 | 4 | Runs all selftests. 5 | 6 | ------------------------------------------------------------------------- 7 | Copyright (c) the Contributors as noted in the AUTHORS file. This 8 | file is part of DAFKA, a decentralized distributed streaming 9 | platform: http://zeromq.org. 10 | 11 | This Source Code Form is subject to the terms of the Mozilla Public 12 | License, v. 2.0. If a copy of the MPL was not distributed with this 13 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 14 | 15 | ################################################################################ 16 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 17 | # Read the zproject/README.md for information about making permanent changes. # 18 | ################################################################################ 19 | ========================================================================= 20 | */ 21 | 22 | #include "dafka_classes.h" 23 | 24 | typedef struct { 25 | const char *testname; // test name, can be called from command line this way 26 | void (*test) (bool); // function to run the test (or NULL for private tests) 27 | bool stable; // true if class is declared as stable 28 | bool pub; // true if class is declared as public 29 | const char *subtest; // name of private subtest to run 30 | } test_item_t; 31 | 32 | static test_item_t 33 | all_tests [] = { 34 | #ifdef DAFKA_BUILD_DRAFT_API 35 | // Tests for stable/draft private classes: 36 | // Now built only with --enable-drafts, so even stable builds are hidden behind the flag 37 | { "dafka_unacked_list", NULL, true, false, "dafka_unacked_list_test" }, 38 | { "dafka_util", NULL, true, false, "dafka_util_test" }, 39 | { "dafka_test_peer", NULL, true, false, "dafka_test_peer_test" }, 40 | { "private_classes", NULL, false, false, "$ALL" }, // compat option for older projects 41 | #endif // DAFKA_BUILD_DRAFT_API 42 | // Tests for stable public classes: 43 | { "dafka_consumer_msg", dafka_consumer_msg_test, true, true, NULL }, 44 | { "dafka_producer_msg", dafka_producer_msg_test, true, true, NULL }, 45 | { "dafka_producer", dafka_producer_test, true, true, NULL }, 46 | { "dafka_consumer", dafka_consumer_test, true, true, NULL }, 47 | { "dafka_proto", dafka_proto_test, true, true, NULL }, 48 | { "dafka_beacon", dafka_beacon_test, true, true, NULL }, 49 | { "dafka_tower", dafka_tower_test, true, true, NULL }, 50 | { "dafka_store", dafka_store_test, true, true, NULL }, 51 | {NULL, NULL, 0, 0, NULL} // Sentinel 52 | }; 53 | 54 | // ------------------------------------------------------------------------- 55 | // Test whether a test is available. 56 | // Return a pointer to a test_item_t if available, NULL otherwise. 57 | // 58 | 59 | test_item_t * 60 | test_available (const char *testname) 61 | { 62 | test_item_t *item; 63 | for (item = all_tests; item->testname; item++) { 64 | if (streq (testname, item->testname)) 65 | return item; 66 | } 67 | return NULL; 68 | } 69 | 70 | // ------------------------------------------------------------------------- 71 | // Run all tests. 72 | // 73 | 74 | static void 75 | test_runall (bool verbose) 76 | { 77 | test_item_t *item; 78 | printf ("Running dafka selftests...\n"); 79 | for (item = all_tests; item->testname; item++) { 80 | if (streq (item->testname, "private_classes")) 81 | continue; 82 | if (!item->subtest) 83 | item->test (verbose); 84 | #ifdef DAFKA_BUILD_DRAFT_API // selftest is still in draft 85 | else 86 | dafka_private_selftest (verbose, item->subtest); 87 | #endif // DAFKA_BUILD_DRAFT_API 88 | } 89 | 90 | printf ("Tests passed OK\n"); 91 | } 92 | 93 | static void 94 | test_list (void) 95 | { 96 | test_item_t *item; 97 | puts ("Available tests:"); 98 | for (item = all_tests; item->testname; item++) 99 | printf (" %-40s - %s %s\n", 100 | item->testname, 101 | item->stable ? "stable" : "draft", 102 | item->pub ? "public" : "private" 103 | ); 104 | } 105 | 106 | static void 107 | test_number (void) 108 | { 109 | int n = 0; 110 | test_item_t *item; 111 | for (item = all_tests; item->testname; item++) { 112 | if (! streq (item->testname, "private_classes")) 113 | n++; 114 | } 115 | printf ("%d\n", n); 116 | } 117 | 118 | int 119 | main (int argc, char **argv) 120 | { 121 | bool verbose = false; 122 | test_item_t *test = 0; 123 | int argn; 124 | for (argn = 1; argn < argc; argn++) { 125 | if (streq (argv [argn], "--help") 126 | || streq (argv [argn], "-h")) { 127 | puts ("dafka_selftest.c [options] ..."); 128 | puts (" --verbose / -v verbose test output"); 129 | puts (" --number / -n report number of tests"); 130 | puts (" --list / -l list all tests"); 131 | puts (" --test / -t [name] run only test 'name'"); 132 | puts (" --continue / -c continue on exception (on Windows)"); 133 | return 0; 134 | } 135 | if (streq (argv [argn], "--verbose") 136 | || streq (argv [argn], "-v")) 137 | verbose = true; 138 | else 139 | if (streq (argv [argn], "--number") 140 | || streq (argv [argn], "-n")) { 141 | test_number (); 142 | return 0; 143 | } 144 | else 145 | if (streq (argv [argn], "--list") 146 | || streq (argv [argn], "-l")) { 147 | test_list (); 148 | return 0; 149 | } 150 | else 151 | if (streq (argv [argn], "--test") 152 | || streq (argv [argn], "-t")) { 153 | argn++; 154 | if (argn >= argc) { 155 | fprintf (stderr, "--test needs an argument\n"); 156 | return 1; 157 | } 158 | test = test_available (argv [argn]); 159 | if (!test) { 160 | fprintf (stderr, "%s not valid, use --list to show tests\n", argv [argn]); 161 | return 1; 162 | } 163 | } 164 | else 165 | if (streq (argv [argn], "--continue") 166 | || streq (argv [argn], "-c")) { 167 | #ifdef _MSC_VER 168 | // When receiving an abort signal, only print to stderr (no dialog) 169 | _set_abort_behavior (0, _WRITE_ABORT_MSG); 170 | #endif 171 | } 172 | else { 173 | printf ("Unknown option: %s\n", argv [argn]); 174 | return 1; 175 | } 176 | } 177 | 178 | #ifdef NDEBUG 179 | printf(" !!! 'assert' macro is disabled, remove NDEBUG from your compilation definitions.\n"); 180 | printf(" tests will be meaningless.\n"); 181 | #endif // 182 | 183 | if (test) { 184 | printf ("Running dafka test '%s'...\n", test->testname); 185 | if (!test->subtest) 186 | test->test (verbose); 187 | #ifdef DAFKA_BUILD_DRAFT_API // selftest is still in draft 188 | else 189 | dafka_private_selftest (verbose, test->subtest); 190 | #endif // DAFKA_BUILD_DRAFT_API 191 | } 192 | else 193 | test_runall (verbose); 194 | 195 | return 0; 196 | } 197 | /* 198 | ################################################################################ 199 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 200 | # Read the zproject/README.md for information about making permanent changes. # 201 | ################################################################################ 202 | */ 203 | -------------------------------------------------------------------------------- /src/dafka_store_reader.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_store_reader - 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFAK_STORE_READER_H_INCLUDED 15 | #define DAFAK_STORE_READER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | typedef struct { 22 | const char* writer_address; 23 | leveldb_t *db; 24 | zconfig_t *config; 25 | } dafka_store_reader_args_t; 26 | 27 | typedef struct _dafka_store_reader_t dafka_store_reader_t; 28 | 29 | // Create new dafka_store_reader actor instance. 30 | DAFKA_EXPORT void 31 | dafka_store_reader_actor (zsock_t *pipe, dafka_store_reader_args_t *args); 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/dafka_store_writer.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_store_writer - 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFAK_STORE_WRITER_H_INCLUDED 15 | #define DAFAK_STORE_WRITER_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | typedef struct { 22 | const char* address; 23 | leveldb_t *db; 24 | zconfig_t *config; 25 | } dafka_store_writer_args_t; 26 | 27 | typedef struct _dafka_store_writer_t dafka_store_writer_t; 28 | 29 | // Create new dafka_store_writer actor instance. 30 | DAFKA_EXPORT void 31 | dafka_store_writer_actor (zsock_t *pipe, dafka_store_writer_args_t *args); 32 | 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/dafka_stored.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_stored - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_stored - 17 | @discuss 18 | TODO: handle heads 19 | TODO: ask for missing 20 | TODO: be persisted 21 | TODO: implement the window 22 | @end 23 | */ 24 | 25 | #include "dafka_classes.h" 26 | 27 | 28 | int main (int argc, char** argv) { 29 | zargs_t *args = zargs_new (argc, argv); 30 | 31 | if (zargs_hasx (args, "--help", "-h", NULL)) { 32 | puts ("Usage: dafka_stored [--verbose] [-c config] [--pub tower-pub-address] [--sub tower-sub-address]"); 33 | return 0; 34 | } 35 | 36 | zconfig_t *config; 37 | 38 | if (zargs_has (args, "-c")) 39 | config = zconfig_load (zargs_get (args, "-c")); 40 | else 41 | config = zconfig_new ("root", NULL); 42 | 43 | if (zargs_has (args, "--verbose")) { 44 | zconfig_put (config, "beacon/verbose", "1"); 45 | zconfig_put (config, "store/verbose", "1"); 46 | } 47 | 48 | if (zargs_has (args, "--pub")) 49 | zconfig_put (config, "beacon/pub_address", zargs_get (args, "--pub")); 50 | 51 | if (zargs_has (args, "--sub")) 52 | zconfig_put (config, "beacon/sub_address", zargs_get (args, "--sub")); 53 | 54 | zactor_t *store = zactor_new (dafka_store_actor, config); 55 | 56 | char* command = zstr_recv (store); 57 | zstr_free (&command); // Interrupted 58 | 59 | zconfig_destroy (&config); 60 | zargs_destroy (&args); 61 | zactor_destroy (&store); 62 | 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /src/dafka_test_peer.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_test_peer - Peer to test interact with producers, consumers and 3 | stores. 4 | 5 | Copyright (c) the Contributors as noted in the AUTHORS file. This 6 | file is part of DAFKA, a decentralized distributed streaming 7 | platform: http://zeromq.org. 8 | 9 | This Source Code Form is subject to the terms of the Mozilla Public 10 | License, v. 2.0. If a copy of the MPL was not distributed with this 11 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | ========================================================================= 13 | */ 14 | 15 | #ifndef DAFKA_TEST_PEER_H_INCLUDED 16 | #define DAFKA_TEST_PEER_H_INCLUDED 17 | 18 | #ifdef __cplusplus 19 | extern "C" { 20 | #endif 21 | 22 | typedef struct _dafka_test_peer_t dafka_test_peer_t; 23 | 24 | // @interface 25 | // Create new dafka_test_peer actor instance. 26 | // 27 | // zactor_t *dafka_test_peer = zactor_new (dafka_test_peer, NULL); 28 | // 29 | // Destroy dafka_test_peer instance. 30 | // 31 | // zactor_destroy (&dafka_test_peer); 32 | // 33 | // This is the dafka_test_peer constructor as a zactor_fn; 34 | DAFKA_EXPORT void 35 | dafka_test_peer (zsock_t *pipe, void *args); 36 | 37 | // Self test of this actor 38 | DAFKA_EXPORT void 39 | dafka_test_peer_test (bool verbose); 40 | 41 | DAFKA_EXPORT void 42 | dafka_test_peer_send_store_hello (zactor_t *self, const char *consumer_address); 43 | 44 | DAFKA_EXPORT void 45 | dafka_test_peer_send_head (zactor_t *self, const char *topic, uint64_t sequence); 46 | 47 | DAFKA_EXPORT void 48 | dafka_test_peer_send_record (zactor_t *self, const char *topic, uint64_t sequence, const char *content); 49 | 50 | DAFKA_EXPORT void 51 | dafka_test_peer_send_fetch (zactor_t *self, const char *address, const char *topic, uint64_t sequence, uint64_t count ); 52 | 53 | DAFKA_EXPORT dafka_proto_t * 54 | dafka_test_peer_recv (zactor_t *self); 55 | 56 | DAFKA_EXPORT const char * 57 | dafka_test_peer_address (zactor_t *self); 58 | 59 | DAFKA_EXPORT void 60 | assert_consumer_hello_msg (dafka_proto_t *msg, int no_of_subjects); 61 | 62 | DAFKA_EXPORT void 63 | assert_get_heads_msg (dafka_proto_t *msg, const char *topic); 64 | 65 | DAFKA_EXPORT void 66 | assert_fetch_msg (dafka_proto_t *msg, const char *topic, uint64_t sequence); 67 | 68 | DAFKA_EXPORT void 69 | assert_consumer_msg (dafka_consumer_msg_t *msg, const char *topic, const char *content); 70 | // @end 71 | 72 | #ifdef __cplusplus 73 | } 74 | #endif 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /src/dafka_tower.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_tower - 3 | 4 | This Source Code Form is subject to the terms of the Mozilla Public 5 | License, v. 2.0. If a copy of the MPL was not distributed with this 6 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | ========================================================================= 8 | */ 9 | 10 | /* 11 | @header 12 | dafka_tower - 13 | @discuss 14 | @end 15 | */ 16 | 17 | #include "dafka_classes.h" 18 | 19 | // Structure of our actor 20 | 21 | struct _dafka_tower_t { 22 | zsock_t *pipe; // Actor command pipe 23 | zpoller_t *poller; // Socket poller 24 | bool terminated; // Did caller ask us to quit? 25 | bool verbose; // Verbose logging enabled? 26 | zsock_t *xsub; 27 | zsock_t *xpub; 28 | char *own_address; 29 | }; 30 | 31 | 32 | // -------------------------------------------------------------------------- 33 | // Create a new dafka_tower instance 34 | 35 | static dafka_tower_t * 36 | dafka_tower_new (zsock_t *pipe, zconfig_t *config) { 37 | dafka_tower_t *self = (dafka_tower_t *) zmalloc (sizeof (dafka_tower_t)); 38 | assert (self); 39 | 40 | self->pipe = pipe; 41 | self->terminated = false; 42 | 43 | if (atoi (zconfig_get (config, "tower/verbose", "0"))) 44 | self->verbose = true; 45 | 46 | const char *sub_address = zconfig_get (config, "tower/sub_address", "tcp://*:5556"); 47 | const char *pub_address = zconfig_get (config, "tower/pub_address", "tcp://*:5557"); 48 | 49 | self->xpub = zsock_new_xpub (NULL); 50 | self->xsub = zsock_new_xsub (NULL); 51 | 52 | zsock_set_xpub_welcome_msg (self->xpub, "W"); 53 | 54 | zsock_bind (self->xpub, "%s", pub_address); 55 | zsock_bind (self->xsub, "%s", sub_address); 56 | 57 | if (self->verbose) { 58 | zsys_info ("Tower: xsub listening on %s", sub_address); 59 | zsys_info ("Tower: xpub listening on %s", pub_address); 60 | } 61 | 62 | self->poller = zpoller_new (self->pipe, self->xsub, self->xpub, NULL); 63 | zpoller_set_nonstop (self->poller, true); 64 | 65 | ziflist_t *iflist = ziflist_new (); 66 | ziflist_first (iflist); 67 | self->own_address = strdup (ziflist_address (iflist)); 68 | ziflist_destroy (&iflist); 69 | 70 | return self; 71 | } 72 | 73 | 74 | // -------------------------------------------------------------------------- 75 | // Destroy the dafka_tower instance 76 | 77 | static void 78 | dafka_tower_destroy (dafka_tower_t **self_p) { 79 | assert (self_p); 80 | if (*self_p) { 81 | dafka_tower_t *self = *self_p; 82 | 83 | zpoller_destroy (&self->poller); 84 | zsock_destroy (&self->xpub); 85 | zsock_destroy (&self->xsub); 86 | zstr_free (&self->own_address); 87 | 88 | // Free object itself 89 | free (self); 90 | *self_p = NULL; 91 | } 92 | } 93 | 94 | // Here we handle incoming message from the node 95 | 96 | static void 97 | dafka_tower_recv_api (dafka_tower_t *self) { 98 | // Get the whole message of the pipe in one go 99 | zmsg_t *request = zmsg_recv (self->pipe); 100 | if (!request) 101 | return; // Interrupted 102 | 103 | char *command = zmsg_popstr (request); 104 | if (streq (command, "$TERM")) { 105 | // The $TERM command is send by zactor_destroy() method 106 | self->terminated = true; 107 | } 108 | else { 109 | zsys_error ("invalid command '%s'", command); 110 | assert (false); 111 | } 112 | zstr_free (&command); 113 | zmsg_destroy (&request); 114 | } 115 | 116 | 117 | // -------------------------------------------------------------------------- 118 | // This is the actor which runs in its own thread. 119 | 120 | void 121 | dafka_tower_actor (zsock_t *pipe, void *args) { 122 | dafka_tower_t *self = dafka_tower_new (pipe, (zconfig_t *) args); 123 | if (!self) 124 | return; // Interrupted 125 | 126 | // Signal actor successfully initiated 127 | zsock_signal (self->pipe, 0); 128 | 129 | if (self->verbose) 130 | zsys_info ("Tower: tower is running..."); 131 | 132 | while (!self->terminated) { 133 | zsock_t *which = (zsock_t *) zpoller_wait (self->poller, -1); 134 | if (which == self->pipe) 135 | dafka_tower_recv_api (self); 136 | else if (which == self->xsub) { 137 | char *sender; 138 | char *host; 139 | int port; 140 | zframe_t *topic = zframe_recv (self->xsub); 141 | zsock_recv (self->xsub, "ssi", &sender, &host, &port); 142 | 143 | const char *peer_address; 144 | if (host && strneq (host, "")) 145 | peer_address = host; 146 | else { 147 | peer_address = zframe_meta (topic, ZMQ_MSG_PROPERTY_PEER_ADDRESS); 148 | 149 | // If the nodes connect over inproc we don't have there address, so we will use 150 | // own address 151 | if (peer_address == NULL) 152 | peer_address = self->own_address; 153 | else if (streq (peer_address, "127.0.0.1")) 154 | peer_address = self->own_address; 155 | } 156 | 157 | char *endpoint = zsys_sprintf ("tcp://%s:%d", peer_address, port); 158 | 159 | // Forwarding the msg 160 | zframe_send (&topic, self->xpub, ZMQ_MORE); 161 | zsock_send (self->xpub, "ss", sender, endpoint); 162 | 163 | zstr_free (&endpoint); 164 | zstr_free (&host); 165 | zstr_free (&sender); 166 | } else if (which == self->xpub) { 167 | zframe_t *subscription = zframe_recv (self->xpub); 168 | 169 | if (self->verbose && (zframe_data (subscription)[0]) == 0) 170 | zsys_debug ("Tower: Received unsubscription %c", zframe_data (subscription)[1]); 171 | 172 | if (self->verbose && (zframe_data (subscription)[0]) == 1) 173 | zsys_debug ("Tower: Received subscription %c", zframe_data (subscription)[1]); 174 | 175 | // We drop welcome subscription, no need to forward them 176 | if (zframe_size (subscription) >= 2 && zframe_data (subscription)[1] == 'W') { 177 | zframe_destroy (&subscription); 178 | } else { 179 | zframe_send (&subscription, self->xsub, 0); 180 | } 181 | } 182 | } 183 | 184 | bool verbose = self->verbose; 185 | dafka_tower_destroy (&self); 186 | 187 | if (verbose) 188 | zsys_info ("Tower: tower stopped"); 189 | } 190 | 191 | // -------------------------------------------------------------------------- 192 | // Self test of this actor. 193 | 194 | // If your selftest reads SCMed fixture data, please keep it in 195 | // src/selftest-ro; if your test creates filesystem objects, please 196 | // do so under src/selftest-rw. 197 | // The following pattern is suggested for C selftest code: 198 | // char *filename = NULL; 199 | // filename = zsys_sprintf ("%s/%s", SELFTEST_DIR_RO, "mytemplate.file"); 200 | // assert (filename); 201 | // ... use the "filename" for I/O ... 202 | // zstr_free (&filename); 203 | // This way the same "filename" variable can be reused for many subtests. 204 | #define SELFTEST_DIR_RO "src/selftest-ro" 205 | #define SELFTEST_DIR_RW "src/selftest-rw" 206 | 207 | void 208 | dafka_tower_test (bool verbose) { 209 | printf (" * dafka_tower: "); 210 | // @selftest 211 | // Simple create/destroy test 212 | /* 213 | zactor_t *dafka_tower = zactor_new (dafka_tower_actor, NULL); 214 | assert (dafka_tower); 215 | 216 | zactor_destroy (&dafka_tower); 217 | */ 218 | // @end 219 | 220 | printf ("OK\n"); 221 | } 222 | -------------------------------------------------------------------------------- /src/dafka_towerd.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_towerd - description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_towerd - 17 | @discuss 18 | TODO: actually use the port params 19 | @end 20 | */ 21 | 22 | #include "dafka_classes.h" 23 | 24 | int main (int argc, char** argv) { 25 | zargs_t *args = zargs_new (argc, argv); 26 | 27 | if (zargs_hasx (args, "--help", "-h", NULL)) { 28 | puts ("Usage: dafka_towerd [-c config] [--pub tower-pub-address] [--sub tower-sub-address] [--verbose]"); 29 | return 0; 30 | } 31 | 32 | zconfig_t *config; 33 | 34 | if (zargs_has (args, "-c")) 35 | config = zconfig_load (zargs_get (args, "-c")); 36 | else 37 | config = zconfig_new ("root", NULL); 38 | 39 | if (zargs_has (args, "--verbose")) 40 | zconfig_put (config, "tower/verbose", "1"); 41 | 42 | if (zargs_has (args, "--pub")) 43 | zconfig_put (config, "tower/pub_address", zargs_get (args, "--pub")); 44 | 45 | if (zargs_has (args, "--sub")) 46 | zconfig_put (config, "tower/sub_address", zargs_get (args, "--sub")); 47 | 48 | zactor_t *tower = zactor_new (dafka_tower_actor, config); 49 | 50 | char* command = zstr_recv (tower); 51 | zstr_free (&command); // Interrupted 52 | 53 | zconfig_destroy (&config); 54 | zargs_destroy (&args); 55 | zactor_destroy (&tower); 56 | 57 | return 0; 58 | } 59 | -------------------------------------------------------------------------------- /src/dafka_unacked_list.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_unacked_list - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_UNACKED_LIST_H_INCLUDED 15 | #define DAFKA_UNACKED_LIST_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | // @interface 22 | // Create a new dafka_unacked_list 23 | DAFKA_PRIVATE dafka_unacked_list_t * 24 | dafka_unacked_list_new (zsock_t *publisher, const char* address, const char *topic); 25 | 26 | // Destroy the dafka_unacked_list 27 | DAFKA_PRIVATE void 28 | dafka_unacked_list_destroy (dafka_unacked_list_t **self_p); 29 | 30 | DAFKA_PRIVATE uint64_t 31 | dafka_unacked_list_push (dafka_unacked_list_t *self, zmq_msg_t *msg); 32 | 33 | DAFKA_PRIVATE void 34 | dafka_unacked_list_ack (dafka_unacked_list_t *self, uint64_t sequence); 35 | 36 | DAFKA_PRIVATE void 37 | dafka_unacked_list_send (dafka_unacked_list_t *self, const char* address, uint64_t sequence, uint32_t count); 38 | 39 | DAFKA_PRIVATE bool 40 | dafka_unacked_list_is_empty (dafka_unacked_list_t *self); 41 | 42 | DAFKA_PRIVATE uint64_t 43 | dafka_unacked_list_last_acked (dafka_unacked_list_t *self); 44 | 45 | 46 | // Self test of this class 47 | DAFKA_PRIVATE void 48 | dafka_unacked_list_test (bool verbose); 49 | 50 | // @end 51 | 52 | #ifdef __cplusplus 53 | } 54 | #endif 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/dafka_util.c: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_util - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | /* 15 | @header 16 | dafka_util - 17 | @discuss 18 | @end 19 | */ 20 | 21 | #include "dafka_classes.h" 22 | 23 | void 24 | uint64_destroy (void **item_p) 25 | { 26 | assert (item_p); 27 | if (*item_p) { 28 | uint64_t *item = (uint64_t *) *item_p; 29 | free (item); 30 | *item_p = NULL; 31 | } 32 | } 33 | 34 | void * 35 | uint64_dup (const void *item) 36 | { 37 | uint64_t *value = (uint64_t *) malloc (sizeof (uint64_t)); 38 | memcpy (value, item, sizeof (uint64_t)); 39 | return value; 40 | } 41 | 42 | int 43 | uint64_cmp (const void *item1, const void *item2) 44 | { 45 | uint64_t value1 = *((uint64_t *) item1); 46 | uint64_t value2 = *((uint64_t *) item2); 47 | if (value1 < value2) 48 | return -1; 49 | else 50 | if (value1 > value2) 51 | return 1; 52 | 53 | return 0; 54 | } 55 | 56 | size_t 57 | uint64_hash (const void *item) 58 | { 59 | uint64_t value = *((uint64_t *) item); 60 | return (size_t) value; 61 | } 62 | 63 | char * 64 | generate_address () { 65 | zuuid_t *uuid = zuuid_new (); 66 | char *address = strdup (zuuid_str (uuid)); 67 | zuuid_destroy (&uuid); 68 | return address; 69 | } 70 | 71 | size_t 72 | uint64_put_be (byte* output, uint64_t value) { 73 | output[0] = (byte) (((value) >> 56) & 255); 74 | output[1] = (byte) (((value) >> 48) & 255); 75 | output[2] = (byte) (((value) >> 40) & 255); 76 | output[3] = (byte) (((value) >> 32) & 255); 77 | output[4] = (byte) (((value) >> 24) & 255); 78 | output[5] = (byte) (((value) >> 16) & 255); 79 | output[6] = (byte) (((value) >> 8) & 255); 80 | output[7] = (byte) (((value)) & 255); 81 | 82 | return 8; 83 | } 84 | 85 | size_t 86 | uint64_get_be (const byte* input, uint64_t *value) { 87 | *value = ((uint64_t) (input [0]) << 56) 88 | + ((uint64_t) (input [1]) << 48) 89 | + ((uint64_t) (input [2]) << 40) 90 | + ((uint64_t) (input [3]) << 32) 91 | + ((uint64_t) (input [4]) << 24) 92 | + ((uint64_t) (input [5]) << 16) 93 | + ((uint64_t) (input [6]) << 8) 94 | + (uint64_t) (input [7]); 95 | 96 | return 8; 97 | } 98 | 99 | bool str_start_with (const char *pre, const char *str) 100 | { 101 | size_t pre_len = strlen(pre); 102 | size_t str_len = strlen(str); 103 | 104 | return str_len < pre_len ? false : strncmp(pre, str, pre_len) == 0; 105 | } 106 | 107 | 108 | void 109 | dafka_util_test (bool verbose) { 110 | 111 | } -------------------------------------------------------------------------------- /src/dafka_util.h: -------------------------------------------------------------------------------- 1 | /* ========================================================================= 2 | dafka_util - class description 3 | 4 | Copyright (c) the Contributors as noted in the AUTHORS file. 5 | This file is part of CZMQ, the high-level C binding for 0MQ: 6 | http://czmq.zeromq.org. 7 | 8 | This Source Code Form is subject to the terms of the Mozilla Public 9 | License, v. 2.0. If a copy of the MPL was not distributed with this 10 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | ========================================================================= 12 | */ 13 | 14 | #ifndef DAFKA_UTIL_H_INCLUDED 15 | #define DAFKA_UTIL_H_INCLUDED 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | DAFKA_PRIVATE char * 22 | generate_address (); 23 | 24 | DAFKA_PRIVATE void 25 | uint64_destroy (void **item_p); 26 | 27 | DAFKA_PRIVATE void * 28 | uint64_dup (const void *item); 29 | 30 | DAFKA_PRIVATE int 31 | uint64_cmp (const void *item1, const void *item2); 32 | 33 | DAFKA_PRIVATE size_t 34 | uint64_hash (const void *item); 35 | 36 | DAFKA_PRIVATE size_t 37 | uint64_put_be (byte* output, uint64_t value); 38 | 39 | DAFKA_PRIVATE size_t 40 | uint64_get_be (const byte* input, uint64_t *value); 41 | 42 | 43 | DAFKA_PRIVATE bool 44 | str_start_with (const char *pre, const char *str); 45 | 46 | DAFKA_PRIVATE void 47 | dafka_util_test (bool verbose); 48 | 49 | 50 | 51 | // @end 52 | 53 | #ifdef __cplusplus 54 | } 55 | #endif 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/libdafka.pc.in: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 3 | # Read the zproject/README.md for information about making permanent changes. # 4 | ################################################################################ 5 | 6 | prefix=@prefix@ 7 | exec_prefix=@exec_prefix@ 8 | libdir=@libdir@ 9 | includedir=@includedir@ 10 | 11 | Name: libdafka 12 | Description: ZeroMQ messaging middleware 13 | Version: @VERSION@ 14 | 15 | Requires.private: @pkg_config_names_private@ 16 | Libs: -L${libdir} -ldafka 17 | Cflags: -I${includedir} @pkg_config_defines@ 18 | Libs.private: @pkg_config_libs_private@ 19 | 20 | ################################################################################ 21 | # THIS FILE IS 100% GENERATED BY ZPROJECT; DO NOT EDIT EXCEPT EXPERIMENTALLY # 22 | # Read the zproject/README.md for information about making permanent changes. # 23 | ################################################################################ 24 | -------------------------------------------------------------------------------- /src/selftest-ro/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zeromq/dafka/c2dc88d0698ce157562378e8386060709e861a96/src/selftest-ro/.gitkeep --------------------------------------------------------------------------------