├── .gitignore ├── .jshintrc ├── .npmignore ├── .travis.yml ├── Gruntfile.js ├── LICENSE ├── README.md ├── build ├── aliases.js ├── copy.json ├── env.json ├── instrument.json ├── jshint.json ├── makeReport.json ├── mochaTest.json └── storeCoverage.json ├── index.js ├── java ├── lib │ ├── activation-1.1.jar │ ├── commons-lang-2.6.jar │ ├── joda-time-1.6.jar │ ├── jopt-simple-3.2.jar │ ├── kafka-clients-0.8.2.2.jar │ ├── kafka_2.9.2-0.8.2.2.jar │ ├── log4j-1.2.15.jar │ ├── lz4-1.2.0.jar │ ├── metrics-core-2.2.0.jar │ ├── scala-library-2.9.2.jar │ ├── slf4j-api-1.7.6.jar │ ├── slf4j-log4j12-1.7.6.jar │ ├── snappy-java-1.1.1.7.jar │ ├── zkclient-0.3.jar │ └── zookeeper-3.4.6.jar ├── resources │ └── log4j.properties └── src │ └── com │ └── liveperson │ └── kafka │ ├── consumer │ ├── ConsumerThread.java │ ├── MultiThreadHLConsumer.java │ └── ThreadExceptionListener.java │ └── producer │ ├── BaseProducer.java │ ├── BinaryProducer.java │ ├── SendResult.java │ └── StringProducer.java ├── lib ├── binaryProducer.js ├── hlConsumer.js ├── producer.js ├── protocol.js ├── stringProducer.js └── util │ └── javaInit.js ├── package.json ├── scripts └── postinstall.js └── test └── js ├── protocol_test.js └── util └── require_helper.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 4 | 5 | *.iml 6 | 7 | ## Directory-based project format: 8 | .idea/ 9 | # if you remove the above rule, at least ignore the following: 10 | 11 | # User-specific stuff: 12 | # .idea/workspace.xml 13 | # .idea/tasks.xml 14 | # .idea/dictionaries 15 | 16 | # Sensitive or high-churn files: 17 | # .idea/dataSources.ids 18 | # .idea/dataSources.xml 19 | # .idea/sqlDataSources.xml 20 | # .idea/dynamic.xml 21 | # .idea/uiDesigner.xml 22 | 23 | # Gradle: 24 | # .idea/gradle.xml 25 | # .idea/libraries 26 | 27 | # Mongo Explorer plugin: 28 | # .idea/mongoSettings.xml 29 | 30 | ## File-based project format: 31 | *.ipr 32 | *.iws 33 | 34 | ## Plugin-specific files: 35 | 36 | # IntelliJ 37 | out/ 38 | 39 | # mpeltonen/sbt-idea plugin 40 | .idea_modules/ 41 | 42 | # JIRA plugin 43 | atlassian-ide-plugin.xml 44 | 45 | # Crashlytics plugin (for Android Studio and IntelliJ) 46 | com_crashlytics_export_strings.xml 47 | crashlytics.properties 48 | crashlytics-build.properties 49 | 50 | .DS_Store 51 | skeletonjs/node_modules 52 | skeletonjs/npm-debug.log 53 | skeletonjs/dist 54 | 55 | node_modules 56 | test/coverage 57 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "curly":true, 3 | "eqeqeq":true, 4 | "undef":true, 5 | "unused":false, 6 | "eqnull":true, 7 | "noarg":true, 8 | "nonew":false, 9 | "esversion": 6, 10 | "node":true, 11 | "devel":false, 12 | "strict": false, 13 | "globalstrict": true, 14 | "globals":{ 15 | "module": true, 16 | "__dirname":true, 17 | "exports": true, 18 | "require":true 19 | } 20 | } -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.log 3 | node_modules 4 | *.xml 5 | *.pom 6 | *.iml 7 | pom.xml.releaseBackup -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4" 4 | env: 5 | - CXX=g++-4.8 6 | addons: 7 | apt: 8 | sources: 9 | - ubuntu-toolchain-r-test 10 | packages: 11 | - g++-4.8 12 | before_install: npm install -g grunt-cli 13 | exclude_paths: 14 | - "gruntfile.js" 15 | - "build/*" 16 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function(grunt) { 2 | var path = require('path'); 3 | 4 | require('time-grunt')(grunt); 5 | require('load-grunt-config')(grunt, { 6 | // path to task.js files, defaults to grunt dir 7 | configPath: path.join(process.cwd(), 'build'), 8 | 9 | // auto grunt.initConfig 10 | init: true, 11 | 12 | // data passed into config. Can use with <%= test %> 13 | data: { 14 | test: false 15 | }, 16 | 17 | // can optionally pass options to load-grunt-tasks. 18 | // If you set to false, it will disable auto loading tasks. 19 | loadGruntTasks: { 20 | pattern: ['grunt-*'], 21 | config: require('./package.json'), 22 | scope: 'devDependencies' 23 | }, 24 | 25 | //can post process config object before it gets passed to grunt 26 | postProcess: function(config) {}, 27 | 28 | //allows to manipulate the config object before it gets merged with the data object 29 | preMerge: function(config, data) {} 30 | }); 31 | 32 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 liveperson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | kafka-java-bridge 2 | ================= 3 | ## for mirroring tests2 4 | [![Built with Grunt](https://cdn.gruntjs.com/builtwith.svg)](http://gruntjs.com/) 5 | [![Build Status](https://travis-ci.org/LivePersonInc/kafka-java-bridge.svg)](https://travis-ci.org/LivePersonInc/kafka-java-bridge) 6 | [![npm version](https://badge.fury.io/js/kafka-java-bridge.svg)](http://badge.fury.io/js/kafka-java-bridge) 7 | [![Dependency Status](https://david-dm.org/LivePersonInc/kafka-java-bridge.svg?theme=shields.io)](https://david-dm.org/LivePersonInc/kafka-java-bridge) 8 | [![devDependency Status](https://david-dm.org/LivePersonInc/kafka-java-bridge/dev-status.svg?theme=shields.io)](https://david-dm.org/LivePersonInc/kafka-java-bridge#info=devDependencies) 9 | [![npm downloads](https://img.shields.io/npm/dm/kafka-java-bridge.svg)](https://img.shields.io/npm/dm/kafka-java-bridge.svg) 10 | [![NPM](https://nodei.co/npm/kafka-java-bridge.png)](https://nodei.co/npm/kafka-java-bridge/) 11 | [![license](https://img.shields.io/npm/l/kafka-java-bridge.svg)](LICENSE) 12 | 13 | Nodejs wrapper for the [JAVA kafka 0.8 client API](http://kafka.apache.org/082/documentation.html). 14 | 15 | 16 | * [Motivation](#motivation) 17 | * [Installation](#installation) 18 | * [Example](#example) 19 | * [Performance and stability](#performance-and-stability) 20 | * [API](#api) 21 | * [Adding your own jars to classpath](#adding-your-own-jars-to-classpath) 22 | * [Java Tier Logging](#java-tier-logging) 23 | * [Troubleshooting](#troubleshooting) 24 | * [Sources](#sources) 25 | * [License](#license) 26 | 27 | Motivation 28 | ========== 29 | 30 | The need to have a production quality kafka0.8 client implementation in Nodejs. Please see: 31 | * [Performance and stability](#performance-and-stability) 32 | for detailed information. 33 | 34 | Installation 35 | ============ 36 | 37 | 1. Make sure you have java v7 or higher installed 38 | 2. Run `npm install kafka-java-bridge` 39 | 40 | Consumer Example 41 | ============ 42 | ```javascript 43 | 44 | var HLConsumer = require("kafka-java-bridge").HLConsumer; 45 | 46 | var consumerOptions = { 47 | zookeeperUrl: "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka", 48 | groupId: "example-consumer-group-id", 49 | topics: ["example-topic1","example-topic2"], 50 | getMetadata: true 51 | }; 52 | 53 | var consumer = new HLConsumer(consumerOptions); 54 | 55 | consumer.start(function (err) { 56 | if (err) { 57 | console.log("Error occurred when starting consumer. err:", err); 58 | } else { 59 | console.log("Started consumer successfully"); 60 | } 61 | }); 62 | 63 | consumer.on("message", function (msg, metadata) { 64 | console.log("On message. message:", msg); 65 | console.log("On message. metadata:", JSON.stringify(metadata)); 66 | }); 67 | 68 | consumer.on("error", function (err) { 69 | console.log("On error. err:", err); 70 | }); 71 | 72 | process.on('SIGINT', function() { 73 | consumer.stop(function(){ 74 | console.log("consumer stopped"); 75 | // Timeout to allow logs to print 76 | setTimeout(function(){ 77 | process.exit(); 78 | } , 300); 79 | }); 80 | }); 81 | 82 | ``` 83 | 84 | Producer Example 85 | ============ 86 | ```javascript 87 | var StringProducer = require('kafka-java-bridge').StringProducer; 88 | var BinaryProducer = require('kafka-java-bridge').BinaryProducer; 89 | 90 | var stringProducer = new StringProducer({bootstrapServers: "broker1:9092, broker2:9092"}); 91 | var binaryProducer = new BinaryProducer({zookeeperUrl: "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka"}); 92 | 93 | const buf = new Buffer([0x0, 0x1, 0x2, 0x3, 0x4]); 94 | binaryProducer.send("myBinaryTopic", buf, function(err, msgMetadata){ 95 | console.log('send msg cb. err = ' + err + '. metadata = ' + JSON.stringify(msgMetadata)); 96 | }); 97 | stringProducer.send("myStringTopic", "testString", function(err, msgMetadata){ 98 | console.log('send msg cb. err = ' + err + '. metadata = ' + JSON.stringify(msgMetadata)); 99 | }); 100 | 101 | process.on('SIGINT', function() { 102 | stringProducer.close(function(err){ 103 | binaryProducer.close(function(err) { 104 | process.exit(); 105 | }); 106 | }); 107 | }); 108 | ``` 109 | Performance and stability 110 | ============ 111 | 112 | ### Performance 113 | 114 | Libraries compared: 115 | 116 | * kafka-java-bridge , this package. 117 | * [kafka-node](https://github.com/SOHU-Co/kafka-node), available High Level Consumer for kafka0.8. 118 | 119 | 1. We show below representative cpu consumption (lower is better) for processing same amount of messages per second(~11K). 120 | 121 | ![image 1](https://cloud.githubusercontent.com/assets/3764373/15296980/8a3ac996-1ba0-11e6-92d2-0e9c69e14d2b.png). 122 | 123 | |*Library name* |*CPU% average*| 124 | |:----------------:|:------------:| 125 | |kafka-java-bridge |11.76 | 126 | |kafka-node |73 | 127 | 128 | 129 | 2. Consumer comparision (number of messages). Tested with 16GB Ram, 4 core machine on Amazon AWS EC2 Instance. (Metircs measured with Newrelic) 130 | 131 | |*Library name* |*Rpm Avg*|*Network Avg*|*Cpu/System Avg*| 132 | |:----------------:|:------------:|:------------:|:------------:| 133 | |kafka-java-bridge |947K |300 Mb/s |6.2% | 134 | |kafka-node |87.5K |75 Mb/s |11.2% | 135 | 136 | ![Kakfa-Java-Bridge RPM](https://s5.postimg.org/qqmxdmpif/6929a69b-bb0f-4191-bb2d-df52cf62e548.jpg) 137 | ![Kafka-Node RPM](https://s5.postimg.org/437o7h9yf/cc9a8b40-5a71-4a3c-94f4-a9ceb097a01c.jpg) 138 | 139 | ### Stability 140 | 141 | Kafka-java-bridge wraps [Confluent](http://www.confluent.io/)'s official Java High Level Consumer. 142 | 143 | While testing [kafka-node](https://github.com/SOHU-Co/kafka-node) we encountered multiple [issues](https://github.com/SOHU-Co/kafka-node/issues) such as: 144 | 145 | * [Duplicate messages](https://github.com/SOHU-Co/kafka-node/issues/218) 146 | * [Rebalancing errors](https://github.com/SOHU-Co/kafka-node/issues/341) 147 | 148 | Those issues along side with the inadequate performance results where the trigger for developing this library. 149 | 150 | 151 | 152 | API 153 | ============ 154 | ### HLConsumer(options) 155 | 156 | Consumer object allows messages fetching from kafka topic. 157 | Each consumer can consume messages from multiple topics. 158 | 159 | ```javascript 160 | 161 | var consumerOptions = { 162 | zookeeperUrl: "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka", 163 | groupId: "example-consumer-group-id", 164 | topic: "example-topic", 165 | serverPort: 3042,// Optional 166 | threadCount: 1,// Optional 167 | properties: {"rebalance.max.retries": "3"}// Optional 168 | }; 169 | 170 | ``` 171 | 172 | 173 | | *Option name* |*Mandatory* |*Type* |*Default value*|*Description*| 174 | |:--------------|:-------------:|:--------|:-------------:|:------------| 175 | | zookeeperUrl | Yes |`String` |`undefined` |Zookeeper connection string.| 176 | | groupId | Yes |`String` |`undefined` |Kafka consumer groupId. From [kafka documentation](http://kafka.apache.org/082/documentation.html#consumerconfigs): groupId is a string that uniquely identifies the group of consumer processes to which this consumer belongs. By setting the same group id multiple processes indicate that they are all part of the same consumer group.| 177 | | topic | No |`String` |`undefined` |Kafka topic name.| 178 | | getMetadata | No |`boolean`|false |If true, message metadata(topic, partition, offset) will be provided with each message. Use false for better performance.| 179 | | topics | Yes |`Array of String` |`undefined` |Kafka topics names array.| 180 | | serverPort | No |`Number` |`3042` |Internal server port used to transfer the messages from the java thread to the node js thread.| 181 | |threadCount | No |`Number` |`1` |The threading model revolves around the number of partitions in your topic and there are some very specific rules. For More information: [kafka consumer groups](https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example)| | getMetadata | No |`Boolean` |`false` |Get message metadata (contains topic, partition and offset ).| 182 | |properties | No |`Object` |`undefined` |Properties names can be found in the following table: [high level consumer properties](http://kafka.apache.org/082/documentation.html#consumerconfigs).| 183 | 184 | 185 | ### Events emitted by the HLConsumer: 186 | 187 | * message: this event is emitted when a message is consumed from kafka. 188 | * error: this event is emitted when an error occurs while consuming messages. 189 | 190 | ### hlConsumer.start(cb) 191 | Start consumer messages from kafka topic. 192 | 193 | cb - callback is called when the consumer is started. 194 | 195 | If callback was called with err it means consumer failed to start. 196 | 197 | ### hlConsumer.stop(cb) 198 | Stop consuming messages. 199 | 200 | cb - callback is called when the consumer is stopped. 201 | 202 | **message/error events can still be emitted until stop callback is called.** 203 | 204 | 205 | ## StringProducer(options) / BinaryProducer(options) 206 | 207 | Producer object produces messages to kafka. With each message topic is specified so one producer can produce messages to multiple topics. 208 | 209 | **StringProducer should be used to send string messages.** 210 | **BinaryProducer should be used to send binary messages.** 211 | 212 | ```javascript 213 | 214 | var producerOptions = { 215 | zookeeperUrl: "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka", 216 | properties: {"client.id": "kafka-java-bridge"}// Optional 217 | }; 218 | OR 219 | var producerOptions = { 220 | bootstrapServers: "kafka:2181,kafka2:2181,kafka3:2181/kafka", 221 | properties: {"client.id": "kafka-java-bridge"}// Optional 222 | }; 223 | 224 | ``` 225 | 226 | 227 | | *Option name* |*Mandatory* |*Type* |*Default value*|*Description*| 228 | |:--------------|:-------------:|:--------|:-------------:|:------------| 229 | | bootstrapServers| NO |`String` |`undefined` |Kafka broker connection string.| 230 | | zookeeperUrl | No |`String` |`undefined` |Zookeeper connection string. If provided, broker list will be retrieved from standard path.| 231 | | properties | No |`Object` |`undefined` |Properties names can be found in the following table: [high level producer properties](http://kafka.apache.org/documentation.html#producerconfigs).| 232 | 233 | ### producer.send(topic, msg, cb) 234 | 235 | topic - target topic name `String`. 236 | 237 | msg - message to be sent to kafka `String` or `Buffer`. 238 | 239 | cb - callback is called when message is sent. with err in case of failure or msg metadata in case of success. 240 | 241 | ### producer.sendWithKey(topic, msg, key, cb) 242 | 243 | topic - target topic name `String`. 244 | 245 | msg - message to be sent to kafka `String` or `Buffer`. 246 | 247 | key - kafka message key `String` or `Buffer`. 248 | 249 | cb - callback is called when message is sent. with err in case of failure or msg metadata in case of success. 250 | 251 | ### producer.sendWithKeyAndPartition(topic, msg, key, partition, cb) 252 | 253 | topic - target topic name `String`. 254 | 255 | msg - message to be sent to kafka `String` or `Buffer`. 256 | 257 | key - kafka message key `String` or `Buffer`. 258 | 259 | partition - target partition `Integer`. 260 | 261 | cb - callback is called when message is sent. with err in case of failure or msg metadata in case of success. 262 | 263 | 264 | Adding Your Own Jars To Classpath 265 | ================================ 266 | 267 | If you wish to add jars to the classpath, it can be done by placing them at: 268 | 269 | {app root path}/kafka-java-bridge/java/lib/yourjar.jar 270 | 271 | 272 | Java Tier Logging 273 | ================= 274 | 275 | By default, underlying java tier logging is disabled. 276 | 277 | If you wish to enable java tier logging you can place your own log4j.properties file at: 278 | 279 | {app root path}/kafka-java-bridge/log4j/log4j.properties 280 | 281 | Troubleshooting 282 | =============== 283 | 284 | In case of installation failure, you may want to take a look at our dependency java npm [installation](https://www.npmjs.com/package/java#installation) and [troubleshooting](https://www.npmjs.com/package/java#troubleshooting) sections. 285 | 286 | If you are working on a windows machine, you may want to look at [windows-build-tools](https://www.npmjs.com/package/windows-build-tools) for native code compilation issues. 287 | 288 | Sources 289 | ======= 290 | 291 | * [kafka consumer groups](https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example). 292 | * [Java book example reference](https://github.com/bkimminich/apache-kafka-book-examples). 293 | 294 | License 295 | ======= 296 | MIT 297 | -------------------------------------------------------------------------------- /build/aliases.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt, options) { 2 | 3 | var tasks = ['jshint', 'env', 'instrument', 'copy', 'mochaTest', 'storeCoverage', 'makeReport']; 4 | return { 5 | 'tasks': ['availabletasks'], 6 | 'default': tasks, 7 | 'test': [ 8 | 'env', 9 | 'instrument', 10 | 'mochaTest', 11 | 'storeCoverage', 12 | 'makeReport' 13 | ] 14 | }; 15 | }; 16 | -------------------------------------------------------------------------------- /build/copy.json: -------------------------------------------------------------------------------- 1 | { 2 | "all": { 3 | "files": [ 4 | { 5 | "expand": true, 6 | "src": [ 7 | "config/*" 8 | ], 9 | "dest": "test/coverage/instrument/", 10 | "filter": "isFile" 11 | } 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /build/env.json: -------------------------------------------------------------------------------- 1 | { 2 | "coverage": { 3 | "APP_DIR_FOR_CODE_COVERAGE": "../../coverage/instrument/lib/" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /build/instrument.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": "lib/**/*.js", 3 | "options": { 4 | "lazy": true, 5 | "basePath": "test/coverage/instrument/" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /build/jshint.json: -------------------------------------------------------------------------------- 1 | { 2 | "options": { 3 | "jshintrc": ".jshintrc", 4 | "node": true 5 | }, 6 | "uses_defaults": ["lib/*.js","lib/**/*.js", "index.js"] 7 | } 8 | -------------------------------------------------------------------------------- /build/makeReport.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "test/coverage/reports/**/*.json", 3 | "options": { 4 | "type": "lcov", 5 | "dir": "test/coverage/reports", 6 | "print": "detail" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /build/mochaTest.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": { 3 | "options": { 4 | "timeout": 10000, 5 | "reporter": "spec", 6 | "quiet": false 7 | }, 8 | "src": ["test/js/*.js"] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /build/storeCoverage.json: -------------------------------------------------------------------------------- 1 | { 2 | "options": { 3 | "dir": "test/coverage/reports" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | exports.HLConsumer = require('./lib/hlConsumer'); 2 | exports.StringProducer = require('./lib/stringProducer'); 3 | exports.BinaryProducer = require('./lib/binaryProducer'); -------------------------------------------------------------------------------- /java/lib/activation-1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/activation-1.1.jar -------------------------------------------------------------------------------- /java/lib/commons-lang-2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/commons-lang-2.6.jar -------------------------------------------------------------------------------- /java/lib/joda-time-1.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/joda-time-1.6.jar -------------------------------------------------------------------------------- /java/lib/jopt-simple-3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/jopt-simple-3.2.jar -------------------------------------------------------------------------------- /java/lib/kafka-clients-0.8.2.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/kafka-clients-0.8.2.2.jar -------------------------------------------------------------------------------- /java/lib/kafka_2.9.2-0.8.2.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/kafka_2.9.2-0.8.2.2.jar -------------------------------------------------------------------------------- /java/lib/log4j-1.2.15.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/log4j-1.2.15.jar -------------------------------------------------------------------------------- /java/lib/lz4-1.2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/lz4-1.2.0.jar -------------------------------------------------------------------------------- /java/lib/metrics-core-2.2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/metrics-core-2.2.0.jar -------------------------------------------------------------------------------- /java/lib/scala-library-2.9.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/scala-library-2.9.2.jar -------------------------------------------------------------------------------- /java/lib/slf4j-api-1.7.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/slf4j-api-1.7.6.jar -------------------------------------------------------------------------------- /java/lib/slf4j-log4j12-1.7.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/slf4j-log4j12-1.7.6.jar -------------------------------------------------------------------------------- /java/lib/snappy-java-1.1.1.7.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/snappy-java-1.1.1.7.jar -------------------------------------------------------------------------------- /java/lib/zkclient-0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/zkclient-0.3.jar -------------------------------------------------------------------------------- /java/lib/zookeeper-3.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LivePersonInc/kafka-java-bridge/b8c50606873e0f7df1e61f689143c7c836637bb8/java/lib/zookeeper-3.4.6.jar -------------------------------------------------------------------------------- /java/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to DEBUG and its only appender to A1. 2 | log4j.rootLogger=OFF, A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n 10 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/consumer/ConsumerThread.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.consumer; 2 | 3 | import kafka.consumer.ConsumerIterator; 4 | import kafka.consumer.KafkaStream; 5 | import kafka.message.MessageAndMetadata; 6 | 7 | import java.io.BufferedOutputStream; 8 | import java.net.Socket; 9 | import java.nio.ByteBuffer; 10 | 11 | /** 12 | * Created by elio on 4/17/2016. 13 | */ 14 | public class ConsumerThread implements Runnable { 15 | 16 | private KafkaStream stream; 17 | private int threadNumber; 18 | private ThreadExceptionListener exceptionListener; 19 | private int consumerServerPort; 20 | private boolean getMetadata; 21 | 22 | public ConsumerThread(KafkaStream stream, int threadNumber, int consumerServerPort, boolean getMetadata, ThreadExceptionListener exceptionListener) { 23 | this.threadNumber = threadNumber; 24 | this.stream = stream; 25 | this.exceptionListener = exceptionListener; 26 | this.consumerServerPort = consumerServerPort; 27 | this.getMetadata = getMetadata; 28 | } 29 | 30 | @Override 31 | public void run() { 32 | 33 | Socket clientSocket = null; 34 | try{ 35 | ConsumerIterator it = stream.iterator(); 36 | 37 | clientSocket = new Socket("localhost", consumerServerPort); 38 | BufferedOutputStream outToServer = new BufferedOutputStream(clientSocket.getOutputStream(), 8192); 39 | while (it.hasNext()) { 40 | byte[] msg; 41 | if(getMetadata){ 42 | MessageAndMetadata messageAndMetadata = it.next(); 43 | msg = messageAndMetadata.message(); 44 | byte[] topic = messageAndMetadata.topic().getBytes(); 45 | long offset = messageAndMetadata.offset(); 46 | int partition = messageAndMetadata.partition(); 47 | outToServer.write(ByteBuffer.allocate(4).putInt(msg.length + 4 + 8 + 4 + topic.length).array()); 48 | outToServer.write(ByteBuffer.allocate(4).putInt(topic.length).array()); 49 | outToServer.write(ByteBuffer.allocate(8).putLong(offset).array()); 50 | outToServer.write(ByteBuffer.allocate(4).putInt(partition).array()); 51 | outToServer.write(topic); 52 | }else{ 53 | msg = it.next().message(); 54 | outToServer.write(ByteBuffer.allocate(4).putInt(msg.length).array()); 55 | } 56 | outToServer.write(msg); 57 | outToServer.flush(); 58 | } 59 | 60 | clientSocket.shutdownOutput(); 61 | }catch(Exception ex){ 62 | exceptionListener.onThreadException(threadNumber, ex); 63 | if(clientSocket != null){ 64 | try{ 65 | clientSocket.shutdownOutput(); 66 | }catch(Exception e){ 67 | 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/consumer/MultiThreadHLConsumer.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.consumer; 2 | 3 | import kafka.consumer.Consumer; 4 | import kafka.consumer.ConsumerConfig; 5 | import kafka.consumer.KafkaStream; 6 | import kafka.javaapi.consumer.ConsumerConnector; 7 | 8 | import java.util.*; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.TimeUnit; 12 | import java.util.concurrent.atomic.AtomicBoolean; 13 | 14 | 15 | /** 16 | * Created by elio on 4/17/2016. 17 | */ 18 | public class MultiThreadHLConsumer { 19 | 20 | private final static int TERMINATION_SECS = 5; 21 | 22 | private ExecutorService executor; 23 | private ConsumerConnector consumer; 24 | private String[] topics; 25 | private int threadCount; 26 | private ThreadExceptionListener exceptionListener; 27 | private int consumerServerPort; 28 | private Properties properties; 29 | private boolean getMetadata; 30 | 31 | public MultiThreadHLConsumer(String zookeeper, String groupId, String[] topics, Properties clientProps, int consumerServerPort, int threadCount, boolean getMetadata, ThreadExceptionListener exceptionListener){ 32 | properties = new Properties(); 33 | properties.put("zookeeper.connect", zookeeper); 34 | properties.put("group.id", groupId); 35 | if(clientProps != null){ 36 | properties.putAll(clientProps); 37 | } 38 | 39 | this.topics = topics; 40 | this.threadCount = threadCount; 41 | this.exceptionListener = exceptionListener; 42 | this.consumerServerPort = consumerServerPort; 43 | this.getMetadata = getMetadata; 44 | } 45 | 46 | public void start(){ 47 | consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(properties)); 48 | Map topicCount = new HashMap(); 49 | for(String topic: topics) { 50 | topicCount.put(topic, threadCount); 51 | } 52 | 53 | Map>> consumerStreams = consumer.createMessageStreams(topicCount); 54 | executor = Executors.newFixedThreadPool(threadCount * this.topics.length); 55 | 56 | try{ 57 | for(String topic: topics){ 58 | int threadNumber = 0; 59 | List> streams = consumerStreams.get(topic); 60 | 61 | for (final KafkaStream stream : streams) { 62 | executor.submit(new ConsumerThread(stream, threadNumber, consumerServerPort, getMetadata, exceptionListener)); 63 | threadNumber++; 64 | } 65 | } 66 | }catch(Exception ex){ 67 | stop(); 68 | throw ex; 69 | } 70 | } 71 | 72 | public void stop(){ 73 | if (consumer != null) { 74 | consumer.shutdown(); 75 | } 76 | 77 | if(executor != null) { 78 | executor.shutdown(); 79 | try { 80 | if(!executor.awaitTermination(TERMINATION_SECS, TimeUnit.SECONDS)){ 81 | executor.shutdownNow(); 82 | } 83 | } catch (InterruptedException e) { 84 | executor.shutdownNow(); 85 | } 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/consumer/ThreadExceptionListener.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.consumer; 2 | 3 | /** 4 | * Created by elio on 4/17/2016. 5 | */ 6 | public interface ThreadExceptionListener { 7 | void onThreadException(int threadNumber, Exception ex); 8 | } 9 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/producer/BaseProducer.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.producer; 2 | 3 | import kafka.cluster.Broker; 4 | import kafka.utils.ZKStringSerializer$; 5 | import kafka.utils.ZkUtils; 6 | import org.I0Itec.zkclient.ZkClient; 7 | import org.apache.commons.lang.StringUtils; 8 | import org.apache.commons.lang.exception.ExceptionUtils; 9 | import org.apache.kafka.clients.producer.Callback; 10 | import org.apache.kafka.clients.producer.KafkaProducer; 11 | import org.apache.kafka.clients.producer.ProducerRecord; 12 | import org.apache.kafka.clients.producer.RecordMetadata; 13 | import scala.collection.JavaConversions; 14 | import scala.collection.Seq; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Properties; 19 | import java.util.concurrent.ExecutionException; 20 | 21 | /** 22 | * Created by elio on 1/9/17. 23 | */ 24 | public abstract class BaseProducer { 25 | private KafkaProducer kafkaProducer; 26 | private final Object syncObj; 27 | private List results; 28 | 29 | protected BaseProducer(Properties props){ 30 | 31 | syncObj = new Object(); 32 | results = new ArrayList(); 33 | 34 | if(props.containsKey("zookeeper.url")){ 35 | props.put("bootstrap.servers", getBrokers(props.getProperty("zookeeper.url"))); 36 | } 37 | 38 | kafkaProducer = createProducer(props); 39 | } 40 | 41 | protected abstract KafkaProducer createProducer(Properties props); 42 | public abstract void sendWithKey(final String msgId, String topic, String value, String key); 43 | public abstract void sendWithKeyAndPartition(final String msgId, String topic, String value, String key, Integer partition); 44 | public abstract void send(final String msgId, String topic, String value); 45 | 46 | protected void send(final String msgId, ProducerRecord producerRecord){ 47 | kafkaProducer.send(producerRecord, new Callback() { 48 | @Override 49 | public void onCompletion(RecordMetadata recordMetadata, Exception e) { 50 | if(msgId == null){ 51 | return; 52 | } 53 | SendResult sendResult; 54 | if(e != null){ 55 | sendResult = new SendResult(msgId, e.getMessage() + "\n" + ExceptionUtils.getFullStackTrace(e)); 56 | }else{ 57 | sendResult = new SendResult(msgId, recordMetadata.partition(), String.valueOf(recordMetadata.offset())); 58 | } 59 | onResult(sendResult); 60 | } 61 | }); 62 | } 63 | 64 | private void onResult(SendResult sendResult){ 65 | synchronized (syncObj){ 66 | results.add(sendResult); 67 | syncObj.notify(); 68 | } 69 | } 70 | 71 | public SendResult[] getResults() throws InterruptedException { 72 | SendResult[] retVal; 73 | synchronized (syncObj){ 74 | if(results.isEmpty()){ 75 | syncObj.wait(); 76 | } 77 | retVal = new SendResult[results.size()]; 78 | retVal = results.toArray(retVal); 79 | results.clear(); 80 | } 81 | return retVal; 82 | } 83 | 84 | public void close(){ 85 | kafkaProducer.close(); 86 | synchronized (syncObj){ 87 | syncObj.notify(); 88 | } 89 | } 90 | 91 | private String getBrokers(String zkHost){ 92 | ZkClient zkclient = new ZkClient(zkHost, 3_000, 3_000, ZKStringSerializer$.MODULE$); 93 | List brokerHosts = new ArrayList(); 94 | Seq allBrokersInCluster = ZkUtils.getAllBrokersInCluster(zkclient); 95 | List brokersList = JavaConversions.asJavaList(allBrokersInCluster); 96 | for(Broker broker: brokersList){ 97 | brokerHosts.add(broker.connectionString()); 98 | } 99 | zkclient.close(); 100 | return StringUtils.join(brokerHosts, ','); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/producer/BinaryProducer.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.producer; 2 | 3 | 4 | import org.apache.kafka.clients.producer.KafkaProducer; 5 | import org.apache.kafka.clients.producer.ProducerRecord; 6 | 7 | import javax.xml.bind.DatatypeConverter; 8 | import java.util.*; 9 | 10 | /** 11 | * Created by elio on 1/5/17. 12 | */ 13 | public class BinaryProducer extends BaseProducer { 14 | 15 | 16 | public BinaryProducer(Properties props){ 17 | super(props); 18 | } 19 | 20 | @Override 21 | protected KafkaProducer createProducer(Properties props) { 22 | return new KafkaProducer(props); 23 | } 24 | 25 | @Override 26 | public void sendWithKey(String msgId, String topic, String value, String key) { 27 | ProducerRecord producerRecord = new ProducerRecord(topic, DatatypeConverter.parseBase64Binary(value), DatatypeConverter.parseBase64Binary(key)); 28 | send(msgId, producerRecord); 29 | } 30 | 31 | @Override 32 | public void sendWithKeyAndPartition(String msgId, String topic, String value, String key, Integer partition) { 33 | ProducerRecord producerRecord = new ProducerRecord(topic, partition, DatatypeConverter.parseBase64Binary(value), DatatypeConverter.parseBase64Binary(key)); 34 | send(msgId, producerRecord); 35 | } 36 | 37 | @Override 38 | public void send(String msgId, String topic, String value) { 39 | ProducerRecord producerRecord = new ProducerRecord(topic, DatatypeConverter.parseBase64Binary(value)); 40 | send(msgId, producerRecord); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/producer/SendResult.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.producer; 2 | 3 | 4 | /** 5 | * Created by elio on 1/8/17. 6 | */ 7 | public class SendResult { 8 | public String msgId; 9 | public int partition; 10 | public String offset; 11 | public String err; 12 | 13 | public SendResult(){ 14 | 15 | } 16 | 17 | public SendResult(String msgId, int partition, String offset) { 18 | this.msgId = msgId; 19 | this.partition = partition; 20 | this.offset = offset; 21 | } 22 | 23 | public SendResult(String msgId, String err) { 24 | this.msgId = msgId; 25 | this.err = err; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /java/src/com/liveperson/kafka/producer/StringProducer.java: -------------------------------------------------------------------------------- 1 | package com.liveperson.kafka.producer; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.ProducerRecord; 5 | 6 | import java.util.Properties; 7 | 8 | /** 9 | * Created by elio on 1/9/17. 10 | */ 11 | public class StringProducer extends BaseProducer { 12 | 13 | public StringProducer(Properties props) { 14 | super(props); 15 | } 16 | 17 | @Override 18 | protected KafkaProducer createProducer(Properties props) { 19 | return new KafkaProducer(props); 20 | } 21 | 22 | @Override 23 | public void sendWithKey(String msgId, String topic, String value, String key) { 24 | ProducerRecord producerRecord = new ProducerRecord(topic, value, key); 25 | send(msgId, producerRecord); 26 | } 27 | 28 | @Override 29 | public void sendWithKeyAndPartition(String msgId, String topic, String value, String key, Integer partition) { 30 | ProducerRecord producerRecord = new ProducerRecord(topic, partition, value, key); 31 | send(msgId, producerRecord); 32 | } 33 | 34 | @Override 35 | public void send(String msgId, String topic, String value) { 36 | ProducerRecord producerRecord = new ProducerRecord(topic, value); 37 | send(msgId, producerRecord); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lib/binaryProducer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elio on 1/9/17. 3 | */ 4 | const Producer = require('./producer'); 5 | 6 | class BinaryProducer extends Producer{ 7 | constructor(options) { 8 | super(options, "com.liveperson.kafka.producer.BinaryProducer", "org.apache.kafka.common.serialization.ByteArraySerializer"); 9 | } 10 | 11 | send(topic, msg, cb) { 12 | super.send(topic, msg.toString('base64'), cb); 13 | } 14 | 15 | sendWithKey(topic, msg, key, cb) { 16 | super.sendWithKey(topic, msg.toString('base64'), key.toString('base64'), cb); 17 | } 18 | 19 | sendWithKeyAndPartition(topic, msg, key, partition, cb) { 20 | super.sendWithKeyAndPartition(topic, msg.toString('base64'), key.toString('base64'), partition, cb); 21 | } 22 | } 23 | 24 | module.exports = BinaryProducer; 25 | -------------------------------------------------------------------------------- /lib/hlConsumer.js: -------------------------------------------------------------------------------- 1 | var events = require("events"); 2 | var util = require("util"); 3 | var javaInit = require("./util/javaInit.js"); 4 | var protocol = require('./protocol'); 5 | var java = javaInit.getJavaInstance(); 6 | var net = require("net"); 7 | 8 | var socketIdSeq = 0; 9 | 10 | function HLConsumer(options) { 11 | if (this instanceof HLConsumer === false) { 12 | return new HLConsumer(options); 13 | } 14 | 15 | this.zookeeperUrl = options.zookeeperUrl; 16 | this.groupId = options.groupId; 17 | this.topics = options.topics; 18 | this.properties = options.properties; 19 | this.getMetadata = options.getMetadata || false; 20 | this.serverPort = options.serverPort || 3042; 21 | this.threadCount = options.threadCount || 1; 22 | this.serverSockets = {}; 23 | 24 | if(!this.topics || this.topics.length === 0){ 25 | this.topics = [options.topic]; 26 | } 27 | 28 | var _boundCreateConsumer; 29 | var _boundStartConsumer; 30 | var _boundStartListener; 31 | 32 | _bindPrivateFunctions.call(this); 33 | 34 | this.start = function (cb) { 35 | if (this.hlConsumer) { 36 | _boundStartConsumer(cb); 37 | return; 38 | } 39 | 40 | _boundCreateConsumer(function (err) { 41 | if (err) { 42 | cb(err); 43 | } else { 44 | _boundStartConsumer(cb); 45 | } 46 | }); 47 | }; 48 | 49 | this.stop = function (cb) { 50 | this.hlConsumer.stop(function () { 51 | var serverClosed = false; 52 | this.server.close(function () { 53 | serverClosed = true; 54 | cb(); 55 | }.bind(this)); 56 | 57 | setTimeout(function () { 58 | if (serverClosed) { 59 | return; 60 | } 61 | for (var socketId in this.serverSockets) { 62 | if (!this.serverSockets.hasOwnProperty(socketId)) { 63 | continue; 64 | } 65 | var socket = this.serverSockets[socketId]; 66 | socket.destroy(); 67 | delete this.serverSockets[socketId]; 68 | } 69 | }.bind(this), 3000); 70 | }.bind(this)); 71 | }; 72 | function _bindPrivateFunctions() { 73 | _boundCreateConsumer = _createConsumer.bind(this); 74 | _boundStartConsumer = _startConsumer.bind(this); 75 | _boundStartListener = _startListener.bind(this); 76 | } 77 | 78 | function _startListener() { 79 | this.server = net.createServer(function (socket) { 80 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 81 | var socketId = socketIdSeq++; 82 | this.serverSockets[socketId] = socket; 83 | var msgCB = function(message, metadata){ 84 | this.emit('message', message, metadata); 85 | }.bind(this); 86 | socket.on("data", function (data) { 87 | var parsingContext = { 88 | offset: 0, 89 | currentMessage: currentMessage, 90 | data: data, 91 | onMsgCB: msgCB, 92 | parseMetadata: this.getMetadata 93 | }; 94 | 95 | protocol.parseData(parsingContext); 96 | }.bind(this)); 97 | 98 | socket.on("error", function (err) { 99 | this.emit("error", err); 100 | }.bind(this)); 101 | 102 | socket.on("close", function (data) { 103 | if (this.serverSockets[socketId]) { // Not coming from server close timeout socket destroy 104 | delete this.serverSockets[socketId]; 105 | } 106 | }.bind(this)); 107 | 108 | }.bind(this)); 109 | 110 | this.server.listen(this.serverPort, "127.0.0.1"); 111 | } 112 | 113 | function _startConsumer(cb) { 114 | _boundStartListener(); 115 | this.hlConsumer.start(function (err) { 116 | if (!err) { 117 | cb(err); 118 | return; 119 | } 120 | this.server.close(function () { 121 | cb(err); 122 | }); 123 | }.bind(this)); 124 | } 125 | 126 | function _createConsumer(cb) { 127 | 128 | var properties = java.newInstanceSync("java.util.Properties"); 129 | if (this.properties) { 130 | for (var prop in this.properties) { 131 | if (!this.properties.hasOwnProperty(prop)) { 132 | continue; 133 | } 134 | properties.putSync(prop, this.properties[prop]); 135 | } 136 | } 137 | 138 | var exceptionListener = java.newProxy("com.liveperson.kafka.consumer.ThreadExceptionListener", { 139 | onThreadException: function (threadNumber, exception) { 140 | this.emit("error", new Error("Exception in thread No\' " + threadNumber + ", message =>" + exception.cause.getMessageSync())); 141 | }.bind(this) 142 | }); 143 | java.newInstance("com.liveperson.kafka.consumer.MultiThreadHLConsumer", this.zookeeperUrl, this.groupId, this.topics, properties, this.serverPort, this.threadCount, this.getMetadata, exceptionListener, function (err, hlConsumer) { 144 | if (!err) { 145 | this.hlConsumer = hlConsumer; 146 | } 147 | cb(err); 148 | }.bind(this)); 149 | } 150 | } 151 | 152 | util.inherits(HLConsumer, events.EventEmitter); 153 | 154 | module.exports = HLConsumer; -------------------------------------------------------------------------------- /lib/producer.js: -------------------------------------------------------------------------------- 1 | var javaInit = require("./util/javaInit.js"); 2 | var java = javaInit.getJavaInstance(); 3 | 4 | 5 | class Producer{ 6 | constructor(options, producerClass, serializerClass){ 7 | this._createProducer(options, producerClass, serializerClass); 8 | } 9 | 10 | 11 | send(topic, msg, cb) { 12 | const msgId = this._addMsgToPending(topic, cb); 13 | this.producer.send(msgId, topic, msg, (err) => { 14 | if(err){ 15 | delete this.pendingResult[msgId]; 16 | if(cb){ 17 | cb(err); 18 | } 19 | } 20 | }); 21 | } 22 | 23 | sendWithKey(topic, msg, key, cb) { 24 | const msgId = this._addMsgToPending(topic, cb); 25 | this.producer.send(msgId, topic, msg, key, function(err){ 26 | if(err){ 27 | delete this.pendingResult[msgId]; 28 | if(cb){ 29 | cb(err); 30 | } 31 | } 32 | }.bind(this)); 33 | } 34 | 35 | sendWithKeyAndPartition(topic, msg, key, partition, cb) { 36 | const msgId = this._addMsgToPending(topic, cb); 37 | this.producer.send(msgId, topic, msg, key, partition, function(err){ 38 | if(err){ 39 | delete this.pendingResult[msgId]; 40 | if(cb){ 41 | cb(err); 42 | } 43 | } 44 | }.bind(this)); 45 | } 46 | 47 | close(cb) { 48 | this.producer.close(function(){ 49 | this.closed = true; 50 | if(cb){ 51 | cb(); 52 | } 53 | }.bind(this)); 54 | } 55 | 56 | _addMsgToPending(topic, cb){ 57 | if(!cb){ 58 | return null; 59 | } 60 | this.msgCount++; 61 | const msgId = this.msgCount.toString(); 62 | this.pendingResult[msgId] = {topic: topic, cb: cb}; 63 | return msgId; 64 | } 65 | 66 | _createProducer(options, producerClass, serializerClass){ 67 | this.msgCount = 0; 68 | this.pendingResult = {}; 69 | var properties = java.newInstanceSync("java.util.Properties"); 70 | properties.putSync("key.serializer", serializerClass); 71 | properties.putSync("value.serializer", serializerClass); 72 | if(options.bootstrapServers){ 73 | properties.putSync("bootstrap.servers", options.bootstrapServers); 74 | } 75 | 76 | if(options.zookeeperUrl){ 77 | properties.putSync("zookeeper.url", options.zookeeperUrl); 78 | } 79 | 80 | if (options.properties) { 81 | for (var prop in options.properties) { 82 | if (!options.properties.hasOwnProperty(prop)) { 83 | continue; 84 | } 85 | 86 | properties.putSync(prop, options.properties[prop]); 87 | } 88 | } 89 | 90 | this.producer = java.newInstanceSync(producerClass, properties); 91 | this._boundGetResults = this._getResults.bind(this); 92 | this._boundGetResultsJavaCB = this._getResultsJavaCB.bind(this); 93 | this._boundGetResults(); 94 | } 95 | 96 | _getResults() { 97 | if(!(this.closed && Object.keys(this.pendingResult).length === 0)){ 98 | this.producer.getResults(this._boundGetResultsJavaCB); 99 | } 100 | } 101 | 102 | _getResultsJavaCB(err, results){ 103 | if(err || !results || results.length === 0){ 104 | setTimeout(this._boundGetResults, 0); 105 | return; 106 | } 107 | 108 | for(let counter = 0; counter < results.length; counter++){ 109 | const result = results[counter]; 110 | const pendingResult = this.pendingResult[result.msgId]; 111 | if(!pendingResult){ 112 | continue; 113 | } 114 | 115 | if(!pendingResult.cb){ 116 | delete this.pendingResult[result.msgId]; 117 | continue; 118 | } 119 | 120 | if(result.err){ 121 | pendingResult.cb(new Error(result.err)); 122 | }else{ 123 | pendingResult.cb(null, {topic:pendingResult.topic, partition: result.partition, offset: result.offset}); 124 | } 125 | 126 | delete this.pendingResult[result.msgId]; 127 | } 128 | 129 | setTimeout(this._boundGetResults, 0); 130 | } 131 | } 132 | 133 | module.exports = Producer; 134 | -------------------------------------------------------------------------------- /lib/protocol.js: -------------------------------------------------------------------------------- 1 | var Int64 = require('node-int64'); 2 | 3 | function parseData(parsingContext) { 4 | var newMessage; 5 | if (parsingContext.currentMessage.remainingSize > 0) { 6 | newMessage = _parseNextMessage(parsingContext); 7 | if (newMessage) { 8 | _onNewMessage(newMessage, parsingContext) 9 | } 10 | } 11 | while (parsingContext.offset < parsingContext.data.length) { 12 | _readMsgSize(parsingContext); 13 | newMessage = _parseNextMessage(parsingContext); 14 | if (newMessage) { 15 | _onNewMessage(newMessage, parsingContext) 16 | } 17 | } 18 | } 19 | 20 | function _onNewMessage(newMessage, parsingContext){ 21 | if(parsingContext.parseMetadata){ 22 | var msgAndMetadata =_parseMetadata(newMessage); 23 | parsingContext.onMsgCB(msgAndMetadata.msg, msgAndMetadata.metadata); 24 | }else{ 25 | parsingContext.onMsgCB(newMessage); 26 | } 27 | } 28 | 29 | function _parseMetadata(fullMessage){ 30 | var topicLength = fullMessage.readInt32BE(0); 31 | var topic = fullMessage.slice(16, 16 + topicLength).toString(); 32 | var value = fullMessage.slice(16 + topicLength); 33 | 34 | var metadata = { 35 | partition: fullMessage.readInt32BE(12), 36 | offset: +(new Int64(fullMessage.slice(4,12))), 37 | topic: topic 38 | }; 39 | 40 | return {msg: value, metadata: metadata}; 41 | } 42 | 43 | function _parseNextMessage(parsingContext) { 44 | if (parsingContext.data.length === parsingContext.offset) { 45 | return; 46 | } 47 | 48 | 49 | var readSize = parsingContext.currentMessage.remainingSize; 50 | if (readSize > parsingContext.data.length - parsingContext.offset) { 51 | readSize = parsingContext.data.length - parsingContext.offset; 52 | } 53 | 54 | var messagePart = parsingContext.data.slice(parsingContext.offset, parsingContext.offset + readSize); 55 | parsingContext.offset += readSize; 56 | parsingContext.currentMessage.remainingSize -= readSize; 57 | if (parsingContext.currentMessage.remainingSize > 0) { 58 | parsingContext.currentMessage.parts.push(messagePart); 59 | return; 60 | } 61 | 62 | var msgBuffer = messagePart; 63 | if (parsingContext.currentMessage.parts.length > 0) { 64 | parsingContext.currentMessage.parts.push(messagePart); 65 | msgBuffer = Buffer.concat(parsingContext.currentMessage.parts); 66 | parsingContext.currentMessage.parts = []; 67 | } 68 | 69 | return msgBuffer; 70 | } 71 | 72 | function _readMsgSize(parsingContext) { 73 | 74 | if (parsingContext.currentMessage.partialSize.size === 0 && parsingContext.offset <= parsingContext.data.length - 4) { 75 | parsingContext.currentMessage.remainingSize = parsingContext.data.readInt32BE(parsingContext.offset); 76 | parsingContext.offset += 4; 77 | return; 78 | } 79 | 80 | var readSize = 4 - parsingContext.currentMessage.partialSize.size; 81 | readSize = Math.min(readSize, parsingContext.data.length - parsingContext.offset); 82 | var sizePart = parsingContext.data.slice(parsingContext.offset, parsingContext.offset + readSize); 83 | parsingContext.offset += readSize; 84 | parsingContext.currentMessage.partialSize.parts.push(sizePart); 85 | parsingContext.currentMessage.partialSize.size += readSize; 86 | if (parsingContext.currentMessage.partialSize.size < 4) { 87 | return; 88 | } 89 | var sizeFullBuf = sizePart; 90 | if (parsingContext.currentMessage.partialSize.parts.length > 1) { 91 | sizeFullBuf = Buffer.concat(parsingContext.currentMessage.partialSize.parts); 92 | } 93 | var msgSize = sizeFullBuf.readInt32BE(); 94 | 95 | parsingContext.currentMessage.partialSize.parts = []; 96 | parsingContext.currentMessage.partialSize.size = 0; 97 | parsingContext.currentMessage.remainingSize = msgSize; 98 | return; 99 | } 100 | 101 | 102 | module.exports = { 103 | parseData: parseData 104 | }; -------------------------------------------------------------------------------- /lib/stringProducer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by elio on 1/9/17. 3 | */ 4 | const Producer = require('./producer'); 5 | 6 | class StringProducer extends Producer{ 7 | constructor(options) { 8 | super(options, "com.liveperson.kafka.producer.StringProducer", "org.apache.kafka.common.serialization.StringSerializer"); 9 | } 10 | } 11 | 12 | module.exports = StringProducer; 13 | -------------------------------------------------------------------------------- /lib/util/javaInit.js: -------------------------------------------------------------------------------- 1 | var fs = require("fs"); 2 | var java = require("java"); 3 | var path = require("path"); 4 | var appRoot = require('app-root-path'); 5 | 6 | 7 | var baseDir = path.resolve(__dirname, '../../java/lib'); 8 | var externalDependenciesDir = appRoot + '/kafka-java-bridge/java/lib'; 9 | var resourcesDir = path.resolve(__dirname, '../../java/resources'); 10 | var externalLog4jPropsFilePath = appRoot + '/kafka-java-bridge/log4j/log4j.properties'; 11 | var internalLog4jPropsFilePath = resourcesDir + '/log4j.properties'; 12 | var log4jPropsFilePath = internalLog4jPropsFilePath; 13 | 14 | if(fs.existsSync(externalLog4jPropsFilePath)){ 15 | log4jPropsFilePath = externalLog4jPropsFilePath; 16 | } 17 | 18 | java.options.push('-Dlog4j.configuration=file:' + log4jPropsFilePath); 19 | 20 | addDependencies(baseDir); 21 | addDependencies(externalDependenciesDir); 22 | 23 | function addDependencies(dir){ 24 | if(!fs.existsSync(dir)){ 25 | return; 26 | } 27 | var dependencies = fs.readdirSync(dir); 28 | 29 | 30 | dependencies.forEach(function (dependency) { 31 | java.classpath.push(baseDir + "/" + dependency); 32 | }); 33 | } 34 | 35 | exports.getJavaInstance = function () { 36 | return java; 37 | }; 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-java-bridge", 3 | "description": "Kafka java client wrapper. Supports kafka version 0.8 and up", 4 | "version": "0.2.8", 5 | "author": { 6 | "name": "LivePersonInc", 7 | "email": "lp-kafka-java-bridge@googlegroups.com" 8 | }, 9 | "contributors": [ 10 | { 11 | "name": "Eli Ostro", 12 | "email": "elio@liveperson.com" 13 | }, 14 | { 15 | "name": "Reem Diab", 16 | "email": "reemd@liveperson.com" 17 | } 18 | ], 19 | "keywords": [ 20 | "kafka", 21 | "java", 22 | "highLevelConsumer", 23 | "high", 24 | "level", 25 | "consumer", 26 | "client", 27 | "kafka client", 28 | "node", 29 | "kafka0.8", 30 | "kafka8", 31 | "zookeeper", 32 | "liveperson" 33 | ], 34 | "main": "./index.js", 35 | "scripts": { 36 | "postinstall": "node ./scripts/postinstall.js", 37 | "start": "node ./index", 38 | "test": "grunt default --verbose" 39 | }, 40 | "dependencies": { 41 | "app-root-path": "^2.0.1", 42 | "java": "~0.8.0", 43 | "node-int64": "^0.4.0" 44 | }, 45 | "devDependencies": { 46 | "chai": "~3.5.0", 47 | "grunt": "~1.0.1", 48 | "grunt-available-tasks": "~0.6.2", 49 | "grunt-contrib-copy": "^1.0.0", 50 | "grunt-contrib-jshint": "~1.1.0", 51 | "grunt-env": "~0.4.4", 52 | "grunt-istanbul": "~0.7.0", 53 | "grunt-mocha-test": "~0.13.2", 54 | "grunt-node-version": "~1.0.0", 55 | "load-grunt-config": "~0.19.1", 56 | "mocha": "~3.2.0", 57 | "mockery": "~2.0.0", 58 | "sinon": "~1.17.2", 59 | "time-grunt": "~1.4.0" 60 | }, 61 | "repository": { 62 | "type": "git", 63 | "url": "https://github.com/LivePersonInc/kafka-java-bridge" 64 | }, 65 | "license": "MIT", 66 | "engines": { 67 | "node": ">=6.9.1", 68 | "npm": ">=3.10.9" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /scripts/postinstall.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var fs = require('fs'); 3 | var execSync = require('child_process').execSync; 4 | 5 | var javaDir = path.join(__dirname, '../java'); 6 | if(!fs.existsSync(javaDir + "/out")){ 7 | fs.mkdirSync(javaDir + "/out"); 8 | } 9 | 10 | execSync("javac -cp " + javaDir + "/lib/\\* -d " + javaDir +"/out " + javaDir + "/src/com/liveperson/kafka/consumer/*.java", {stdio:[0,1,2]}); 11 | execSync("javac -cp " + javaDir + "/lib/\\* -d " + javaDir +"/out " + javaDir + "/src/com/liveperson/kafka/producer/*.java", {stdio:[0,1,2]}); 12 | execSync("jar cvf " + javaDir + "/lib/bridge.jar -C " + javaDir + "/out/ .", {stdio:[0,1,2]}); 13 | -------------------------------------------------------------------------------- /test/js/protocol_test.js: -------------------------------------------------------------------------------- 1 | var expect = require("chai").expect; 2 | var requireHelper = require("./util/require_helper"); 3 | var protocol = requireHelper('protocol'); 4 | 5 | describe("Protocol tests", function () { 6 | 7 | it("Test single message", function () { 8 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 9 | var buffer = new Buffer(14); 10 | var messages = []; 11 | buffer.writeInt32BE(10, 0); 12 | buffer.write("0123456789", 4); 13 | var parsingContext = { 14 | offset: 0, 15 | currentMessage: currentMessage, 16 | data: buffer, 17 | onMsgCB:function(message){ 18 | messages.push(message); 19 | } 20 | }; 21 | protocol.parseData(parsingContext); 22 | expect(messages.length).to.equal(1); 23 | expect(messages[0].toString()).to.equal("0123456789"); 24 | expect(parsingContext.offset).to.equal(buffer.length); 25 | }); 26 | 27 | it("Test multiple messages", function () { 28 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 29 | var buffer = new Buffer(27); 30 | var messages = []; 31 | buffer.writeInt32BE(10, 0); 32 | buffer.write("0123456789", 4); 33 | buffer.writeInt32BE(9, 14); 34 | buffer.write("abcdefghi", 18); 35 | var parsingContext = { 36 | offset: 0, 37 | currentMessage: currentMessage, 38 | data: buffer, 39 | onMsgCB:function(message){ 40 | messages.push(message); 41 | } 42 | }; 43 | protocol.parseData(parsingContext); 44 | expect(messages.length).to.equal(2); 45 | expect(messages[0].toString()).to.equal("0123456789"); 46 | expect(messages[1].toString()).to.equal("abcdefghi"); 47 | expect(parsingContext.offset).to.equal(buffer.length); 48 | }); 49 | 50 | 51 | it("Test partial message update current message", function () { 52 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 53 | var buffer = new Buffer(14); 54 | var messages = []; 55 | buffer.writeInt32BE(15, 0); 56 | buffer.write("0123456789", 4); 57 | var parsingContext = { 58 | offset: 0, 59 | currentMessage: currentMessage, 60 | data: buffer, 61 | onMsgCB:function(message){ 62 | messages.push(message); 63 | } 64 | }; 65 | protocol.parseData(parsingContext); 66 | expect(messages.length).to.equal(0); 67 | expect(currentMessage.remainingSize).to.equal(5); 68 | expect(parsingContext.offset).to.equal(buffer.length); 69 | }); 70 | 71 | it("Test remaining message part", function () { 72 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 73 | var buffer = new Buffer(14); 74 | var messages = []; 75 | buffer.writeInt32BE(15, 0); 76 | buffer.write("0123456789", 4); 77 | var parsingContext = { 78 | offset: 0, 79 | currentMessage: currentMessage, 80 | data: buffer, 81 | onMsgCB:function(message){ 82 | messages.push(message); 83 | } 84 | }; 85 | protocol.parseData(parsingContext); 86 | expect(messages.length).to.equal(0); 87 | expect(currentMessage.remainingSize).to.equal(5); 88 | expect(parsingContext.offset).to.equal(buffer.length); 89 | buffer = new Buffer(5); 90 | buffer.write("abcde", 0); 91 | parsingContext = { 92 | offset: 0, 93 | currentMessage: currentMessage, 94 | data: buffer, 95 | onMsgCB:function(message){ 96 | messages.push(message); 97 | } 98 | }; 99 | messages = []; 100 | protocol.parseData(parsingContext); 101 | expect(messages.length).to.equal(1); 102 | expect(currentMessage.remainingSize).to.equal(0); 103 | expect(messages[0].toString()).to.equal("0123456789abcde"); 104 | expect(parsingContext.offset).to.equal(buffer.length); 105 | 106 | }); 107 | 108 | it("Test message partial message new data still not complete", function () { 109 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 110 | var buffer = new Buffer(14); 111 | var messages = []; 112 | buffer.writeInt32BE(20, 0); 113 | buffer.write("0123456789", 4); 114 | var parsingContext = { 115 | offset: 0, 116 | currentMessage: currentMessage, 117 | data: buffer, 118 | onMsgCB:function(message){ 119 | messages.push(message); 120 | } 121 | }; 122 | protocol.parseData(parsingContext); 123 | expect(messages.length).to.equal(0); 124 | expect(currentMessage.remainingSize).to.equal(10); 125 | expect(parsingContext.offset).to.equal(buffer.length); 126 | buffer = new Buffer(5); 127 | buffer.write("abcde", 0); 128 | parsingContext = { 129 | offset: 0, 130 | currentMessage: currentMessage, 131 | data: buffer, 132 | onMsgCB:function(message){ 133 | messages.push(message); 134 | } 135 | }; 136 | messages = []; 137 | protocol.parseData(parsingContext); 138 | expect(messages.length).to.equal(0); 139 | expect(currentMessage.remainingSize).to.equal(5); 140 | expect(parsingContext.offset).to.equal(buffer.length); 141 | }); 142 | 143 | it("Test partial size", function () { 144 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 145 | var buffer = new Buffer(2); 146 | var messages = []; 147 | buffer.writeInt16BE(0, 0); 148 | var parsingContext = { 149 | offset: 0, 150 | currentMessage: currentMessage, 151 | data: buffer, 152 | onMsgCB:function(message){ 153 | messages.push(message); 154 | } 155 | }; 156 | protocol.parseData(parsingContext); 157 | expect(messages.length).to.equal(0); 158 | expect(currentMessage.remainingSize).to.equal(0); 159 | expect(currentMessage.partialSize.size).to.equal(2); 160 | expect(currentMessage.partialSize.parts.length).to.equal(1); 161 | expect(parsingContext.offset).to.equal(buffer.length); 162 | }); 163 | 164 | it("Test partial size size complete", function () { 165 | var currentMessage = {remainingSize: 0, parts: [], partialSize: {size: 0, parts: []}}; 166 | var buffer = new Buffer(2); 167 | var messages = []; 168 | buffer.writeInt16BE(0, 0); 169 | var parsingContext = { 170 | offset: 0, 171 | currentMessage: currentMessage, 172 | data: buffer, 173 | onMsgCB:function(message){ 174 | messages.push(message); 175 | } 176 | }; 177 | messages = []; 178 | protocol.parseData(parsingContext); 179 | expect(messages.length).to.equal(0); 180 | expect(currentMessage.remainingSize).to.equal(0); 181 | expect(currentMessage.partialSize.size).to.equal(2); 182 | expect(currentMessage.partialSize.parts.length).to.equal(1); 183 | expect(parsingContext.offset).to.equal(buffer.length); 184 | buffer = new Buffer(12); 185 | buffer.writeInt16BE(10, 0); 186 | buffer.write("0123456789", 2); 187 | parsingContext = { 188 | offset: 0, 189 | currentMessage: currentMessage, 190 | data: buffer, 191 | onMsgCB:function(message){ 192 | messages.push(message); 193 | } 194 | }; 195 | protocol.parseData(parsingContext); 196 | expect(messages.length).to.equal(1); 197 | expect(messages[0].toString()).to.equal("0123456789"); 198 | expect(parsingContext.offset).to.equal(buffer.length); 199 | }); 200 | }); -------------------------------------------------------------------------------- /test/js/util/require_helper.js: -------------------------------------------------------------------------------- 1 | module.exports = function (path) { 2 | return require((process.env.APP_DIR_FOR_CODE_COVERAGE || '../../../lib/') + path); 3 | }; 4 | --------------------------------------------------------------------------------