├── .gitignore ├── .npmrc ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── lib ├── client │ ├── base.js │ ├── consumer.js │ ├── index.js │ └── producer.js ├── observables │ ├── consumer.js │ └── producer.js └── operators │ ├── json-message.js │ └── kafka-message.js ├── package-lock.json ├── package.json └── test ├── integration ├── helpers │ └── helper.js ├── observables │ ├── consumer.spec.js │ └── producer.spec.js └── operators │ ├── json-message.spec.js │ └── kafka-message.spec.js ├── support ├── integration.json └── unit.json └── unit ├── consumer.spec.js ├── helpers └── helper.js ├── kafka-observable.spec.js ├── mock ├── mock-bunyan.js └── mock-no-kafka.js ├── operators.spec.js └── producer.spec.js /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | node_modules 3 | .idea 4 | npm-debug.log 5 | coverage -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmjs.org/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "6.4.0" 5 | - "8.2.1" 6 | - "8.9.1" 7 | - "9.2.0" 8 | jobs: 9 | include: 10 | - stage: coverage 11 | node_js: 12 | - "9.2.0" 13 | env: 14 | global: 15 | - CC_TEST_REPORTER_ID=d3ce1b1e6b62d8d585ece975cb481c0c3fcfdb664b95dbcb49f6cfc4c95da519 16 | before_script: 17 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 18 | - chmod +x ./cc-test-reporter 19 | - ./cc-test-reporter before-build 20 | script: npm run coverage 21 | after_script: 22 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT 23 | - stage: integration 24 | node_js: 25 | - "9.2.0" 26 | services: 27 | - docker 28 | before_install: 29 | - docker pull spotify/kafka 30 | script: npm run integration-test 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Apigee Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![node](https://img.shields.io/node/v/kafka-observable.svg)]() 2 | [![Build Status][badge-travis]][travis] 3 | [![Test Coverage][badge-coverage]][coverage] 4 | [![license][badge-license]][license] 5 | 6 | # kafka-observable 7 | 8 | __kafka-observable__ is the easiest way to exchange messages through kafka with [node.js](https://nodejs.org). 9 | 10 | 11 | Using the solid [no-kafka](https://github.com/oleksiyk/kafka) as default client, __kafka-observable__ creates [RxJS](https://github.com/ReactiveX/rxjs) observables 12 | that can be manipulated as if you were using Kafka Streams, but with a familiar interface 13 | to javascript developers. 14 | 15 | ### Why observables? 16 | 17 | Think of observables as collections which elements arrive over time. You may iterate over 18 | them, which means you can also apply filter, map or reduce. 19 | 20 | Many of the operations provided by observables are very similar to the capabilities available 21 | in Kafka Streams, including the ability to use window (accumulate values for a period). 22 | 23 | ### Installation 24 | 25 | ```bash 26 | npm install --save kafka-observable 27 | ``` 28 | 29 | ### Example usage 30 | 31 | Imagine your customers can subscribe to out-of-stock products in your online store to 32 | receive emails when the product is in stock. Your stock management publishes updates 33 | to a kafka topic called __inventory_updates__ and you mailer consumes from __notifications__. 34 | 35 | ```javascript 36 | const opts = { 37 | brokers: 'kafka://kafka-broker.example.com:9092', 38 | groupId: 'inventory-notifications' 39 | }; 40 | 41 | const KafkaObservable = require('kafka-observable')(opts); 42 | // assumes getWatchers will execute a network request and returns an observable 43 | const getWatchers = require('./lib/observables/watchers'); 44 | 45 | const subscription = KafkaObservable.fromTopic('inventory_updates') 46 | // gets messge as JSON 47 | .let(KafkaObservable.JSONMessage()) 48 | // just arrived 49 | .filter(({inventory}) => inventory.previous === 0 && inventory.current > 0) 50 | // gets watchers, format message and concat 2 dimensional observable 51 | .concatMap(product => 52 | getWatchers(product.id) 53 | .map(watchers => ({ watchers, product })) 54 | // sends formated message to new topic and concats 2 dimensional observable 55 | .concatMap(message => KafkaObservable.toTopic('notifications', message)); 56 | 57 | subscription.subscribe(success => console.log(success), err => console.error(err)); 58 | ``` 59 | 60 | ## Methods 61 | 62 | ### fromTopic(topic, options, adapterFactory = defaultAdapterFactory) 63 | 64 | Creates an observable that will consume from a Kafka topic. 65 | 66 | #### Parameters 67 | * topic (String) - topic name 68 | * options (Object) - client options 69 | * adapterFactory (Object) - client adapter (Defaults to no-kafka adapter) 70 | 71 | #### Examples 72 | 73 | KafkaObservable as a function: 74 | ```javascript 75 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' }; 76 | 77 | const KafkaObservable = require('kafka-observable')(opts); 78 | const consumer = KafkaObservable.fromTopic('my_topic') 79 | .map(({message}) => message.value.toString('utf8')); 80 | 81 | consumer.subscribe(message => console.info(message)); 82 | ``` 83 | 84 | Passing __options__ parameter to fromTopic: 85 | ```javascript 86 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' }; 87 | 88 | const KafkaObservable = require('kafka-observable'); 89 | const consumer = KafkaObservable.fromTopic('my_topic', opts) 90 | .map(({message}) => message.value.toString('utf8')); 91 | 92 | consumer.subscribe(message => console.info(message)); 93 | ``` 94 | 95 | #### Options 96 | 97 | Below are the main options for the consumer. For more consumer options, please 98 | refer to [no-kafka](https://github.com/oleksiyk/kafka) options (in case you use the provided default adapter). 99 | 100 | | Option | Required | Type | Default | Description | 101 | |---|---|---|---|---| 102 | | brokers | yes | Array/String | - | list of Kafka brokers 103 | | groupId | yes | String| - | consumer group id 104 | | autoCommit | no | boolean |true | commits the message offset automatically if no exception is thrown 105 | | strategy | no | String | Default | name of the assignment strategy for the consumer (Default/Consistent/WeightedRoundRobin) 106 | 107 | ### toTopic(topic, messages, options, adapterFactory = defaultAdapterFactory) 108 | 109 | Creates an observable that publishes messages to a Kafka topic. 110 | 111 | #### Parameters 112 | * topic (String) - topic name 113 | * messages (String|Array|Observable) - messages to be published in kafka topic 114 | * options (Object) - client options 115 | * adapterFactory (Object) - client adapter (Defaults to no-kafka adapter) 116 | 117 | #### Examples 118 | 119 | KafkaObservable as a function: 120 | ```javascript 121 | const opts = { brokers: 'kafka://127.0.0.1:9092' }; 122 | const messages = [{key: 'value1'}, {key: 'value2'}]; 123 | 124 | const KafkaObservable = require('kafka-observable')(opts); 125 | const producer = KafkaObservable.toTopic('my_topic', messages); 126 | 127 | producer.subscribe(message => console.info(message)); 128 | ``` 129 | 130 | Passing __options__ parameter to toTopic: 131 | ```javascript 132 | const opts = { brokers: 'kafka://127.0.0.1:9092' }; 133 | const messages = Observable.from([{key: 'value1'}, {key: 'value2'}]); 134 | 135 | const KafkaObservable = require('kafka-observable'); 136 | const producer = KafkaObservable.toTopic('my_topic', messages, opts); 137 | 138 | producer.subscribe(message => console.info(message)); 139 | ``` 140 | 141 | #### Options 142 | 143 | Below are the main options for the producer. For more producer options, please 144 | refer to [no-kafka](https://github.com/oleksiyk/kafka) options (in case you use the provided default adapter). 145 | 146 | | Option | Required | Type | Default | Description | 147 | |---|---|---|---|---| 148 | | brokers | yes | Array/String | - | list of Kafka brokers 149 | | partitioner | no | prototype/String | Default | name (Default/HashCRC32) or prototype (instance of Kafka.DefaultPartitioner) to use as producer partitioner 150 | 151 | ### TextMessage(mapper = (x) => x) 152 | 153 | Convenience operator which converts a Buffer message value into utf8 string. 154 | 155 | #### Parameters 156 | * mapper (Function) - mapper function 157 | 158 | #### Example 159 | 160 | ```javascript 161 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' }; 162 | 163 | const KafkaObservable = require('kafka-observable'); 164 | const consumer = KafkaObservable.fromTopic('my_topic', opts) 165 | .let(KafkaObservable.TextMessage()); 166 | 167 | consumer.subscribe(message => console.info(message)); 168 | ``` 169 | 170 | ### JSONMessage(mapper = (x) => x) 171 | 172 | Convenience operator provided to deserialize an object from a JSON message. 173 | 174 | #### Parameters 175 | * mapper (Function) - mapper function 176 | 177 | #### Example 178 | 179 | ```javascript 180 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' }; 181 | 182 | const KafkaObservable = require('kafka-observable'); 183 | const consumer = KafkaObservable.fromTopic('my_topic', opts) 184 | .let(KafkaObservable.JSONMessage()); 185 | 186 | consumer.subscribe(json => console.info(json.key)); 187 | ``` 188 | 189 | ## Custom Kafka Adapter 190 | 191 | If you don't want to use [no-kafka](https://github.com/oleksiyk/kafka) you can write an adapter for your client which respects 192 | the interface established by the code in `lib/client`. 193 | 194 | #### Why an adapter? 195 | 196 | I currently use an internal kafka client at Netflix with an interface very similar to this adapter 197 | and I wanted it to work out-of-the-box. 198 | 199 | ## Development 200 | 201 | #### Unit tests 202 | ```bash 203 | npm install 204 | npm run unit-test 205 | ``` 206 | 207 | #### Integration tests 208 | *requires docker to be installed and accessible through the **docker** command* 209 | ```bash 210 | npm install 211 | docker pull spotify/kafka 212 | npm run unit-test 213 | ``` 214 | 215 | #### Test coverage 216 | *based on unit tests* 217 | ```bash 218 | npm install 219 | npm run coverage 220 | open coverage/lcov-report/index.html 221 | ``` 222 | 223 | #### Documentation 224 | ```bash 225 | npm install 226 | npm run gen-docs 227 | open out/index.html 228 | ``` 229 | ___ 230 | ### License: [MIT](https://github.com/ghermeto/kafka-observable/blob/master/LICENSE) 231 | 232 | [badge-license]: https://img.shields.io/badge/License-MIT-green.svg 233 | [license]: https://github.com/ghermeto/kafka-observable/blob/master/LICENSE 234 | [badge-travis]: https://api.travis-ci.org/ghermeto/kafka-observable.svg?branch=master 235 | [travis]: https://travis-ci.org/ghermeto/kafka-observable 236 | [badge-coverage]: https://codeclimate.com/github/ghermeto/kafka-observable/badges/coverage.svg 237 | [coverage]: https://codeclimate.com/github/ghermeto/kafka-observable/coverage 238 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creates kafka consumer and producer observables given a kafka client adapter 3 | * and a options object. 4 | * 5 | * @example consumer 6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' }; 7 | * const KafkaObservable = require('kafka-observable')(opts); 8 | * const observable = KafkaObservable.fromTopic('my_topic'); 9 | * 10 | * @example producer 11 | * const opts = { brokers: 'kafka://127.0.0.1:9092' }; 12 | * const KafkaObservable = require('kafka-observable')(opts); 13 | * const observable = KafkaObservable.toTopic('my_topic', 'my message'); 14 | * 15 | * @module KafkaObservable 16 | * @author ghermeto 17 | **/ 18 | 'use strict'; 19 | 20 | const Consumer = require('./lib/observables/consumer'); 21 | const Producer = require('./lib/observables/producer'); 22 | 23 | const jsonMessage = require('./lib/operators/json-message'); 24 | const kafkaMessage = require('./lib/operators/kafka-message'); 25 | 26 | /** 27 | * @param {Object} options client options 28 | * @param {boolean} options.autoCommit automatically commits read message 29 | * @param {Object|String} options.partitioner name or class for partitioner implementation 30 | * @param {String} options.strategy name of the assignment strategy 31 | * @param {Object} clientFactory your custom kafka client adapter 32 | * @returns {ObservableFactory} 33 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options. 34 | */ 35 | function KafkaObservable(options, clientFactory) { 36 | /** 37 | * Object capable of creating both the consumer and producer observables 38 | * @inner 39 | * @name ObservableFactory 40 | * @typedef {Object} ObservableFactory 41 | * @property {Function} fromTopic same as fromTopic from prototype 42 | * @property {Function} toTopic same as toTopic from prototype 43 | */ 44 | return { 45 | /** 46 | * Creates a Consumer observable 47 | * @param {string} topic topic to subscribe 48 | * @param {Object} opts kafka client options 49 | * @param {Object} client kafka adapter 50 | * @returns {Observable} 51 | * @see {@link observables/Consumer} 52 | */ 53 | fromTopic: 54 | (topic, opts = options, client = clientFactory) => 55 | Consumer.create(topic, opts, client), 56 | /** 57 | * Creates a Producer observable 58 | * @param {string} topic topic to subscribe 59 | * @param {Array|Observable} messages message(s) to be sent 60 | * @param {Object} opts kafka client options 61 | * @param {Object} client kafka adapter 62 | * @returns {Observable} 63 | * @see {@link observables/Producer} 64 | */ 65 | toTopic: 66 | (topic, messages, opts = options, client = clientFactory) => 67 | Producer.create(topic, messages, opts, client), 68 | 69 | /** 70 | * Operator: formats a message as a JSON object 71 | * @function 72 | * @param {Function} mapper mapping function 73 | */ 74 | JSONMessage: jsonMessage, 75 | /** 76 | * Operator: formats a message as a text message 77 | * @function 78 | * @param {Function} mapper mapping function 79 | */ 80 | TextMessage: kafkaMessage 81 | } 82 | } 83 | 84 | // observables 85 | KafkaObservable.fromTopic = KafkaObservable().fromTopic; 86 | KafkaObservable.toTopic = KafkaObservable().toTopic; 87 | 88 | // operators 89 | KafkaObservable.JSONMessage = jsonMessage; 90 | KafkaObservable.TextMessage = kafkaMessage; 91 | 92 | module.exports = KafkaObservable; -------------------------------------------------------------------------------- /lib/client/base.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Base Adapter class. 3 | * 4 | * @author ghermeto 5 | **/ 6 | 7 | const EventEmitter = require('events'); 8 | const bunyan = require('bunyan'); 9 | const shortid = require('shortid'); 10 | 11 | /** 12 | * Base pipeline class extended by both Consumer and Producer classes 13 | * @class Base 14 | */ 15 | class Base extends EventEmitter { 16 | 17 | constructor(opts) { 18 | super(); 19 | this.isValid(opts); 20 | this.opts = opts; 21 | this.clientId = opts.clientId || `kafka-observable-${shortid.generate()}`; 22 | this.brokers = Base.formatBrokers(opts.brokers); 23 | 24 | this.log = bunyan.createLogger({ 25 | name: this.clientId, 26 | level: process.env.LOG_LEVEL 27 | }); 28 | } 29 | 30 | /** 31 | * Convert to array or throw if parameter is not string or array 32 | * @param {String|Array} brokers single or multiple brokers URL 33 | * @return {Array.<*>} 34 | */ 35 | static formatBrokers(brokers) { 36 | const className = brokers.constructor.name; 37 | if (className !== 'Array' && className !== 'String') { 38 | throw new TypeError('invalid brokers parameter'); 39 | } 40 | return [].concat(brokers); 41 | } 42 | 43 | /** 44 | * Validate options. Throws if invalid. 45 | * @param {Object} opts user defined options 46 | */ 47 | isValid(opts) { 48 | if (!opts.brokers) { 49 | throw new Error('missing vipaddress or brokers parameter'); 50 | } 51 | } 52 | 53 | /** 54 | * To be overridden by consumer and producer defaults 55 | * @return {Object} default client options 56 | */ 57 | defaults() { 58 | return { 59 | connectionString: this.brokers.join(','), 60 | logger: {logFunction: this.logging.bind(this)} 61 | }; 62 | } 63 | 64 | /** 65 | * @return {Object} client options 66 | */ 67 | clientOptions() { 68 | return Object.assign({}, this.defaults(), this.opts); 69 | } 70 | 71 | /** 72 | * Emits error event if listener error is found. 73 | * EventEmitter throws if an error event is emitted without listeners 74 | * @see https://nodejs.org/api/events.html#events_error_events 75 | * @param {Object} err error object 76 | */ 77 | handleError(err) { 78 | this.log.error(err); 79 | if (this.listenerCount('error') > 0) { 80 | this.emit('error', err); 81 | } 82 | return Promise.reject(err); 83 | } 84 | 85 | /** 86 | * Pipes logs through Bunyan and fire events 87 | * @param {String} level log level 88 | * @param {Array} args 89 | */ 90 | logging(level, ...args) { 91 | this.log[level.toLowerCase()](...args); 92 | this.monitoring(level, ...args); 93 | } 94 | 95 | /** 96 | * Parses through the logs to emit meaningful events 97 | * @param {String} level log level 98 | * @param {Array} args 99 | */ 100 | monitoring(level, ...args) { 101 | if (level === bunyan.ERROR) { 102 | this.handleError(args[3]); 103 | } 104 | } 105 | 106 | } 107 | 108 | module.exports = Base; -------------------------------------------------------------------------------- /lib/client/consumer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kafka Consumer Adapter. It adapts no-kafka group consumer to fits the 3 | * KafkaObservable interface. 4 | * 5 | * @example consumer 6 | * const kafkaClient = require('./lib/client'); 7 | * const consumer = kafkaClient.consumer({ 8 | * brokers: 'https://exemple.com:7123', 9 | * groupId: 'audit', 10 | * }); 11 | * 12 | * consumer.on('error', err => console.error(err)); 13 | * consumer.subscribe('my_topic', (messageSet, topic, partition) => { 14 | * messageSet.forEach(promise => promise.then(({offset, message}) => { 15 | * console.info(topic, partition, message.value.toString('utf8'), offset); 16 | * return consumer.client().commitOffset({t opic, partition, offset }); 17 | * })); 18 | * }); 19 | * 20 | * @author ghermeto 21 | **/ 22 | const Kafka = require('no-kafka'); 23 | const Base = require('./base'); 24 | 25 | /** 26 | * Enables subscribing to a topic in the pipeline 27 | * @class Consumer 28 | */ 29 | class Consumer extends Base { 30 | constructor(opts) { 31 | super(opts); 32 | } 33 | 34 | /** 35 | * Validates required options 36 | * @private 37 | * @override 38 | * @param {Object} opts user defined options 39 | */ 40 | isValid(opts) { 41 | super.isValid(opts); 42 | if (!opts.groupId) { throw new Error('missing consumer groupId'); } 43 | } 44 | 45 | /** 46 | * Dynamically generates consumer default configuration properties 47 | * @private 48 | * @override 49 | * @returns {Object} Default configs 50 | */ 51 | defaults() { 52 | const base = super.defaults(); 53 | return Object.assign({ 54 | idleTimeout: 100, 55 | heartbeatTimeout: 100, 56 | clientId: this.clientId 57 | }, base); 58 | } 59 | 60 | /** 61 | * Returns the consumer client once it has subscribed to a topic. 62 | * 63 | * @return {Kafka.GroupConsumer|GroupConsumer} 64 | */ 65 | client() { 66 | return this.consumer; 67 | } 68 | 69 | /** 70 | * Convert strategy name to object. 71 | * Possible values are: WeightedRoundRobin, Consistent and Default. 72 | * @private 73 | * @param {String} name strategy name. 74 | */ 75 | strategy(name = 'Default') { 76 | const className = `${name}AssignmentStrategy`; 77 | if (!Kafka[className]) { 78 | this.log.warn(`Strategy ${className} not found. Using default.`); 79 | return new Kafka.DefaultAssignmentStrategy(); 80 | } 81 | return new Kafka[className](); 82 | } 83 | 84 | /** 85 | * First unsubscribe from the topic and then ends the client connection. 86 | * 87 | * @param {String} topic topic to unsubscribe 88 | * @return {Promise} 89 | */ 90 | end(topic) { 91 | return this.client() 92 | .unsubscribe(topic) 93 | .then(() => this.client().end()); 94 | } 95 | 96 | /** 97 | * Commits the received message 98 | * 99 | * @param {String} topic topic which had the message 100 | * @param {Number} partition partition which had the message 101 | * @param {Number} offset message offset in the topic partition 102 | * @return {Promise} 103 | */ 104 | commit(topic, partition, offset) { 105 | return this.client().commitOffset({ topic, partition, offset }); 106 | } 107 | 108 | /** 109 | * Subscribe to a list of channels passing a callback 110 | * 111 | * @param {Array} topics topics to subscribe 112 | * @param {Function} handler function to be executed when a message arrives 113 | */ 114 | subscribe(topics, handler) { 115 | const options = this.clientOptions(); 116 | this.consumer = new Kafka.GroupConsumer(options); 117 | 118 | const strategies = [{ 119 | strategy: this.strategy(this.opts.strategy), 120 | metadata: this.opts.metadata, 121 | subscriptions: [].concat(topics), 122 | handler 123 | }]; 124 | 125 | return this.consumer.init(strategies) 126 | .catch(err => this.handleError(err)); 127 | } 128 | } 129 | 130 | module.exports = Consumer; -------------------------------------------------------------------------------- /lib/client/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kafka Client Adapter - adapts the no-kafka client to the expected interface 3 | * 4 | * @example subscriber 5 | * const kafkaClient = require('./lib/client'); 6 | * const consumer = kafkaClient.consumer({ 7 | * brokers: 'https://exemple.com:7123', 8 | * groupId: 'audit', 9 | * }); 10 | * 11 | * consumer.on('error', err => console.error(err)); 12 | * consumer.subscribe('my_topic', (messageSet, topic, partition) => { 13 | * messageSet.forEach(promise => promise.then(({offset, message}) => { 14 | * console.info(topic, partition, message.value.toString('utf8'), offset); 15 | * return consumer.client().commitOffset({t opic, partition, offset }); 16 | * })); 17 | * }); 18 | * 19 | * @example producer 20 | * const kafkaClient = require('./lib/client'); 21 | * const producer = kafkaClient.producer({brokers: 'https://exemple.com:7123'}); 22 | * 23 | * const message = { hello: 'world', ok: true }; 24 | * producer.on('error', err => console.error(err)); 25 | * producer.publish('my_topic', message); 26 | * 27 | * @author ghermeto 28 | **/ 29 | 'use strict'; 30 | 31 | const Producer = require('./producer'); 32 | const Consumer = require('./consumer'); 33 | 34 | exports.producer = opts => new Producer(opts); 35 | exports.consumer = opts => new Consumer(opts); 36 | -------------------------------------------------------------------------------- /lib/client/producer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kafka Producer Adapter. It adapts no-kafka producer to the format expected 3 | * by the KafkaObservable interface 4 | * 5 | * @example producer 6 | * const kafkaClient = require('./lib/client'); 7 | * const producer = kafkaClient.producer({brokers: 'https://exemple.com:7123'}); 8 | * 9 | * const message = { hello: 'world', ok: true }; 10 | * producer.on('error', err => console.error(err)); 11 | * producer.publish('my_topic', message); 12 | * 13 | * @author ghermeto 14 | **/ 15 | 16 | const Kafka = require('no-kafka'); 17 | const Base = require('./base'); 18 | const shortid = require('shortid'); 19 | 20 | /** 21 | * @class Producer 22 | */ 23 | class Producer extends Base { 24 | constructor(opts) { 25 | super(opts); 26 | const options = this.clientOptions(); 27 | this.producer = new Kafka.Producer(Object.assign({}, options)); 28 | } 29 | 30 | /** 31 | * @return {Object} client options 32 | */ 33 | clientOptions() { 34 | const base = super.clientOptions(); 35 | const fixed = { partitioner: this.partitioner(this.opts.partitioner) }; 36 | return Object.assign({}, base, fixed); 37 | } 38 | 39 | /** 40 | * Dynamically generates producer default configuration properties 41 | * @private 42 | * @override 43 | * @returns {Object} Default configs 44 | */ 45 | defaults() { 46 | const base = super.defaults(); 47 | const defaults = { timeout: 1000 }; 48 | return Object.assign({}, base, defaults); 49 | } 50 | 51 | /** 52 | * Initializes the producer if necessary or use the cached producer 53 | * @private 54 | * @return {Promise} active producer 55 | */ 56 | init() { 57 | if (this.active) { return this.active; } 58 | this.active = this.producer.init(); 59 | return this.active; 60 | } 61 | 62 | /** 63 | * Returns a partitioner instance from a string or from a class. 64 | * @private 65 | * @param {String|Object} nameOrClass name of the partitioner or class 66 | * @return {Object} partitioner instance 67 | */ 68 | partitioner(nameOrClass = 'Default') { 69 | if (typeof nameOrClass === 'object') { 70 | return new nameOrClass(); 71 | } 72 | const className = `${nameOrClass}Partitioner`; 73 | if (!Kafka[className]) { 74 | this.log.warn(`Partitioner ${className} not found. Using default.`); 75 | return new Kafka.DefaultPartitioner(); 76 | } 77 | return new Kafka[className](); 78 | } 79 | 80 | /** 81 | * Serializes messages 82 | * @static 83 | * @private 84 | * @param {Object|String} message 85 | * @return {String} serialized message 86 | */ 87 | static serialize(message) { 88 | return typeof message === 'string' ? message : JSON.stringify(message); 89 | } 90 | 91 | /** 92 | * Returns the producer client once it has sent at least one message. 93 | * @return {Kafka.Producer|Producer} 94 | */ 95 | client() { 96 | return this.producer; 97 | } 98 | 99 | /** 100 | * Terminates the client connection. 101 | * @return {Promise} 102 | */ 103 | end() { 104 | return this.client().end(); 105 | } 106 | 107 | /** 108 | * Publishes a message to Kafka 109 | * @param {String} topic topic to publish on 110 | * @param {Object|String} message message to be sent 111 | * @param {String} key message key 112 | * @return {Promise} 113 | */ 114 | publish(topic, message, key = `${this.clientId}-${shortid.generate()}`) { 115 | const serialized = Producer.serialize(message); 116 | const payload = {topic: topic, message: {key: key, value: serialized}}; 117 | 118 | return this.init() 119 | .then(() => this.producer.send(payload)) 120 | .catch(err => this.handleError(err)); 121 | } 122 | } 123 | 124 | module.exports = Producer; -------------------------------------------------------------------------------- /lib/observables/consumer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kafka Consumer Observable. 3 | * 4 | * @example consumer 5 | * const consumerObservable = require('./lib/consumer'); 6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' }; 7 | * const observable = consumerObservable.create('my_topic', opts); 8 | * 9 | * observable 10 | * .do(m => m.commit()) 11 | * .map(m => m.message) 12 | * .subscribe(message => console.info(message)); 13 | * 14 | * @module observables/Consumer 15 | * @author ghermeto 16 | **/ 17 | 18 | const bunyan = require('bunyan'); 19 | const defaultFactory = require('../client'); 20 | const Observable = require('rxjs').Observable; 21 | 22 | const observerError = 'Error executing observer. Message not committed.'; 23 | 24 | /** 25 | * logger 26 | * @constant 27 | * @type {Object} 28 | */ 29 | const log = bunyan.createLogger({ 30 | name: 'KafkaObservable', 31 | level: process.env.LOG_LEVEL 32 | }); 33 | 34 | /** 35 | * default consumer options 36 | * @constant 37 | * @type {Object} 38 | */ 39 | const defaults = { 40 | autoCommit: true 41 | }; 42 | 43 | /** 44 | * Creates a selector function with the kafka consumer in scope 45 | * 46 | * @private 47 | * @param {Observer} observer 48 | * @param {Object | defaultFactory} kafkaConsumer 49 | */ 50 | const selectorFactory = 51 | (observer, kafkaConsumer) => 52 | /** 53 | * Handles a kafka message. Will consume the message and pass it to 54 | * the observable 55 | * 56 | * @private 57 | * @param {Array} messageSet list of messages received 58 | * @param {String} topic topic which the message was received 59 | * @param {Number} partition partion on which the message was received 60 | */ 61 | (messageSet, topic, partition) => 62 | messageSet.map(({offset, message}) => { 63 | const value = message.value; 64 | const decoded = value ? value.toString('utf8') : null; 65 | log.trace(topic, partition, offset, decoded); 66 | 67 | // pre-wired commit function as observable 68 | const commit = () => 69 | Observable 70 | .fromPromise(kafkaConsumer 71 | .commit(topic, partition, offset)); 72 | 73 | if (kafkaConsumer.opts.autoCommit) { 74 | // if we will commit here, no need to return commit function 75 | try { 76 | observer.next({ topic, partition, offset, message }); 77 | // must return commit as a promise 78 | return kafkaConsumer.commit(topic, partition, offset); 79 | } catch (err) { 80 | log.error(observerError, err); 81 | } 82 | } 83 | 84 | observer.next({ topic, partition, offset, message, commit }); 85 | }); 86 | 87 | /** 88 | * Observable as result from messages in a kafka topic 89 | * 90 | * @private 91 | * @param {Object} consumer 92 | * @param {String} topic kafka topic 93 | * @return {Observable} 94 | */ 95 | const fromTopic = (consumer, topic) => { 96 | const observable = Observable.create(observer => { 97 | consumer 98 | .subscribe(topic, selectorFactory(observer, consumer)) 99 | .catch(err => observer.error(err)); 100 | 101 | return () => consumer.end(topic); 102 | }); 103 | 104 | observable._adapter = consumer; 105 | return observable; 106 | }; 107 | 108 | /** 109 | * Consumer factory. 110 | * 111 | * @param {String} topic topic name 112 | * @param {Object} options 113 | * @param {Array|String} options.brokers comma separated list of brokers 114 | * @param {Object | defaultFactory} clientFactory 115 | * @return {Observable} 116 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options. 117 | */ 118 | exports.create = function (topic, options, clientFactory = defaultFactory) { 119 | const opts = Object.assign({}, defaults, options); 120 | try { 121 | const kafkaConsumer = clientFactory.consumer(opts); 122 | return fromTopic(kafkaConsumer, topic); 123 | } catch (err) { 124 | return Observable.throw(err); 125 | } 126 | }; 127 | -------------------------------------------------------------------------------- /lib/observables/producer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Kafka Producer Observable. 3 | * 4 | * @example producer 5 | * const producerObservable = require('./lib/producer'); 6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' }; 7 | * const observable = producerObservable.create('my_topic', 'message', opts); 8 | * 9 | * observable.subscribe(result => console.info(result)); 10 | * 11 | * @example producer - using arrays 12 | * const producerObservable = require('./lib/producer'); 13 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' }; 14 | * const messages = ['first message', 'second message']; 15 | * const observable = producerObservable.create('my_topic', messages, opts); 16 | * 17 | * observable.subscribe(result => console.info(result)); 18 | * 19 | * @example producer - using observables 20 | * const producerObservable = require('./lib/producer'); 21 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' }; 22 | * const messages = Observable.from(['first message', 'second message']); 23 | * const observable = producerObservable.create('my_topic', messages, opts); 24 | * 25 | * observable.subscribe(result => console.info(result)); 26 | * 27 | * @module observables/Producer 28 | * @author ghermeto 29 | **/ 30 | 31 | const defaultFactory = require('../client'); 32 | const Observable = require('rxjs').Observable; 33 | 34 | /** 35 | * Makes sure messages is formatted as an array or an observable 36 | * 37 | * @private 38 | * @param {*} messages 39 | * @return {Observable|Array} 40 | */ 41 | const formatMessages = 42 | messages => 43 | (messages instanceof Observable) ? messages : [].concat(messages); 44 | 45 | /** 46 | * Observable as result of publishing messages to Kafka 47 | * 48 | * @private 49 | * @param {String} topic 50 | * @param {Array|Observable} messages 51 | * @param {Object} producer 52 | */ 53 | const toTopic = 54 | (topic, messages, producer) => { 55 | const observable = Observable.create(observer => { 56 | Observable.from(messages) 57 | // formats and sends the message through kafka 58 | .concatMap(message => { 59 | const {value = message, key} = message; 60 | const result = producer.publish(topic, value, key); 61 | return Observable.fromPromise(result); 62 | }) 63 | // removes surrounding array and flattens 64 | .concatMap(result => Observable.from(result)) 65 | .catch(err => observer.error(err)) 66 | .subscribe({ 67 | next: result => observer.next(result), 68 | error: err => observer.error(err), 69 | complete: () => observer.complete() 70 | }); 71 | return () => { 72 | return producer.end(); 73 | }; 74 | }); 75 | 76 | observable._adapter = producer; 77 | return observable; 78 | }; 79 | 80 | /** 81 | * Producer factory. 82 | * 83 | * @param {String} topic topic name 84 | * @param {Array|Observable} messages 85 | * @param {Object} options 86 | * @param {Array|String} options.brokers comma separated list of brokers 87 | * @param {Object|defaultFactory} clientFactory client factory 88 | * @return {Observable} 89 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options. 90 | */ 91 | exports.create = (topic, messages, options, clientFactory = defaultFactory) => { 92 | try { 93 | const kafkaProducer = clientFactory.producer(options); 94 | return toTopic(topic, formatMessages(messages), kafkaProducer); 95 | } catch (err) { 96 | return Observable.throw(err); 97 | } 98 | }; 99 | -------------------------------------------------------------------------------- /lib/operators/json-message.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Observable operator: extracts and deserialize an object from a JSON message. 3 | * 4 | * @example 5 | * const opts = { brokers: 'kafka://127.0.0.1:9092' }; 6 | * const KafkaObservable = require('kafka-observable')(opts); 7 | * KafkaObservable.fromTopic('my_topic') 8 | * .let(KafkaObservable.JSONMessage()) 9 | * .subscribe(message => console.info(message)); 10 | * 11 | * @module operators/JSONMessage 12 | * @author ghermeto 13 | **/ 14 | 'use strict'; 15 | 16 | const Observable = require('rxjs').Observable; 17 | const kafkaMessage = require('./kafka-message'); 18 | 19 | const jsonMessage = (mapper = x => x) => 20 | (source) => 21 | Observable.create(observer => 22 | source 23 | .let(kafkaMessage()) 24 | .map(message => { 25 | try { return JSON.parse(message); } 26 | catch(err) { observer.error(err); } 27 | }) 28 | .subscribe( 29 | json => { 30 | try { observer.next(mapper(json)); } 31 | catch(err) { observer.error(err); } 32 | }, 33 | err => observer.error(err), 34 | () => observer.complete())); 35 | 36 | /** 37 | * @function JSONMessage 38 | * @param {Function} mapper mapping function 39 | */ 40 | module.exports = jsonMessage; -------------------------------------------------------------------------------- /lib/operators/kafka-message.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Observable operator: extracts a string value from a kafka message 3 | * 4 | * @example 5 | * const opts = { brokers: 'kafka://127.0.0.1:9092' }; 6 | * const KafkaObservable = require('kafka-observable')(opts); 7 | * KafkaObservable.fromTopic('my_topic') 8 | * .let(KafkaObservable.TextMessage()) 9 | * .subscribe(message => console.info(message)); 10 | * 11 | * @module operators/TextMessage 12 | * @author ghermeto 13 | **/ 14 | 'use strict'; 15 | 16 | const Observable = require('rxjs').Observable; 17 | 18 | const kafkaMessage = (mapper = x => x) => 19 | (source) => 20 | Observable.create(observer => 21 | source 22 | .map(({message}) => message.value.toString('utf8')) 23 | .subscribe( 24 | message => { 25 | try { observer.next(mapper(message)); } 26 | catch(err) { observer.error(err); } 27 | }, 28 | err => observer.error(err), 29 | () => observer.complete())); 30 | 31 | /** 32 | * @function TextMessage 33 | * @param {Function} mapper mapping function 34 | */ 35 | module.exports = kafkaMessage; -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-observable", 3 | "version": "0.0.3", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/bluebird": { 8 | "version": "3.5.0", 9 | "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.0.tgz", 10 | "integrity": "sha1-JjNHCk6r6aR82aRf2yDtX5NAe8o=" 11 | }, 12 | "@types/lodash": { 13 | "version": "4.14.107", 14 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.107.tgz", 15 | "integrity": "sha512-afvjfP2rl3yvtv2qrCRN23zIQcDinF+munMJCoHEw2BXF22QJogTlVfNPTACQ6ieDyA6VnyKT4WLuN/wK368ng==" 16 | }, 17 | "JSONStream": { 18 | "version": "1.3.2", 19 | "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz", 20 | "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=", 21 | "dev": true, 22 | "requires": { 23 | "jsonparse": "1.3.1", 24 | "through": "2.3.8" 25 | } 26 | }, 27 | "abbrev": { 28 | "version": "1.0.9", 29 | "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz", 30 | "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=", 31 | "dev": true 32 | }, 33 | "align-text": { 34 | "version": "0.1.4", 35 | "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", 36 | "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", 37 | "dev": true, 38 | "requires": { 39 | "kind-of": "3.2.2", 40 | "longest": "1.0.1", 41 | "repeat-string": "1.6.1" 42 | } 43 | }, 44 | "amdefine": { 45 | "version": "1.0.1", 46 | "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", 47 | "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", 48 | "dev": true 49 | }, 50 | "argparse": { 51 | "version": "1.0.10", 52 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", 53 | "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", 54 | "dev": true, 55 | "requires": { 56 | "sprintf-js": "1.0.3" 57 | } 58 | }, 59 | "async": { 60 | "version": "1.5.2", 61 | "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", 62 | "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", 63 | "dev": true 64 | }, 65 | "babylon": { 66 | "version": "7.0.0-beta.19", 67 | "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.19.tgz", 68 | "integrity": "sha512-Vg0C9s/REX6/WIXN37UKpv5ZhRi6A4pjHlpkE34+8/a6c2W1Q692n3hmc+SZG5lKRnaExLUbxtJ1SVT+KaCQ/A==", 69 | "dev": true 70 | }, 71 | "balanced-match": { 72 | "version": "1.0.0", 73 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 74 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" 75 | }, 76 | "bin-protocol": { 77 | "version": "3.0.4", 78 | "resolved": "https://registry.npmjs.org/bin-protocol/-/bin-protocol-3.0.4.tgz", 79 | "integrity": "sha1-RlqdNQb+sOEmtStbIWDZNuFbJ/Q=", 80 | "requires": { 81 | "lodash": "4.16.4", 82 | "long": "3.2.0", 83 | "protocol-buffers-schema": "3.3.2" 84 | } 85 | }, 86 | "bl": { 87 | "version": "1.2.2", 88 | "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", 89 | "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", 90 | "dev": true, 91 | "requires": { 92 | "readable-stream": "2.3.6", 93 | "safe-buffer": "5.1.1" 94 | }, 95 | "dependencies": { 96 | "process-nextick-args": { 97 | "version": "2.0.0", 98 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", 99 | "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", 100 | "dev": true 101 | }, 102 | "readable-stream": { 103 | "version": "2.3.6", 104 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", 105 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", 106 | "dev": true, 107 | "requires": { 108 | "core-util-is": "1.0.2", 109 | "inherits": "2.0.3", 110 | "isarray": "1.0.0", 111 | "process-nextick-args": "2.0.0", 112 | "safe-buffer": "5.1.1", 113 | "string_decoder": "1.1.1", 114 | "util-deprecate": "1.0.2" 115 | } 116 | }, 117 | "string_decoder": { 118 | "version": "1.1.1", 119 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 120 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 121 | "dev": true, 122 | "requires": { 123 | "safe-buffer": "5.1.1" 124 | } 125 | } 126 | } 127 | }, 128 | "bluebird": { 129 | "version": "3.5.1", 130 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 131 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 132 | }, 133 | "brace-expansion": { 134 | "version": "1.1.11", 135 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 136 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 137 | "requires": { 138 | "balanced-match": "1.0.0", 139 | "concat-map": "0.0.1" 140 | } 141 | }, 142 | "buffer-crc32": { 143 | "version": "0.2.13", 144 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", 145 | "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" 146 | }, 147 | "bunyan": { 148 | "version": "1.8.12", 149 | "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", 150 | "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", 151 | "requires": { 152 | "dtrace-provider": "0.8.6", 153 | "moment": "2.22.1", 154 | "mv": "2.1.1", 155 | "safe-json-stringify": "1.1.0" 156 | } 157 | }, 158 | "caller-id": { 159 | "version": "0.1.0", 160 | "resolved": "https://registry.npmjs.org/caller-id/-/caller-id-0.1.0.tgz", 161 | "integrity": "sha1-Wb2sCJPRLDhxQIJ5Ix+XRYNk8Hs=", 162 | "dev": true, 163 | "requires": { 164 | "stack-trace": "0.0.10" 165 | } 166 | }, 167 | "camelcase": { 168 | "version": "1.2.1", 169 | "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", 170 | "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=", 171 | "dev": true, 172 | "optional": true 173 | }, 174 | "catharsis": { 175 | "version": "0.8.9", 176 | "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.9.tgz", 177 | "integrity": "sha1-mMyJDKZS3S7w5ws3klMQ/56Q/Is=", 178 | "dev": true, 179 | "requires": { 180 | "underscore-contrib": "0.3.0" 181 | } 182 | }, 183 | "center-align": { 184 | "version": "0.1.3", 185 | "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", 186 | "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", 187 | "dev": true, 188 | "optional": true, 189 | "requires": { 190 | "align-text": "0.1.4", 191 | "lazy-cache": "1.0.4" 192 | } 193 | }, 194 | "cliui": { 195 | "version": "2.1.0", 196 | "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", 197 | "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", 198 | "dev": true, 199 | "optional": true, 200 | "requires": { 201 | "center-align": "0.1.3", 202 | "right-align": "0.1.3", 203 | "wordwrap": "0.0.2" 204 | }, 205 | "dependencies": { 206 | "wordwrap": { 207 | "version": "0.0.2", 208 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", 209 | "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=", 210 | "dev": true, 211 | "optional": true 212 | } 213 | } 214 | }, 215 | "concat-map": { 216 | "version": "0.0.1", 217 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 218 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" 219 | }, 220 | "concat-stream": { 221 | "version": "1.5.2", 222 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", 223 | "integrity": "sha1-cIl4Yk2FavQaWnQd790mHadSwmY=", 224 | "dev": true, 225 | "requires": { 226 | "inherits": "2.0.3", 227 | "readable-stream": "2.0.6", 228 | "typedarray": "0.0.6" 229 | } 230 | }, 231 | "connection-parse": { 232 | "version": "0.0.7", 233 | "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", 234 | "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=" 235 | }, 236 | "core-util-is": { 237 | "version": "1.0.2", 238 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 239 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", 240 | "dev": true 241 | }, 242 | "debug": { 243 | "version": "3.1.0", 244 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 245 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 246 | "dev": true, 247 | "requires": { 248 | "ms": "2.0.0" 249 | } 250 | }, 251 | "decamelize": { 252 | "version": "1.2.0", 253 | "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", 254 | "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", 255 | "dev": true, 256 | "optional": true 257 | }, 258 | "deep-is": { 259 | "version": "0.1.3", 260 | "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", 261 | "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", 262 | "dev": true 263 | }, 264 | "docker-modem": { 265 | "version": "1.0.6", 266 | "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.6.tgz", 267 | "integrity": "sha512-kDwWa5QaiVMB8Orbb7nXdGdwEZHKfEm7iPwglXe1KorImMpmGNlhC7A5LG0p8rrCcz1J4kJhq/o63lFjDdj8rQ==", 268 | "dev": true, 269 | "requires": { 270 | "JSONStream": "1.3.2", 271 | "debug": "3.1.0", 272 | "readable-stream": "1.0.34", 273 | "split-ca": "1.0.1" 274 | }, 275 | "dependencies": { 276 | "isarray": { 277 | "version": "0.0.1", 278 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", 279 | "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", 280 | "dev": true 281 | }, 282 | "readable-stream": { 283 | "version": "1.0.34", 284 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", 285 | "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", 286 | "dev": true, 287 | "requires": { 288 | "core-util-is": "1.0.2", 289 | "inherits": "2.0.3", 290 | "isarray": "0.0.1", 291 | "string_decoder": "0.10.31" 292 | } 293 | } 294 | } 295 | }, 296 | "dockerode": { 297 | "version": "2.5.5", 298 | "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.5.5.tgz", 299 | "integrity": "sha512-H3HX18xKmy51wqpPHvGDwPOotJMy9l/AWfiaVu4imrgBGr384rINEB2FwTwoYU++krkZjseVYyiVK8CnRz2tkw==", 300 | "dev": true, 301 | "requires": { 302 | "concat-stream": "1.5.2", 303 | "docker-modem": "1.0.6", 304 | "tar-fs": "1.12.0" 305 | } 306 | }, 307 | "dtrace-provider": { 308 | "version": "0.8.6", 309 | "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.6.tgz", 310 | "integrity": "sha1-QooiOv4DQl0s1tY0f99AxmkDVj0=", 311 | "optional": true, 312 | "requires": { 313 | "nan": "2.10.0" 314 | } 315 | }, 316 | "end-of-stream": { 317 | "version": "1.4.1", 318 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", 319 | "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", 320 | "dev": true, 321 | "requires": { 322 | "once": "1.4.0" 323 | } 324 | }, 325 | "escape-string-regexp": { 326 | "version": "1.0.5", 327 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 328 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", 329 | "dev": true 330 | }, 331 | "escodegen": { 332 | "version": "1.8.1", 333 | "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz", 334 | "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=", 335 | "dev": true, 336 | "requires": { 337 | "esprima": "2.7.3", 338 | "estraverse": "1.9.3", 339 | "esutils": "2.0.2", 340 | "optionator": "0.8.2", 341 | "source-map": "0.2.0" 342 | } 343 | }, 344 | "esprima": { 345 | "version": "2.7.3", 346 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", 347 | "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=", 348 | "dev": true 349 | }, 350 | "estraverse": { 351 | "version": "1.9.3", 352 | "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz", 353 | "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", 354 | "dev": true 355 | }, 356 | "esutils": { 357 | "version": "2.0.2", 358 | "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", 359 | "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", 360 | "dev": true 361 | }, 362 | "exit": { 363 | "version": "0.1.2", 364 | "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", 365 | "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", 366 | "dev": true 367 | }, 368 | "fast-levenshtein": { 369 | "version": "2.0.6", 370 | "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", 371 | "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", 372 | "dev": true 373 | }, 374 | "fs.realpath": { 375 | "version": "1.0.0", 376 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 377 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 378 | "dev": true 379 | }, 380 | "glob": { 381 | "version": "6.0.4", 382 | "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", 383 | "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", 384 | "optional": true, 385 | "requires": { 386 | "inflight": "1.0.6", 387 | "inherits": "2.0.3", 388 | "minimatch": "3.0.4", 389 | "once": "1.4.0", 390 | "path-is-absolute": "1.0.1" 391 | } 392 | }, 393 | "graceful-fs": { 394 | "version": "4.1.11", 395 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", 396 | "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", 397 | "dev": true 398 | }, 399 | "handlebars": { 400 | "version": "4.0.11", 401 | "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz", 402 | "integrity": "sha1-Ywo13+ApS8KB7a5v/F0yn8eYLcw=", 403 | "dev": true, 404 | "requires": { 405 | "async": "1.5.2", 406 | "optimist": "0.6.1", 407 | "source-map": "0.4.4", 408 | "uglify-js": "2.8.29" 409 | }, 410 | "dependencies": { 411 | "source-map": { 412 | "version": "0.4.4", 413 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", 414 | "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", 415 | "dev": true, 416 | "requires": { 417 | "amdefine": "1.0.1" 418 | } 419 | } 420 | } 421 | }, 422 | "has-flag": { 423 | "version": "1.0.0", 424 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", 425 | "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", 426 | "dev": true 427 | }, 428 | "hashring": { 429 | "version": "3.2.0", 430 | "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", 431 | "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", 432 | "requires": { 433 | "connection-parse": "0.0.7", 434 | "simple-lru-cache": "0.0.2" 435 | } 436 | }, 437 | "inflight": { 438 | "version": "1.0.6", 439 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 440 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 441 | "requires": { 442 | "once": "1.4.0", 443 | "wrappy": "1.0.2" 444 | } 445 | }, 446 | "inherits": { 447 | "version": "2.0.3", 448 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 449 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" 450 | }, 451 | "is-buffer": { 452 | "version": "1.1.6", 453 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", 454 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", 455 | "dev": true 456 | }, 457 | "isarray": { 458 | "version": "1.0.0", 459 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 460 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", 461 | "dev": true 462 | }, 463 | "isexe": { 464 | "version": "2.0.0", 465 | "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", 466 | "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", 467 | "dev": true 468 | }, 469 | "istanbul": { 470 | "version": "0.4.5", 471 | "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz", 472 | "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=", 473 | "dev": true, 474 | "requires": { 475 | "abbrev": "1.0.9", 476 | "async": "1.5.2", 477 | "escodegen": "1.8.1", 478 | "esprima": "2.7.3", 479 | "glob": "5.0.15", 480 | "handlebars": "4.0.11", 481 | "js-yaml": "3.11.0", 482 | "mkdirp": "0.5.1", 483 | "nopt": "3.0.6", 484 | "once": "1.4.0", 485 | "resolve": "1.1.7", 486 | "supports-color": "3.2.3", 487 | "which": "1.3.0", 488 | "wordwrap": "1.0.0" 489 | }, 490 | "dependencies": { 491 | "glob": { 492 | "version": "5.0.15", 493 | "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", 494 | "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=", 495 | "dev": true, 496 | "requires": { 497 | "inflight": "1.0.6", 498 | "inherits": "2.0.3", 499 | "minimatch": "3.0.4", 500 | "once": "1.4.0", 501 | "path-is-absolute": "1.0.1" 502 | } 503 | } 504 | } 505 | }, 506 | "jasmine": { 507 | "version": "2.99.0", 508 | "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-2.99.0.tgz", 509 | "integrity": "sha1-jKctEC5jm4Z8ZImFbg4YqceqQrc=", 510 | "dev": true, 511 | "requires": { 512 | "exit": "0.1.2", 513 | "glob": "7.1.2", 514 | "jasmine-core": "2.99.1" 515 | }, 516 | "dependencies": { 517 | "glob": { 518 | "version": "7.1.2", 519 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 520 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 521 | "dev": true, 522 | "requires": { 523 | "fs.realpath": "1.0.0", 524 | "inflight": "1.0.6", 525 | "inherits": "2.0.3", 526 | "minimatch": "3.0.4", 527 | "once": "1.4.0", 528 | "path-is-absolute": "1.0.1" 529 | } 530 | } 531 | } 532 | }, 533 | "jasmine-core": { 534 | "version": "2.99.1", 535 | "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-2.99.1.tgz", 536 | "integrity": "sha1-5kAN8ea1bhMLYcS80JPap/boyhU=", 537 | "dev": true 538 | }, 539 | "js-yaml": { 540 | "version": "3.11.0", 541 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz", 542 | "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==", 543 | "dev": true, 544 | "requires": { 545 | "argparse": "1.0.10", 546 | "esprima": "4.0.0" 547 | }, 548 | "dependencies": { 549 | "esprima": { 550 | "version": "4.0.0", 551 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", 552 | "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==", 553 | "dev": true 554 | } 555 | } 556 | }, 557 | "js2xmlparser": { 558 | "version": "3.0.0", 559 | "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-3.0.0.tgz", 560 | "integrity": "sha1-P7YOqgicVED5MZ9RdgzNB+JJlzM=", 561 | "dev": true, 562 | "requires": { 563 | "xmlcreate": "1.0.2" 564 | } 565 | }, 566 | "jsdoc": { 567 | "version": "3.5.5", 568 | "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.5.5.tgz", 569 | "integrity": "sha512-6PxB65TAU4WO0Wzyr/4/YhlGovXl0EVYfpKbpSroSj0qBxT4/xod/l40Opkm38dRHRdQgdeY836M0uVnJQG7kg==", 570 | "dev": true, 571 | "requires": { 572 | "babylon": "7.0.0-beta.19", 573 | "bluebird": "3.5.1", 574 | "catharsis": "0.8.9", 575 | "escape-string-regexp": "1.0.5", 576 | "js2xmlparser": "3.0.0", 577 | "klaw": "2.0.0", 578 | "marked": "0.3.19", 579 | "mkdirp": "0.5.1", 580 | "requizzle": "0.2.1", 581 | "strip-json-comments": "2.0.1", 582 | "taffydb": "2.6.2", 583 | "underscore": "1.8.3" 584 | } 585 | }, 586 | "jsonparse": { 587 | "version": "1.3.1", 588 | "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", 589 | "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", 590 | "dev": true 591 | }, 592 | "kind-of": { 593 | "version": "3.2.2", 594 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", 595 | "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", 596 | "dev": true, 597 | "requires": { 598 | "is-buffer": "1.1.6" 599 | } 600 | }, 601 | "klaw": { 602 | "version": "2.0.0", 603 | "resolved": "https://registry.npmjs.org/klaw/-/klaw-2.0.0.tgz", 604 | "integrity": "sha1-WcEo4Nxc5BAgEVEZTuucv4WGUPY=", 605 | "dev": true, 606 | "requires": { 607 | "graceful-fs": "4.1.11" 608 | } 609 | }, 610 | "lazy-cache": { 611 | "version": "1.0.4", 612 | "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", 613 | "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=", 614 | "dev": true, 615 | "optional": true 616 | }, 617 | "levn": { 618 | "version": "0.3.0", 619 | "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", 620 | "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", 621 | "dev": true, 622 | "requires": { 623 | "prelude-ls": "1.1.2", 624 | "type-check": "0.3.2" 625 | } 626 | }, 627 | "lodash": { 628 | "version": "4.16.4", 629 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.16.4.tgz", 630 | "integrity": "sha1-Ac4wa5utExnypVKGdPiCl663ASc=" 631 | }, 632 | "long": { 633 | "version": "3.2.0", 634 | "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", 635 | "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s=" 636 | }, 637 | "longest": { 638 | "version": "1.0.1", 639 | "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", 640 | "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=", 641 | "dev": true 642 | }, 643 | "marked": { 644 | "version": "0.3.19", 645 | "resolved": "https://registry.npmjs.org/marked/-/marked-0.3.19.tgz", 646 | "integrity": "sha512-ea2eGWOqNxPcXv8dyERdSr/6FmzvWwzjMxpfGB/sbMccXoct+xY+YukPD+QTUZwyvK7BZwcr4m21WBOW41pAkg==", 647 | "dev": true 648 | }, 649 | "minimatch": { 650 | "version": "3.0.4", 651 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 652 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 653 | "requires": { 654 | "brace-expansion": "1.1.11" 655 | } 656 | }, 657 | "minimist": { 658 | "version": "0.0.8", 659 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 660 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" 661 | }, 662 | "mkdirp": { 663 | "version": "0.5.1", 664 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 665 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 666 | "requires": { 667 | "minimist": "0.0.8" 668 | } 669 | }, 670 | "mock-require": { 671 | "version": "2.0.2", 672 | "resolved": "https://registry.npmjs.org/mock-require/-/mock-require-2.0.2.tgz", 673 | "integrity": "sha1-HqpxqtIwE3c9En3H6Ro/u0g31g0=", 674 | "dev": true, 675 | "requires": { 676 | "caller-id": "0.1.0" 677 | } 678 | }, 679 | "moment": { 680 | "version": "2.22.1", 681 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz", 682 | "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==", 683 | "optional": true 684 | }, 685 | "ms": { 686 | "version": "2.0.0", 687 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 688 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", 689 | "dev": true 690 | }, 691 | "murmur-hash-js": { 692 | "version": "1.0.0", 693 | "resolved": "https://registry.npmjs.org/murmur-hash-js/-/murmur-hash-js-1.0.0.tgz", 694 | "integrity": "sha1-UEEEkmnJZjPIZjhpYLL0KJ515bA=" 695 | }, 696 | "mv": { 697 | "version": "2.1.1", 698 | "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", 699 | "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", 700 | "optional": true, 701 | "requires": { 702 | "mkdirp": "0.5.1", 703 | "ncp": "2.0.0", 704 | "rimraf": "2.4.5" 705 | } 706 | }, 707 | "nan": { 708 | "version": "2.10.0", 709 | "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", 710 | "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==", 711 | "optional": true 712 | }, 713 | "ncp": { 714 | "version": "2.0.0", 715 | "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", 716 | "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", 717 | "optional": true 718 | }, 719 | "nice-simple-logger": { 720 | "version": "1.0.1", 721 | "resolved": "https://registry.npmjs.org/nice-simple-logger/-/nice-simple-logger-1.0.1.tgz", 722 | "integrity": "sha1-D55khSe+e+PkmrdvqMjAmK+VG/Y=", 723 | "requires": { 724 | "lodash": "4.16.4" 725 | } 726 | }, 727 | "no-kafka": { 728 | "version": "3.2.9", 729 | "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.2.9.tgz", 730 | "integrity": "sha1-jLSk8aDVDqYUXFvAZ6A1Dl5CmMc=", 731 | "requires": { 732 | "@types/bluebird": "3.5.0", 733 | "@types/lodash": "4.14.107", 734 | "bin-protocol": "3.0.4", 735 | "bluebird": "3.5.1", 736 | "buffer-crc32": "0.2.13", 737 | "hashring": "3.2.0", 738 | "lodash": "4.16.4", 739 | "murmur-hash-js": "1.0.0", 740 | "nice-simple-logger": "1.0.1", 741 | "wrr-pool": "1.1.3" 742 | } 743 | }, 744 | "nopt": { 745 | "version": "3.0.6", 746 | "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", 747 | "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", 748 | "dev": true, 749 | "requires": { 750 | "abbrev": "1.0.9" 751 | } 752 | }, 753 | "once": { 754 | "version": "1.4.0", 755 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 756 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 757 | "requires": { 758 | "wrappy": "1.0.2" 759 | } 760 | }, 761 | "optimist": { 762 | "version": "0.6.1", 763 | "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", 764 | "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", 765 | "dev": true, 766 | "requires": { 767 | "minimist": "0.0.8", 768 | "wordwrap": "0.0.3" 769 | }, 770 | "dependencies": { 771 | "wordwrap": { 772 | "version": "0.0.3", 773 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", 774 | "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", 775 | "dev": true 776 | } 777 | } 778 | }, 779 | "optionator": { 780 | "version": "0.8.2", 781 | "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", 782 | "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", 783 | "dev": true, 784 | "requires": { 785 | "deep-is": "0.1.3", 786 | "fast-levenshtein": "2.0.6", 787 | "levn": "0.3.0", 788 | "prelude-ls": "1.1.2", 789 | "type-check": "0.3.2", 790 | "wordwrap": "1.0.0" 791 | } 792 | }, 793 | "path-is-absolute": { 794 | "version": "1.0.1", 795 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 796 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" 797 | }, 798 | "prelude-ls": { 799 | "version": "1.1.2", 800 | "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", 801 | "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", 802 | "dev": true 803 | }, 804 | "process-nextick-args": { 805 | "version": "1.0.7", 806 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", 807 | "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", 808 | "dev": true 809 | }, 810 | "protocol-buffers-schema": { 811 | "version": "3.3.2", 812 | "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz", 813 | "integrity": "sha512-Xdayp8sB/mU+sUV4G7ws8xtYMGdQnxbeIfLjyO9TZZRJdztBGhlmbI5x1qcY4TG5hBkIKGnc28i7nXxaugu88w==" 814 | }, 815 | "pump": { 816 | "version": "1.0.3", 817 | "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", 818 | "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", 819 | "dev": true, 820 | "requires": { 821 | "end-of-stream": "1.4.1", 822 | "once": "1.4.0" 823 | } 824 | }, 825 | "readable-stream": { 826 | "version": "2.0.6", 827 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", 828 | "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", 829 | "dev": true, 830 | "requires": { 831 | "core-util-is": "1.0.2", 832 | "inherits": "2.0.3", 833 | "isarray": "1.0.0", 834 | "process-nextick-args": "1.0.7", 835 | "string_decoder": "0.10.31", 836 | "util-deprecate": "1.0.2" 837 | } 838 | }, 839 | "repeat-string": { 840 | "version": "1.6.1", 841 | "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", 842 | "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", 843 | "dev": true 844 | }, 845 | "requizzle": { 846 | "version": "0.2.1", 847 | "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.1.tgz", 848 | "integrity": "sha1-aUPDUwxNmn5G8c3dUcFY/GcM294=", 849 | "dev": true, 850 | "requires": { 851 | "underscore": "1.6.0" 852 | }, 853 | "dependencies": { 854 | "underscore": { 855 | "version": "1.6.0", 856 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", 857 | "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", 858 | "dev": true 859 | } 860 | } 861 | }, 862 | "resolve": { 863 | "version": "1.1.7", 864 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", 865 | "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", 866 | "dev": true 867 | }, 868 | "right-align": { 869 | "version": "0.1.3", 870 | "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", 871 | "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", 872 | "dev": true, 873 | "optional": true, 874 | "requires": { 875 | "align-text": "0.1.4" 876 | } 877 | }, 878 | "rimraf": { 879 | "version": "2.4.5", 880 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", 881 | "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", 882 | "optional": true, 883 | "requires": { 884 | "glob": "6.0.4" 885 | } 886 | }, 887 | "rxjs": { 888 | "version": "5.5.10", 889 | "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.10.tgz", 890 | "integrity": "sha512-SRjimIDUHJkon+2hFo7xnvNC4ZEHGzCRwh9P7nzX3zPkCGFEg/tuElrNR7L/rZMagnK2JeH2jQwPRpmyXyLB6A==", 891 | "requires": { 892 | "symbol-observable": "1.0.1" 893 | } 894 | }, 895 | "safe-buffer": { 896 | "version": "5.1.1", 897 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", 898 | "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", 899 | "dev": true 900 | }, 901 | "safe-json-stringify": { 902 | "version": "1.1.0", 903 | "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz", 904 | "integrity": "sha512-EzBtUaFH9bHYPc69wqjp0efJI/DPNHdFbGE3uIMn4sVbO0zx8vZ8cG4WKxQfOpUOKsQyGBiT2mTqnCw+6nLswA==", 905 | "optional": true 906 | }, 907 | "shortid": { 908 | "version": "2.2.8", 909 | "resolved": "https://registry.npmjs.org/shortid/-/shortid-2.2.8.tgz", 910 | "integrity": "sha1-AzsRfWoul1gE9vCWnb59PQs1UTE=" 911 | }, 912 | "simple-lru-cache": { 913 | "version": "0.0.2", 914 | "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", 915 | "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=" 916 | }, 917 | "source-map": { 918 | "version": "0.2.0", 919 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz", 920 | "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=", 921 | "dev": true, 922 | "optional": true, 923 | "requires": { 924 | "amdefine": "1.0.1" 925 | } 926 | }, 927 | "split-ca": { 928 | "version": "1.0.1", 929 | "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", 930 | "integrity": "sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=", 931 | "dev": true 932 | }, 933 | "sprintf-js": { 934 | "version": "1.0.3", 935 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", 936 | "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", 937 | "dev": true 938 | }, 939 | "stack-trace": { 940 | "version": "0.0.10", 941 | "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", 942 | "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", 943 | "dev": true 944 | }, 945 | "string_decoder": { 946 | "version": "0.10.31", 947 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", 948 | "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", 949 | "dev": true 950 | }, 951 | "strip-json-comments": { 952 | "version": "2.0.1", 953 | "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", 954 | "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", 955 | "dev": true 956 | }, 957 | "supports-color": { 958 | "version": "3.2.3", 959 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", 960 | "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=", 961 | "dev": true, 962 | "requires": { 963 | "has-flag": "1.0.0" 964 | } 965 | }, 966 | "symbol-observable": { 967 | "version": "1.0.1", 968 | "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.1.tgz", 969 | "integrity": "sha1-g0D8RwLDEi310iKI+IKD9RPT/dQ=" 970 | }, 971 | "taffydb": { 972 | "version": "2.6.2", 973 | "resolved": "https://registry.npmjs.org/taffydb/-/taffydb-2.6.2.tgz", 974 | "integrity": "sha1-fLy2S1oUG2ou/CxdLGe04VCyomg=", 975 | "dev": true 976 | }, 977 | "tar-fs": { 978 | "version": "1.12.0", 979 | "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.12.0.tgz", 980 | "integrity": "sha1-pqgFU9ilTHPeHQrg553ncDVgXh0=", 981 | "dev": true, 982 | "requires": { 983 | "mkdirp": "0.5.1", 984 | "pump": "1.0.3", 985 | "tar-stream": "1.5.5" 986 | } 987 | }, 988 | "tar-stream": { 989 | "version": "1.5.5", 990 | "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.5.tgz", 991 | "integrity": "sha512-mQdgLPc/Vjfr3VWqWbfxW8yQNiJCbAZ+Gf6GDu1Cy0bdb33ofyiNGBtAY96jHFhDuivCwgW1H9DgTON+INiXgg==", 992 | "dev": true, 993 | "requires": { 994 | "bl": "1.2.2", 995 | "end-of-stream": "1.4.1", 996 | "readable-stream": "2.0.6", 997 | "xtend": "4.0.1" 998 | } 999 | }, 1000 | "through": { 1001 | "version": "2.3.8", 1002 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 1003 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", 1004 | "dev": true 1005 | }, 1006 | "type-check": { 1007 | "version": "0.3.2", 1008 | "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", 1009 | "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", 1010 | "dev": true, 1011 | "requires": { 1012 | "prelude-ls": "1.1.2" 1013 | } 1014 | }, 1015 | "typedarray": { 1016 | "version": "0.0.6", 1017 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", 1018 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", 1019 | "dev": true 1020 | }, 1021 | "uglify-js": { 1022 | "version": "2.8.29", 1023 | "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", 1024 | "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", 1025 | "dev": true, 1026 | "optional": true, 1027 | "requires": { 1028 | "source-map": "0.5.7", 1029 | "uglify-to-browserify": "1.0.2", 1030 | "yargs": "3.10.0" 1031 | }, 1032 | "dependencies": { 1033 | "source-map": { 1034 | "version": "0.5.7", 1035 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", 1036 | "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", 1037 | "dev": true, 1038 | "optional": true 1039 | } 1040 | } 1041 | }, 1042 | "uglify-to-browserify": { 1043 | "version": "1.0.2", 1044 | "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", 1045 | "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", 1046 | "dev": true, 1047 | "optional": true 1048 | }, 1049 | "underscore": { 1050 | "version": "1.8.3", 1051 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", 1052 | "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=", 1053 | "dev": true 1054 | }, 1055 | "underscore-contrib": { 1056 | "version": "0.3.0", 1057 | "resolved": "https://registry.npmjs.org/underscore-contrib/-/underscore-contrib-0.3.0.tgz", 1058 | "integrity": "sha1-ZltmwkeD+PorGMn4y7Dix9SMJsc=", 1059 | "dev": true, 1060 | "requires": { 1061 | "underscore": "1.6.0" 1062 | }, 1063 | "dependencies": { 1064 | "underscore": { 1065 | "version": "1.6.0", 1066 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", 1067 | "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", 1068 | "dev": true 1069 | } 1070 | } 1071 | }, 1072 | "util-deprecate": { 1073 | "version": "1.0.2", 1074 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1075 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", 1076 | "dev": true 1077 | }, 1078 | "which": { 1079 | "version": "1.3.0", 1080 | "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz", 1081 | "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==", 1082 | "dev": true, 1083 | "requires": { 1084 | "isexe": "2.0.0" 1085 | } 1086 | }, 1087 | "window-size": { 1088 | "version": "0.1.0", 1089 | "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", 1090 | "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=", 1091 | "dev": true, 1092 | "optional": true 1093 | }, 1094 | "wordwrap": { 1095 | "version": "1.0.0", 1096 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", 1097 | "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", 1098 | "dev": true 1099 | }, 1100 | "wrappy": { 1101 | "version": "1.0.2", 1102 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1103 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 1104 | }, 1105 | "wrr-pool": { 1106 | "version": "1.1.3", 1107 | "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.3.tgz", 1108 | "integrity": "sha1-/a0i8uofMDY//l14HPeUl6d/8H4=", 1109 | "requires": { 1110 | "lodash": "4.16.4" 1111 | } 1112 | }, 1113 | "xmlcreate": { 1114 | "version": "1.0.2", 1115 | "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-1.0.2.tgz", 1116 | "integrity": "sha1-+mv3YqYKQT+z3Y9LA8WyaSONMI8=", 1117 | "dev": true 1118 | }, 1119 | "xtend": { 1120 | "version": "4.0.1", 1121 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", 1122 | "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", 1123 | "dev": true 1124 | }, 1125 | "yargs": { 1126 | "version": "3.10.0", 1127 | "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", 1128 | "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", 1129 | "dev": true, 1130 | "optional": true, 1131 | "requires": { 1132 | "camelcase": "1.2.1", 1133 | "cliui": "2.1.0", 1134 | "decamelize": "1.2.0", 1135 | "window-size": "0.1.0" 1136 | } 1137 | } 1138 | } 1139 | } 1140 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-observable", 3 | "version": "0.0.3", 4 | "description": "kafka consumer and producer as observables", 5 | "main": "index.js", 6 | "scripts": { 7 | "prepublish": "npm test", 8 | "test": "npm run unit-test", 9 | "clean": "rm -rf ./node_modules ./out ./coverage", 10 | "gen-docs": "node_modules/.bin/jsdoc -r -R README.md -P package.json index.js lib/observables/* lib/operators/*", 11 | "coverage": "JASMINE_CONFIG_PATH=test/support/unit.json node_modules/.bin/istanbul cover --include-all-sources node_modules/.bin/jasmine", 12 | "unit-test": "JASMINE_CONFIG_PATH=test/support/unit.json node_modules/.bin/jasmine", 13 | "integration-test": "LOG_LEVEL=trace JASMINE_CONFIG_PATH=test/support/integration.json node_modules/.bin/jasmine" 14 | }, 15 | "engines": { 16 | "node": ">=6.4.0" 17 | }, 18 | "keywords": [ 19 | "kafka", 20 | "rx", 21 | "observable" 22 | ], 23 | "author": "gui.hermeto@gmail.com", 24 | "license": "MIT", 25 | "repository": { 26 | "type": "git", 27 | "url": "https://github.com/ghermeto/kafka-observable.git" 28 | }, 29 | "publishConfig": { 30 | "registry": "https://registry.npmjs.org/" 31 | }, 32 | "dependencies": { 33 | "bunyan": "^1.8.12", 34 | "no-kafka": "^3.2.6", 35 | "rxjs": "^5.5.2", 36 | "shortid": "^2.2.8" 37 | }, 38 | "devDependencies": { 39 | "dockerode": "^2.5.3", 40 | "istanbul": "^0.4.5", 41 | "jasmine": "^2.8.0", 42 | "jsdoc": "^3.5.5", 43 | "mock-require": "^2.0.2" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /test/integration/helpers/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Docker = require('dockerode'); 4 | jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000; 5 | 6 | const topic = 'test_kafka'; 7 | const createTopic = () => ([ 8 | '/opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh', 9 | '--create', 10 | '--zookeeper', 'localhost:2181', 11 | '--replication-factor', '1', 12 | '--partitions', '1', 13 | '--topic', topic 14 | ]); 15 | 16 | /** 17 | * @type {Docker} 18 | */ 19 | global.docker = new Docker(); 20 | 21 | /** 22 | * end handler for jasmine 23 | * @param {Function} done jasmine done function 24 | * @return {Function} handler that calls done the expected way 25 | **/ 26 | global.finish_test = function finish_test(done) { 27 | return function (err) { 28 | if (err) { done.fail(err); } 29 | done(); 30 | }; 31 | }; 32 | 33 | /** 34 | * delay func 35 | * @param {Number} millis time in milliseconds 36 | * @return {Promise} resolves after given time 37 | */ 38 | global.delay = millis => 39 | new Promise((resolve) => setTimeout(()=> resolve(), millis)); 40 | 41 | /** 42 | * Before integration tests starts, create the container, start it and create 43 | * the topic in kafka required for testing. 44 | */ 45 | beforeAll(function (done) { 46 | docker.createContainer({ 47 | Image: 'spotify/kafka', 48 | AttachStdout: true, 49 | AttachStderr: true, 50 | Hostname: 'kafka', 51 | StopTimeout: 5, 52 | ExposedPorts: { '9092/tcp':{} }, 53 | HostConfig: { PortBindings: { '9092/tcp': [{ HostPort: '9092' }] } } 54 | }) 55 | .then(container => container.start()) 56 | .then(container => global.container = container) 57 | .then(() => console.info('waiting for kafka to start...') || delay(6000)) 58 | .then(() => container.exec({ Cmd: createTopic() })) 59 | .then(exec => exec.start()) 60 | .then(() => delay(3000)) 61 | .then(() => console.info(`${topic} topic created...`) || done()); 62 | }); 63 | 64 | /** 65 | * test to see if container was successfully started 66 | */ 67 | it('should have a valid container', done => { 68 | container.inspect(function (err, data) { 69 | if (err || !data) { done.fail(err || 'no container data'); } 70 | done(); 71 | }); 72 | }); 73 | 74 | /** 75 | * shutdown procedure. We need to stop and remove the container not only on 76 | * afterAll, but anytime we stop test tests. 77 | */ 78 | const gracefulShutdown = () => global.container 79 | .stop() 80 | .then(container => container.remove()); 81 | 82 | process.on('SIGTERM', gracefulShutdown); 83 | process.on('SIGINT', gracefulShutdown); 84 | 85 | afterAll(done => gracefulShutdown().then(done)); 86 | -------------------------------------------------------------------------------- /test/integration/observables/consumer.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const kafkaClient = require('../../../lib/client'); 5 | const consumerObservable = require('../../../lib/observables/consumer'); 6 | 7 | /** 8 | * brokerRedirection is required because no-kafka client will try to use 9 | * container given hostname and not the ip specified in brokers 10 | */ 11 | const opts = { 12 | groupId: 'test', 13 | brokers: '127.0.0.1:9092', 14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' } 15 | }; 16 | 17 | const producer = kafkaClient.producer(opts); 18 | 19 | describe('consumer', () => { 20 | let subscription; 21 | 22 | describe('without autoCommit', () => { 23 | let options; 24 | 25 | beforeEach(()=> { 26 | options = Object.assign({ autoCommit: false }, opts); 27 | }); 28 | 29 | it('should get a message as an observable', done => { 30 | const observable = consumerObservable.create('test_kafka', options); 31 | subscription = observable 32 | .subscribe( 33 | ({ message, commit }) => { 34 | const msg = message.value.toString('utf8'); 35 | expect(msg).toEqual('my message'); 36 | commit().subscribe(() => done()); 37 | }, 38 | err => done.fail(err) 39 | ); 40 | 41 | delay(4000) 42 | .then(() => producer.publish('test_kafka', 'my message')); 43 | }); 44 | 45 | it('should be able to map messages', done => { 46 | const observable = consumerObservable.create('test_kafka', options); 47 | subscription = observable 48 | .map( ({ message, commit }) => ({ 49 | commit, 50 | message: message.value.toString('utf8'), 51 | mapped: true 52 | }) ) 53 | .subscribe( 54 | ({ message, commit, mapped }) => { 55 | expect(mapped).toBe(true); 56 | expect(message).toEqual('my message'); 57 | commit().subscribe(() => done()); 58 | }, 59 | err => done.fail(err) 60 | ); 61 | 62 | delay(5000) 63 | .then(() => producer.publish('test_kafka', 'my message')); 64 | }); 65 | 66 | it('should be able to reduce messages', done => { 67 | const observable = consumerObservable.create('test_kafka', options); 68 | subscription = observable 69 | .do(m => m.commit().subscribe()) 70 | .takeUntil(Observable.interval(6000)) 71 | .reduce((acc, m) => acc + 1, 0) 72 | .subscribe(number => { 73 | expect(number).toBe(3); 74 | done(); 75 | }); 76 | 77 | delay(5000) 78 | .then(() => producer.publish('test_kafka', 'first message')) 79 | .then(() => producer.publish('test_kafka', 'second message')) 80 | .then(() => producer.publish('test_kafka', 'third message')); 81 | }); 82 | 83 | it('should filter messages', done => { 84 | const observable = consumerObservable.create('test_kafka', options); 85 | subscription = observable 86 | /** 87 | * here are have to concatMap through the commit because ".take" 88 | * will immediately call unsubscribe and close the connection, 89 | * so the last commit will fail if done after ".take" 90 | */ 91 | .concatMap(m => m.commit().map(() => m)) 92 | .take(4) 93 | .map(({message}) => message.value.toString('utf8')) 94 | .filter(message => message.startsWith('a')) 95 | .reduce((acc, message) => acc + 1, 0) 96 | .subscribe(number => { 97 | expect(number).toBe(2); 98 | done(); 99 | }); 100 | 101 | delay(5000) 102 | .then(() => producer.publish('test_kafka', 'banana')) 103 | .then(() => producer.publish('test_kafka', 'avocado')) 104 | .then(() => delay(500)) 105 | .then(() => producer.publish('test_kafka', 'coconut')) 106 | .then(() => producer.publish('test_kafka', 'apple')); 107 | }); 108 | }); 109 | 110 | describe('with autoCommit', () => { 111 | const getOffset = observable => 112 | observable 113 | ._adapter 114 | .client() 115 | .offset('test_kafka'); 116 | 117 | it('should get a message as an observable', done => { 118 | const observable = consumerObservable.create('test_kafka', opts); 119 | subscription = observable 120 | .subscribe( 121 | ({ message }) => { 122 | const msg = message.value.toString('utf8'); 123 | expect(msg).toEqual('my message'); 124 | done(); 125 | }, 126 | err => done.fail(err) 127 | ); 128 | 129 | delay(5000) 130 | .then(() => producer.publish('test_kafka', 'my message')); 131 | }); 132 | 133 | it('should commit the offset', done => { 134 | const observable = consumerObservable.create('test_kafka', opts); 135 | subscription = observable 136 | .take(1) 137 | .map(({ message }) => message.value.toString('utf8')) 138 | .subscribe( 139 | msg => expect(msg).toEqual('my message'), 140 | err => done.fail(err) 141 | ); 142 | 143 | getOffset(observable) 144 | .then(startOffset => { 145 | subscription.add(() => { 146 | getOffset(observable) 147 | .then(offset => expect(offset).toBe(startOffset + 1)) 148 | .then(done); 149 | }); 150 | }); 151 | 152 | delay(5000) 153 | .then(() => producer.publish('test_kafka', 'my message')); 154 | }); 155 | 156 | it('should filter messages', done => { 157 | const observable = consumerObservable.create('test_kafka', opts); 158 | subscription = observable 159 | .take(4) 160 | .map(({message}) => message.value.toString('utf8')) 161 | .filter(message => message.startsWith('a')) 162 | .reduce((acc, message) => acc + 1, 0) 163 | .subscribe(number => { 164 | expect(number).toBe(2); 165 | done(); 166 | }); 167 | 168 | delay(5000) 169 | .then(() => producer.publish('test_kafka', 'banana')) 170 | .then(() => producer.publish('test_kafka', 'avocado')) 171 | .then(() => producer.publish('test_kafka', 'coconut')) 172 | .then(() => producer.publish('test_kafka', 'apple')); 173 | }); 174 | 175 | it('should not commit message if observer throws', done => { 176 | const observable = consumerObservable.create('test_kafka', opts); 177 | const adapter = observable._adapter; 178 | spyOn(adapter, 'commit').and.callThrough(); 179 | 180 | subscription = observable 181 | .take(1) 182 | .map(({ message }) => message.value.toString('utf8')) 183 | .catch(err => done.fail(err)) 184 | .subscribe({ 185 | next: () => { throw "Ops!" }, 186 | error: err => done.fail(err) 187 | }); 188 | 189 | delay(5000) 190 | .then(() => producer.publish('test_kafka', 'my message')) 191 | .then(() => subscription.add(() => { 192 | expect(adapter.commit).not.toHaveBeenCalled(); 193 | done(); 194 | })); 195 | }); 196 | 197 | }); 198 | 199 | afterEach(() => { 200 | subscription.unsubscribe(); 201 | }); 202 | 203 | }); -------------------------------------------------------------------------------- /test/integration/observables/producer.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const consumerObservable = require('../../../lib/observables/consumer'); 5 | const producerObservable = require('../../../lib/observables/producer'); 6 | 7 | /** 8 | * brokerRedirection is required because no-kafka client will try to use 9 | * container given hostname and not the ip specified in brokers 10 | */ 11 | const opts = { 12 | groupId: 'test', 13 | brokers: '127.0.0.1:9092', 14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' } 15 | }; 16 | 17 | describe('producer', () => { 18 | let consumerSubscription, producerSubscription; 19 | 20 | it('should send a single message', done => { 21 | const msg = 'producer1'; 22 | const consumer = consumerObservable.create('test_kafka', opts); 23 | const producer = producerObservable.create('test_kafka', msg, opts); 24 | 25 | consumerSubscription = consumer 26 | .takeUntil(Observable.interval(6000)) 27 | .subscribe(() => {}, err => done.fail(err), () => done()); 28 | 29 | delay(5000) 30 | .then(() => { 31 | producerSubscription = producer 32 | .subscribe( 33 | ({topic, error}) => { 34 | expect(topic).toEqual('test_kafka'); 35 | expect(error).toBeNull(); 36 | }, 37 | err => done.fail(err) 38 | ); 39 | }); 40 | }); 41 | 42 | it('should send many messages', done => { 43 | const msgs = ['first', 'second']; 44 | const consumer = consumerObservable.create('test_kafka', opts); 45 | const producer = producerObservable.create('test_kafka', msgs, opts); 46 | 47 | consumerSubscription = consumer 48 | .take(2) 49 | .subscribe(() => {}, err => done.fail(err), () => done()); 50 | 51 | delay(5000) 52 | .then(() => { 53 | producerSubscription = producer 54 | .subscribe( 55 | ({topic, error}) => { 56 | expect(topic).toEqual('test_kafka'); 57 | expect(error).toBeNull(); 58 | }, 59 | err => done.fail(err) 60 | ); 61 | }); 62 | }); 63 | 64 | it('should send messages from an observable', done => { 65 | const msgs = Observable.from(['first', 'second', 'third']); 66 | const consumer = consumerObservable.create('test_kafka', opts); 67 | const producer = producerObservable.create('test_kafka', msgs, opts); 68 | 69 | consumerSubscription = consumer 70 | .take(3) 71 | .subscribe(() => {}, err => done.fail(err), () => done()); 72 | 73 | delay(5000) 74 | .then(() => { 75 | producerSubscription = producer 76 | .subscribe( 77 | ({topic, error}) => { 78 | expect(topic).toEqual('test_kafka'); 79 | expect(error).toBeNull(); 80 | }, 81 | err => done.fail(err) 82 | ); 83 | }); 84 | }); 85 | 86 | afterEach(() => { 87 | consumerSubscription.unsubscribe(); 88 | producerSubscription.unsubscribe(); 89 | }); 90 | 91 | }); 92 | -------------------------------------------------------------------------------- /test/integration/operators/json-message.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const kafkaClient = require('../../../lib/client'); 5 | const KafkaObservable = require('../../../index'); 6 | 7 | /** 8 | * brokerRedirection is required because no-kafka client will try to use 9 | * container given hostname and not the ip specified in brokers 10 | */ 11 | const opts = { 12 | groupId: 'test', 13 | brokers: '127.0.0.1:9092', 14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' } 15 | }; 16 | 17 | const producer = kafkaClient.producer(opts); 18 | 19 | describe('JSONMessage', () => { 20 | let subscription; 21 | 22 | it('should get a message value as JSON', done => { 23 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 24 | subscription = observable 25 | .take(1) 26 | .let(KafkaObservable.JSONMessage()) 27 | .subscribe( 28 | json => { 29 | expect(json.key).toBeDefined(); 30 | expect(json.key).toEqual('test-value'); 31 | }, 32 | err => done.fail(err), 33 | () => done() 34 | ); 35 | 36 | delay(5000) 37 | .then(() => producer.publish('test_kafka', {key: 'test-value'})); 38 | }); 39 | 40 | it('should be available when called in the instance', done => { 41 | const kafka = KafkaObservable(opts); 42 | const observable = kafka.fromTopic('test_kafka'); 43 | subscription = observable 44 | .take(1) 45 | .let(kafka.JSONMessage()) 46 | .subscribe( 47 | json => { 48 | expect(json.key).toBeDefined(); 49 | expect(json.key).toEqual('test-value'); 50 | }, 51 | err => done.fail(err), 52 | () => done() 53 | ); 54 | 55 | delay(5000) 56 | .then(() => producer.publish('test_kafka', {key: 'test-value'})); 57 | }); 58 | 59 | afterEach(() => { 60 | subscription.unsubscribe(); 61 | }); 62 | 63 | }); -------------------------------------------------------------------------------- /test/integration/operators/kafka-message.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const kafkaClient = require('../../../lib/client'); 5 | const KafkaObservable = require('../../../index'); 6 | 7 | /** 8 | * brokerRedirection is required because no-kafka client will try to use 9 | * container given hostname and not the ip specified in brokers 10 | */ 11 | const opts = { 12 | groupId: 'test', 13 | brokers: '127.0.0.1:9092', 14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' } 15 | }; 16 | 17 | const producer = kafkaClient.producer(opts); 18 | 19 | describe('TextMessage', () => { 20 | let subscription; 21 | 22 | it('should get a message value as text', done => { 23 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 24 | subscription = observable 25 | .take(1) 26 | .let(KafkaObservable.TextMessage()) 27 | .subscribe( 28 | msg => expect(msg).toEqual('my message'), 29 | err => done.fail(err), 30 | () => done() 31 | ); 32 | 33 | delay(4000) 34 | .then(() => producer.publish('test_kafka', 'my message')); 35 | }); 36 | 37 | afterEach(() => { 38 | subscription.unsubscribe(); 39 | }); 40 | 41 | }); -------------------------------------------------------------------------------- /test/support/integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec_dir": "test/integration", 3 | "spec_files": [ 4 | "**/*.spec.js" 5 | ], 6 | "helpers": [ 7 | "helpers/**/*.js" 8 | ], 9 | "stopSpecOnExpectationFailure": true, 10 | "random": false 11 | } 12 | -------------------------------------------------------------------------------- /test/support/unit.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec_dir": "test/unit", 3 | "spec_files": [ 4 | "**/*.spec.js" 5 | ], 6 | "helpers": [ 7 | "mock/**/*.js", 8 | "helpers/**/*.js" 9 | ], 10 | "stopSpecOnExpectationFailure": true, 11 | "random": false 12 | } 13 | -------------------------------------------------------------------------------- /test/unit/consumer.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | 5 | const consumerObservable = require('../../lib/observables/consumer'); 6 | const opts = { 7 | groupId: 'test', 8 | brokers: '127.0.0.1:9092' 9 | }; 10 | 11 | describe('consumer', () => { 12 | let subscription, observable, producer; 13 | beforeEach(() => { 14 | observable = consumerObservable.create('test_kafka', opts); 15 | producer = mockProducer(); 16 | }); 17 | 18 | it('should get a message as an observable', done => { 19 | subscription = observable 20 | .subscribe( 21 | ({ message }) => { 22 | const msg = message.value.toString('utf8'); 23 | expect(msg).toEqual('my message'); 24 | done(); 25 | }, 26 | err => done.fail(err) 27 | ); 28 | 29 | delay(100) 30 | .then(() => producer.publish('test_kafka', 'my message')); 31 | }); 32 | 33 | it('should automatically commit message', done => { 34 | const adapter = observable._adapter; 35 | spyOn(adapter, 'commit').and.callThrough(); 36 | subscription = observable 37 | .take(1) 38 | .subscribe(); 39 | 40 | delay(100) 41 | .then(() => producer.publish('test_kafka', 'my message')) 42 | .then(() => subscription.add(() => { 43 | expect(adapter.commit).toHaveBeenCalled(); 44 | done(); 45 | })); 46 | }); 47 | 48 | it('should be able to map messages', done => { 49 | subscription = observable 50 | .map( ({ message }) => ({ 51 | message: message.value.toString('utf8'), 52 | mapped: true 53 | }) ) 54 | .subscribe( 55 | ({ message, mapped }) => { 56 | expect(mapped).toBe(true); 57 | expect(message).toEqual('my message'); 58 | done() 59 | }, 60 | err => done.fail(err) 61 | ); 62 | 63 | delay(100) 64 | .then(() => producer.publish('test_kafka', 'my message')); 65 | }); 66 | 67 | it('should be able to reduce messages', done => { 68 | subscription = observable 69 | .takeUntil(Observable.interval(500)) 70 | .reduce((acc, m) => acc + 1, 0) 71 | .subscribe(number => { 72 | expect(number).toBe(3); 73 | done(); 74 | }); 75 | 76 | delay(100) 77 | .then(() => producer.publish('test_kafka', 'first message')) 78 | .then(() => producer.publish('test_kafka', 'second message')) 79 | .then(() => producer.publish('test_kafka', 'third message')); 80 | }); 81 | 82 | it('should filter messages', done => { 83 | subscription = observable 84 | .take(4) 85 | .map(({message}) => message.value.toString('utf8')) 86 | .filter(message => message.startsWith('a')) 87 | .reduce((acc, message) => acc + 1, 0) 88 | .subscribe(number => { 89 | expect(number).toBe(2); 90 | done(); 91 | }); 92 | 93 | delay(100) 94 | .then(() => producer.publish('test_kafka', 'abacate')) 95 | .then(() => producer.publish('test_kafka', 'banana')) 96 | .then(() => producer.publish('test_kafka', 'coconut')) 97 | .then(() => producer.publish('test_kafka', 'apple')); 98 | }); 99 | 100 | it('should catch if an error is thrown before subscribe', done => { 101 | subscription = observable 102 | .map(({message}) => { 103 | const msg = message.value.toString('utf8'); 104 | if (msg ==='throw') { throw 'Ops!' } 105 | return msg; 106 | }) 107 | .catch(err => Observable.of('pass')) 108 | .subscribe(message => message !== 'throw' ? done() : done.fail()); 109 | 110 | delay(100) 111 | .then(() => producer.publish('test_kafka', 'throw')); 112 | }); 113 | 114 | it('should log error observer throw', done => { 115 | spyOn(bunyan, 'error'); 116 | 117 | subscription = observable 118 | .subscribe(() => { throw 'Ops!'; }); 119 | 120 | delay(100) 121 | .then(() => producer.publish('test_kafka', 'throw')) 122 | .then(() => subscription.add(()=> { 123 | expect(bunyan.error).toHaveBeenCalled(); 124 | done(); 125 | })); 126 | }); 127 | 128 | it('observable has a non standard _adapter attached to it', () => { 129 | expect(observable._adapter).toBeDefined(); 130 | }); 131 | 132 | it('should use DefaultAssignmentStrategy if strategy is not found', done => { 133 | const options = Object.assign({strategy: 'ops'}, opts); 134 | observable = consumerObservable.create('test_kafka', options); 135 | subscription = observable 136 | .subscribe(() => { 137 | const client = observable._adapter.client(); 138 | const strategy = client.strategies[0].strategy.constructor.name; 139 | expect(strategy).toEqual('DefaultAssignmentStrategy'); 140 | done(); 141 | }); 142 | 143 | delay(100) 144 | .then(() => producer.publish('test_kafka', 'message')); 145 | }); 146 | 147 | it('should error if strategy is invalid', done => { 148 | const options = Object.assign({strategy: 'ops'}, opts); 149 | observable = consumerObservable.create('test_kafka', options); 150 | subscription = observable 151 | .subscribe(() => { 152 | const client = observable._adapter.client(); 153 | const strategy = client.strategies[0].strategy.constructor.name; 154 | expect(strategy).toEqual('DefaultAssignmentStrategy'); 155 | done(); 156 | }); 157 | 158 | delay(100) 159 | .then(() => producer.publish('test_kafka', 'message')); 160 | }); 161 | 162 | it('should error if groupId is not defined', done => { 163 | const options = Object.assign({}, opts, { groupId: undefined }); 164 | observable = consumerObservable.create('test_kafka', options); 165 | subscription = observable 166 | .subscribe({ next: () => done.fail(), error: err => done() }); 167 | }); 168 | 169 | it('should error if brokers is not defined', done => { 170 | const options = Object.assign({}, opts, { brokers: undefined }); 171 | observable = consumerObservable.create('test_kafka', options); 172 | subscription = observable 173 | .subscribe({ next: () => done.fail(), error: err => done() }); 174 | }); 175 | 176 | it('should error if brokers is not a string or an array', done => { 177 | const options = Object.assign({}, opts, { brokers: {} }); 178 | observable = consumerObservable.create('test_kafka', options); 179 | subscription = observable 180 | .subscribe({ next: () => done.fail(), error: err => done() }); 181 | }); 182 | 183 | it('should error if initial client subscription fails', done => { 184 | const options = Object.assign({metadata: 1}, opts); 185 | observable = consumerObservable.create('test_kafka', options); 186 | subscription = observable 187 | .subscribe({ next: () => done.fail(), error: err => done() }); 188 | }); 189 | 190 | it('should work for null message', done => { 191 | observable = consumerObservable.create('test_kafka', opts); 192 | subscription = observable 193 | .subscribe(({message}) => { 194 | expect(message.value).toBeNull(); 195 | done(); 196 | }); 197 | 198 | delay(100) 199 | .then(() => producer.publish('test_kafka', null)); 200 | }); 201 | 202 | describe('without auto-commit', () => { 203 | let options; 204 | 205 | beforeEach(() => { 206 | options = Object.assign({ autoCommit: false }, opts) ; 207 | observable = consumerObservable.create('test_kafka', options); 208 | }); 209 | 210 | it('should have a commit function', done => { 211 | subscription = observable 212 | .subscribe( 213 | ({ commit }) => { 214 | expect(commit).toBeDefined(); 215 | expect(typeof commit).toEqual('function'); 216 | expect(commit() instanceof Observable).toBe(true); 217 | done(); 218 | }, 219 | err => done.fail(err) 220 | ); 221 | 222 | delay(100) 223 | .then(() => producer.publish('test_kafka', 'my message')); 224 | }); 225 | 226 | it('should not commit message', done => { 227 | const adapter = observable._adapter; 228 | spyOn(adapter, 'commit').and.callThrough(); 229 | subscription = observable 230 | .take(1) 231 | .subscribe(); 232 | 233 | delay(100) 234 | .then(() => producer.publish('test_kafka', 'my message')) 235 | .then(() => subscription.add(() => { 236 | expect(adapter.commit).not.toHaveBeenCalled(); 237 | done(); 238 | })); 239 | }); 240 | }); 241 | 242 | 243 | afterEach(() => { 244 | subscription.unsubscribe(); 245 | }); 246 | 247 | }); -------------------------------------------------------------------------------- /test/unit/helpers/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * end handler for jasmine 5 | * @param {Function} done jasmine done function 6 | * @return {Function} handler that calls done the expected way 7 | **/ 8 | global.finish_test = function finish_test(done) { 9 | return function (err) { 10 | if (err) { done.fail(err); } 11 | done(); 12 | }; 13 | }; 14 | 15 | /** 16 | * delay func 17 | * @param {Number} millis time in milliseconds 18 | * @return {Promise} resolves after given time 19 | */ 20 | global.delay = millis => 21 | new Promise((resolve) => setTimeout(()=> resolve(), millis)); 22 | -------------------------------------------------------------------------------- /test/unit/kafka-observable.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const KafkaObservable = require('../../index'); 5 | 6 | const opts = { 7 | groupId: 'test', 8 | brokers: '127.0.0.1:9092' 9 | }; 10 | 11 | describe('fromTopic', () => { 12 | let subscription; 13 | beforeEach(() => { 14 | 15 | }); 16 | 17 | it('should inherit options', done => { 18 | const producer = mockProducer(); 19 | const consumer = KafkaObservable(opts).fromTopic('test_kafka'); 20 | 21 | subscription = consumer 22 | .take(1) 23 | .map(({message}) => message.value.toString('utf8')) 24 | .subscribe( 25 | message => expect(message).toEqual('my message'), 26 | err => done.fail(err), 27 | () => done() 28 | ); 29 | 30 | delay(100) 31 | .then(() => producer.publish('test_kafka', 'my message')); 32 | }); 33 | 34 | it('should accept options', done => { 35 | const producer = mockProducer(); 36 | const consumer = KafkaObservable.fromTopic('test_kafka', opts); 37 | 38 | subscription = consumer 39 | .take(1) 40 | .map(({message}) => message.value.toString('utf8')) 41 | .subscribe( 42 | message => expect(message).toEqual('my message'), 43 | err => done.fail(err), 44 | () => done() 45 | ); 46 | 47 | delay(100) 48 | .then(() => producer.publish('test_kafka', 'my message')); 49 | }); 50 | 51 | afterEach(() => { 52 | subscription.unsubscribe(); 53 | }); 54 | 55 | }); 56 | 57 | describe('toTopic', () => { 58 | let consumerSubscription, producerSubscription; 59 | 60 | it('should send a message', done => { 61 | const msg = 'message'; 62 | const observable = KafkaObservable(opts); 63 | const consumer = observable.fromTopic('test_kafka'); 64 | const producer = observable.toTopic('test_kafka', msg); 65 | 66 | consumerSubscription = consumer 67 | .take(1) 68 | .subscribe(() => {}, err => done.fail(err), () => done()); 69 | 70 | delay(100) 71 | .then(() => { 72 | producerSubscription = producer 73 | .subscribe( 74 | ({topic, error}) => { 75 | expect(topic).toEqual('test_kafka'); 76 | expect(error).toBeNull(); 77 | }, 78 | err => done.fail(err) 79 | ); 80 | }); 81 | }); 82 | 83 | afterEach(() => { 84 | consumerSubscription.unsubscribe(); 85 | producerSubscription.unsubscribe(); 86 | }); 87 | }); -------------------------------------------------------------------------------- /test/unit/mock/mock-bunyan.js: -------------------------------------------------------------------------------- 1 | /** 2 | * mocks no-kafka module 3 | * @author ghermeto 4 | * @version 0.0.1 5 | **/ 6 | const mock = require('mock-require'); 7 | 8 | const bunyan = { 9 | log: () => {}, 10 | trace: () => {}, 11 | debug: () => {}, 12 | info: () => {}, 13 | warn: () => {}, 14 | error: () => {} 15 | }; 16 | 17 | mock('bunyan', { createLogger: () => bunyan }); 18 | mock.reRequire('bunyan'); 19 | 20 | module.exports = { bunyan }; 21 | beforeAll(() => { 22 | global.bunyan = bunyan; 23 | }); 24 | -------------------------------------------------------------------------------- /test/unit/mock/mock-no-kafka.js: -------------------------------------------------------------------------------- 1 | /** 2 | * mocks no-kafka module 3 | * @author ghermeto 4 | * @version 0.0.1 5 | **/ 6 | const mock = require('mock-require'); 7 | const shortid = require('shortid'); 8 | 9 | const mockKafka = { 10 | offset: 0, 11 | partition: 0, 12 | registry: {} 13 | }; 14 | 15 | class GroupConsumer { 16 | constructor(options) { 17 | this.options = options; 18 | } 19 | 20 | init(strategies) { 21 | const metadata = strategies[0].metadata; 22 | if (metadata && metadata !== 'object') { 23 | return Promise.reject(); 24 | } 25 | 26 | this.strategies = strategies; 27 | strategies.forEach(strategy => { 28 | 29 | mockKafka.registry = strategy.subscriptions 30 | .reduce((registry, topic) => { 31 | const current = registry[topic] || []; 32 | const updatedList = current.concat(strategy.handler); 33 | return Object.assign({}, registry, {[topic]: updatedList}); 34 | }, mockKafka.registry); 35 | }); 36 | return Promise.resolve(mockKafka.offset); 37 | } 38 | 39 | commitOffset() { 40 | return Promise.resolve(mockKafka.offset++); 41 | } 42 | 43 | unsubscribe(topic) { 44 | delete mockKafka.registry[topic]; 45 | return Promise.resolve(); 46 | } 47 | 48 | end() { 49 | this.strategies = null; 50 | mockKafka.registry = {}; 51 | } 52 | } 53 | 54 | class Producer { 55 | constructor(options) { 56 | this.options = options; 57 | options.logger.logFunction('trace', ['mock']); 58 | } 59 | 60 | init() { 61 | return Promise.resolve(); 62 | } 63 | 64 | end() { 65 | return Promise.resolve(); 66 | } 67 | 68 | send({ topic, message }) { 69 | const error = this.options.metadata ? new Error('test') : null; 70 | mockProducer().publish(topic, message.value, message.key); 71 | return Promise.resolve([{ topic, error }]); 72 | } 73 | } 74 | 75 | const mockProducer = () => { 76 | return { 77 | publish(topic, message, key = shortid.generate()) { 78 | const handlerList = mockKafka.registry[topic]; 79 | if (!mockKafka.registry[topic]) { 80 | throw 'No consumer in such topic'; 81 | } 82 | 83 | const msg = this.createMessage(message, key); 84 | 85 | handlerList.forEach(handler => { 86 | handler([msg], topic, mockKafka.partition); 87 | }); 88 | }, 89 | 90 | createMessage(message, key) { 91 | return { 92 | offset: mockKafka.offset, 93 | messageSize: message ? message.length : 0, 94 | message: { 95 | key: key ? Buffer.from(key, 'utf8') : null, 96 | value: message 97 | ? typeof message === 'string' 98 | ? Buffer.from(message, 'utf8') 99 | : message 100 | : null 101 | } 102 | } 103 | } 104 | 105 | } 106 | }; 107 | 108 | const mockKafkaClient = { 109 | DefaultAssignmentStrategy: class DefaultAssignmentStrategy { }, 110 | DefaultPartitioner: class DefaultPartitioner { }, 111 | GroupConsumer: GroupConsumer, 112 | Producer: Producer 113 | }; 114 | 115 | mock('no-kafka', mockKafkaClient); 116 | mock.reRequire('no-kafka'); 117 | 118 | module.exports = { mockKafkaClient, mockProducer }; 119 | beforeAll(() => { 120 | global.kafkaClient = mockKafkaClient; 121 | global.mockProducer = mockProducer; 122 | }); 123 | -------------------------------------------------------------------------------- /test/unit/operators.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const kafkaClient = require('../../lib/client'); 5 | const KafkaObservable = require('../../index'); 6 | 7 | const opts = { 8 | groupId: 'test', 9 | brokers: '127.0.0.1:9092' 10 | }; 11 | 12 | const producer = kafkaClient.producer(opts); 13 | 14 | describe('JSONMessage operator', () => { 15 | let subscription; 16 | 17 | it('should get a message value as JSON', done => { 18 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 19 | subscription = observable 20 | .take(1) 21 | .let(KafkaObservable.JSONMessage()) 22 | .subscribe( 23 | json => { 24 | expect(json.key).toBeDefined(); 25 | expect(json.key).toEqual('test-value'); 26 | }, 27 | err => done.fail(err), 28 | () => done() 29 | ); 30 | 31 | delay(100) 32 | .then(() => producer.publish('test_kafka', {key: 'test-value'})); 33 | }); 34 | 35 | it('should error if not JSON', done => { 36 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 37 | subscription = observable 38 | .take(1) 39 | .let(KafkaObservable.JSONMessage()) 40 | .subscribe(() => done.fail(), err => done()); 41 | 42 | delay(100) 43 | .then(() => producer.publish('test_kafka', 'test')); 44 | }); 45 | 46 | it('should error if mapper throws', done => { 47 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 48 | subscription = observable 49 | .take(1) 50 | .let(KafkaObservable.JSONMessage(x => { throw 'Ops!'; })) 51 | .subscribe(() => done.fail(), err => done()); 52 | 53 | delay(100) 54 | .then(() => producer.publish('test_kafka', {key: 'test-value'})); 55 | }); 56 | 57 | it('should be available when called in the instance', done => { 58 | const kafka = KafkaObservable(opts); 59 | const observable = kafka.fromTopic('test_kafka'); 60 | subscription = observable 61 | .take(1) 62 | .let(kafka.JSONMessage()) 63 | .subscribe( 64 | json => { 65 | expect(json.key).toBeDefined(); 66 | expect(json.key).toEqual('test-value'); 67 | }, 68 | err => done.fail(err), 69 | () => done() 70 | ); 71 | 72 | delay(100) 73 | .then(() => producer.publish('test_kafka', {key: 'test-value'})); 74 | }); 75 | 76 | afterEach(() => { 77 | subscription.unsubscribe(); 78 | }); 79 | 80 | }); 81 | 82 | describe('TextMessage operator', () => { 83 | let subscription; 84 | 85 | it('should get a message value as text', done => { 86 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 87 | subscription = observable 88 | .take(1) 89 | .let(KafkaObservable.TextMessage()) 90 | .subscribe( 91 | msg => expect(msg).toEqual('my message'), 92 | err => done.fail(err), 93 | () => done() 94 | ); 95 | 96 | delay(100) 97 | .then(() => producer.publish('test_kafka', 'my message')); 98 | }); 99 | 100 | it('should error if mapper throws', done => { 101 | const observable = KafkaObservable.fromTopic('test_kafka', opts); 102 | subscription = observable 103 | .take(1) 104 | .let(KafkaObservable.TextMessage(x => { throw 'Ops!'; })) 105 | .subscribe(() => done.fail(), err => done()); 106 | 107 | delay(100) 108 | .then(() => producer.publish('test_kafka', 'my message')); 109 | }); 110 | 111 | afterEach(() => { 112 | subscription.unsubscribe(); 113 | }); 114 | 115 | }); -------------------------------------------------------------------------------- /test/unit/producer.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Observable = require('rxjs').Observable; 4 | const consumerObservable = require('../../lib/observables/consumer'); 5 | const producerObservable = require('../../lib/observables/producer'); 6 | 7 | const opts = { 8 | groupId: 'test', 9 | brokers: '127.0.0.1:9092' 10 | }; 11 | 12 | describe('producer', () => { 13 | let consumerSubscription, producerSubscription; 14 | 15 | it('should send a single message', done => { 16 | const msg = 'message'; 17 | const consumer = consumerObservable.create('test_kafka', opts); 18 | const producer = producerObservable.create('test_kafka', msg, opts); 19 | 20 | consumerSubscription = consumer 21 | .take(1) 22 | .subscribe(() => {}, err => done.fail(err), () => done()); 23 | 24 | delay(100) 25 | .then(() => { 26 | producerSubscription = producer 27 | .subscribe( 28 | ({topic, error}) => { 29 | expect(topic).toEqual('test_kafka'); 30 | expect(error).toBeNull(); 31 | }, 32 | err => done.fail(err) 33 | ); 34 | }); 35 | }); 36 | 37 | it('should send many messages', done => { 38 | const msgs = ['first', 'second']; 39 | const consumer = consumerObservable.create('test_kafka', opts); 40 | const producer = producerObservable.create('test_kafka', msgs, opts); 41 | 42 | consumerSubscription = consumer 43 | .take(2) 44 | .subscribe(() => {}, err => done.fail(err), () => done()); 45 | 46 | delay(100) 47 | .then(() => { 48 | producerSubscription = producer 49 | .subscribe( 50 | ({topic, error}) => { 51 | expect(topic).toEqual('test_kafka'); 52 | expect(error).toBeNull(); 53 | }, 54 | err => done.fail(err) 55 | ); 56 | }); 57 | }); 58 | 59 | it('should send messages from an observable', done => { 60 | const msgs = Observable.from(['first', 'second', 'third']); 61 | const consumer = consumerObservable.create('test_kafka', opts); 62 | const producer = producerObservable.create('test_kafka', msgs, opts); 63 | 64 | consumerSubscription = consumer 65 | .take(3) 66 | .subscribe(() => {}, err => done.fail(err), () => done()); 67 | 68 | delay(100) 69 | .then(() => { 70 | producerSubscription = producer 71 | .subscribe( 72 | ({topic, error}) => { 73 | expect(topic).toEqual('test_kafka'); 74 | expect(error).toBeNull(); 75 | }, 76 | err => done.fail(err) 77 | ); 78 | }); 79 | }); 80 | 81 | it('should error if brokers is not defined', done => { 82 | const msgs = ['first', 'second']; 83 | const options = Object.assign({}, opts, { brokers: undefined }); 84 | const producer = producerObservable.create('test_kafka', msgs, options); 85 | producerSubscription = producer 86 | .subscribe({ next: () => done.fail(), error: err => done() }); 87 | }); 88 | 89 | it('should error if brokers is not a string or an array', done => { 90 | const msgs = ['first', 'second']; 91 | const options = Object.assign({}, opts, { brokers: {} }); 92 | const producer = producerObservable.create('test_kafka', msgs, options); 93 | producerSubscription = producer 94 | .subscribe({ next: () => done.fail(), error: err => done() }); 95 | }); 96 | 97 | it('should propagate error if observable gives an error', done => { 98 | const msgs = Observable.throw('test'); 99 | const producer = producerObservable.create('test_kafka', msgs, opts); 100 | producerSubscription = producer 101 | .subscribe({ 102 | next: () => done.fail(), 103 | error: err => done(), 104 | complete: () => done() 105 | }); 106 | }); 107 | 108 | it('should default to DefaultPartitioner', done => { 109 | const msgs = ['first', 'second']; 110 | const options = Object.assign({ partitioner: 'NotFound'}, opts); 111 | const consumer = consumerObservable.create('test_kafka', opts); 112 | const producer = producerObservable.create('test_kafka', msgs, options); 113 | consumerSubscription = consumer 114 | .take(2) 115 | .subscribe(() => {}, err => done.fail(err), () => done()); 116 | 117 | delay(100) 118 | .then(() => { 119 | producerSubscription = producer 120 | .subscribe(() => { 121 | const client = producer._adapter.client(); 122 | const partitioner = client.options.partitioner; 123 | 124 | expect(partitioner.constructor.name) 125 | .toEqual('DefaultPartitioner'); 126 | 127 | done(); 128 | }, 129 | err => done.fail(err) 130 | ); 131 | }); 132 | 133 | }); 134 | 135 | it('should serialize an object as message', done => { 136 | const msgs = [{ object: true }]; 137 | const consumer = consumerObservable.create('test_kafka', opts); 138 | const producer = producerObservable.create('test_kafka', msgs, opts); 139 | consumerSubscription = consumer 140 | .take(1) 141 | .map(({message}) => message.value.toString('utf8')) 142 | .map(message => JSON.parse(message)) 143 | .subscribe( 144 | json => expect(json.object).toBe(true), 145 | err => done.fail(err), 146 | () => done() 147 | ); 148 | 149 | delay(100) 150 | .then(() => { 151 | producerSubscription = producer 152 | .subscribe(() => {}, err => done.fail(err)); 153 | }); 154 | 155 | }); 156 | 157 | afterEach(() => { 158 | consumerSubscription.unsubscribe(); 159 | producerSubscription.unsubscribe(); 160 | }); 161 | 162 | }); 163 | --------------------------------------------------------------------------------