├── .gitignore
├── .npmrc
├── .travis.yml
├── LICENSE
├── README.md
├── index.js
├── lib
├── client
│ ├── base.js
│ ├── consumer.js
│ ├── index.js
│ └── producer.js
├── observables
│ ├── consumer.js
│ └── producer.js
└── operators
│ ├── json-message.js
│ └── kafka-message.js
├── package-lock.json
├── package.json
└── test
├── integration
├── helpers
│ └── helper.js
├── observables
│ ├── consumer.spec.js
│ └── producer.spec.js
└── operators
│ ├── json-message.spec.js
│ └── kafka-message.spec.js
├── support
├── integration.json
└── unit.json
└── unit
├── consumer.spec.js
├── helpers
└── helper.js
├── kafka-observable.spec.js
├── mock
├── mock-bunyan.js
└── mock-no-kafka.js
├── operators.spec.js
└── producer.spec.js
/.gitignore:
--------------------------------------------------------------------------------
1 | out
2 | node_modules
3 | .idea
4 | npm-debug.log
5 | coverage
--------------------------------------------------------------------------------
/.npmrc:
--------------------------------------------------------------------------------
1 | registry=https://registry.npmjs.org/
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | sudo: false
3 | node_js:
4 | - "6.4.0"
5 | - "8.2.1"
6 | - "8.9.1"
7 | - "9.2.0"
8 | jobs:
9 | include:
10 | - stage: coverage
11 | node_js:
12 | - "9.2.0"
13 | env:
14 | global:
15 | - CC_TEST_REPORTER_ID=d3ce1b1e6b62d8d585ece975cb481c0c3fcfdb664b95dbcb49f6cfc4c95da519
16 | before_script:
17 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
18 | - chmod +x ./cc-test-reporter
19 | - ./cc-test-reporter before-build
20 | script: npm run coverage
21 | after_script:
22 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
23 | - stage: integration
24 | node_js:
25 | - "9.2.0"
26 | services:
27 | - docker
28 | before_install:
29 | - docker pull spotify/kafka
30 | script: npm run integration-test
31 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2015 Apigee Corporation
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | []()
2 | [![Build Status][badge-travis]][travis]
3 | [![Test Coverage][badge-coverage]][coverage]
4 | [![license][badge-license]][license]
5 |
6 | # kafka-observable
7 |
8 | __kafka-observable__ is the easiest way to exchange messages through kafka with [node.js](https://nodejs.org).
9 |
10 |
11 | Using the solid [no-kafka](https://github.com/oleksiyk/kafka) as default client, __kafka-observable__ creates [RxJS](https://github.com/ReactiveX/rxjs) observables
12 | that can be manipulated as if you were using Kafka Streams, but with a familiar interface
13 | to javascript developers.
14 |
15 | ### Why observables?
16 |
17 | Think of observables as collections which elements arrive over time. You may iterate over
18 | them, which means you can also apply filter, map or reduce.
19 |
20 | Many of the operations provided by observables are very similar to the capabilities available
21 | in Kafka Streams, including the ability to use window (accumulate values for a period).
22 |
23 | ### Installation
24 |
25 | ```bash
26 | npm install --save kafka-observable
27 | ```
28 |
29 | ### Example usage
30 |
31 | Imagine your customers can subscribe to out-of-stock products in your online store to
32 | receive emails when the product is in stock. Your stock management publishes updates
33 | to a kafka topic called __inventory_updates__ and you mailer consumes from __notifications__.
34 |
35 | ```javascript
36 | const opts = {
37 | brokers: 'kafka://kafka-broker.example.com:9092',
38 | groupId: 'inventory-notifications'
39 | };
40 |
41 | const KafkaObservable = require('kafka-observable')(opts);
42 | // assumes getWatchers will execute a network request and returns an observable
43 | const getWatchers = require('./lib/observables/watchers');
44 |
45 | const subscription = KafkaObservable.fromTopic('inventory_updates')
46 | // gets messge as JSON
47 | .let(KafkaObservable.JSONMessage())
48 | // just arrived
49 | .filter(({inventory}) => inventory.previous === 0 && inventory.current > 0)
50 | // gets watchers, format message and concat 2 dimensional observable
51 | .concatMap(product =>
52 | getWatchers(product.id)
53 | .map(watchers => ({ watchers, product }))
54 | // sends formated message to new topic and concats 2 dimensional observable
55 | .concatMap(message => KafkaObservable.toTopic('notifications', message));
56 |
57 | subscription.subscribe(success => console.log(success), err => console.error(err));
58 | ```
59 |
60 | ## Methods
61 |
62 | ### fromTopic(topic, options, adapterFactory = defaultAdapterFactory)
63 |
64 | Creates an observable that will consume from a Kafka topic.
65 |
66 | #### Parameters
67 | * topic (String) - topic name
68 | * options (Object) - client options
69 | * adapterFactory (Object) - client adapter (Defaults to no-kafka adapter)
70 |
71 | #### Examples
72 |
73 | KafkaObservable as a function:
74 | ```javascript
75 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' };
76 |
77 | const KafkaObservable = require('kafka-observable')(opts);
78 | const consumer = KafkaObservable.fromTopic('my_topic')
79 | .map(({message}) => message.value.toString('utf8'));
80 |
81 | consumer.subscribe(message => console.info(message));
82 | ```
83 |
84 | Passing __options__ parameter to fromTopic:
85 | ```javascript
86 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' };
87 |
88 | const KafkaObservable = require('kafka-observable');
89 | const consumer = KafkaObservable.fromTopic('my_topic', opts)
90 | .map(({message}) => message.value.toString('utf8'));
91 |
92 | consumer.subscribe(message => console.info(message));
93 | ```
94 |
95 | #### Options
96 |
97 | Below are the main options for the consumer. For more consumer options, please
98 | refer to [no-kafka](https://github.com/oleksiyk/kafka) options (in case you use the provided default adapter).
99 |
100 | | Option | Required | Type | Default | Description |
101 | |---|---|---|---|---|
102 | | brokers | yes | Array/String | - | list of Kafka brokers
103 | | groupId | yes | String| - | consumer group id
104 | | autoCommit | no | boolean |true | commits the message offset automatically if no exception is thrown
105 | | strategy | no | String | Default | name of the assignment strategy for the consumer (Default/Consistent/WeightedRoundRobin)
106 |
107 | ### toTopic(topic, messages, options, adapterFactory = defaultAdapterFactory)
108 |
109 | Creates an observable that publishes messages to a Kafka topic.
110 |
111 | #### Parameters
112 | * topic (String) - topic name
113 | * messages (String|Array|Observable) - messages to be published in kafka topic
114 | * options (Object) - client options
115 | * adapterFactory (Object) - client adapter (Defaults to no-kafka adapter)
116 |
117 | #### Examples
118 |
119 | KafkaObservable as a function:
120 | ```javascript
121 | const opts = { brokers: 'kafka://127.0.0.1:9092' };
122 | const messages = [{key: 'value1'}, {key: 'value2'}];
123 |
124 | const KafkaObservable = require('kafka-observable')(opts);
125 | const producer = KafkaObservable.toTopic('my_topic', messages);
126 |
127 | producer.subscribe(message => console.info(message));
128 | ```
129 |
130 | Passing __options__ parameter to toTopic:
131 | ```javascript
132 | const opts = { brokers: 'kafka://127.0.0.1:9092' };
133 | const messages = Observable.from([{key: 'value1'}, {key: 'value2'}]);
134 |
135 | const KafkaObservable = require('kafka-observable');
136 | const producer = KafkaObservable.toTopic('my_topic', messages, opts);
137 |
138 | producer.subscribe(message => console.info(message));
139 | ```
140 |
141 | #### Options
142 |
143 | Below are the main options for the producer. For more producer options, please
144 | refer to [no-kafka](https://github.com/oleksiyk/kafka) options (in case you use the provided default adapter).
145 |
146 | | Option | Required | Type | Default | Description |
147 | |---|---|---|---|---|
148 | | brokers | yes | Array/String | - | list of Kafka brokers
149 | | partitioner | no | prototype/String | Default | name (Default/HashCRC32) or prototype (instance of Kafka.DefaultPartitioner) to use as producer partitioner
150 |
151 | ### TextMessage(mapper = (x) => x)
152 |
153 | Convenience operator which converts a Buffer message value into utf8 string.
154 |
155 | #### Parameters
156 | * mapper (Function) - mapper function
157 |
158 | #### Example
159 |
160 | ```javascript
161 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' };
162 |
163 | const KafkaObservable = require('kafka-observable');
164 | const consumer = KafkaObservable.fromTopic('my_topic', opts)
165 | .let(KafkaObservable.TextMessage());
166 |
167 | consumer.subscribe(message => console.info(message));
168 | ```
169 |
170 | ### JSONMessage(mapper = (x) => x)
171 |
172 | Convenience operator provided to deserialize an object from a JSON message.
173 |
174 | #### Parameters
175 | * mapper (Function) - mapper function
176 |
177 | #### Example
178 |
179 | ```javascript
180 | const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'test' };
181 |
182 | const KafkaObservable = require('kafka-observable');
183 | const consumer = KafkaObservable.fromTopic('my_topic', opts)
184 | .let(KafkaObservable.JSONMessage());
185 |
186 | consumer.subscribe(json => console.info(json.key));
187 | ```
188 |
189 | ## Custom Kafka Adapter
190 |
191 | If you don't want to use [no-kafka](https://github.com/oleksiyk/kafka) you can write an adapter for your client which respects
192 | the interface established by the code in `lib/client`.
193 |
194 | #### Why an adapter?
195 |
196 | I currently use an internal kafka client at Netflix with an interface very similar to this adapter
197 | and I wanted it to work out-of-the-box.
198 |
199 | ## Development
200 |
201 | #### Unit tests
202 | ```bash
203 | npm install
204 | npm run unit-test
205 | ```
206 |
207 | #### Integration tests
208 | *requires docker to be installed and accessible through the **docker** command*
209 | ```bash
210 | npm install
211 | docker pull spotify/kafka
212 | npm run unit-test
213 | ```
214 |
215 | #### Test coverage
216 | *based on unit tests*
217 | ```bash
218 | npm install
219 | npm run coverage
220 | open coverage/lcov-report/index.html
221 | ```
222 |
223 | #### Documentation
224 | ```bash
225 | npm install
226 | npm run gen-docs
227 | open out/index.html
228 | ```
229 | ___
230 | ### License: [MIT](https://github.com/ghermeto/kafka-observable/blob/master/LICENSE)
231 |
232 | [badge-license]: https://img.shields.io/badge/License-MIT-green.svg
233 | [license]: https://github.com/ghermeto/kafka-observable/blob/master/LICENSE
234 | [badge-travis]: https://api.travis-ci.org/ghermeto/kafka-observable.svg?branch=master
235 | [travis]: https://travis-ci.org/ghermeto/kafka-observable
236 | [badge-coverage]: https://codeclimate.com/github/ghermeto/kafka-observable/badges/coverage.svg
237 | [coverage]: https://codeclimate.com/github/ghermeto/kafka-observable/coverage
238 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Creates kafka consumer and producer observables given a kafka client adapter
3 | * and a options object.
4 | *
5 | * @example
consumer
6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' };
7 | * const KafkaObservable = require('kafka-observable')(opts);
8 | * const observable = KafkaObservable.fromTopic('my_topic');
9 | *
10 | * @example producer
11 | * const opts = { brokers: 'kafka://127.0.0.1:9092' };
12 | * const KafkaObservable = require('kafka-observable')(opts);
13 | * const observable = KafkaObservable.toTopic('my_topic', 'my message');
14 | *
15 | * @module KafkaObservable
16 | * @author ghermeto
17 | **/
18 | 'use strict';
19 |
20 | const Consumer = require('./lib/observables/consumer');
21 | const Producer = require('./lib/observables/producer');
22 |
23 | const jsonMessage = require('./lib/operators/json-message');
24 | const kafkaMessage = require('./lib/operators/kafka-message');
25 |
26 | /**
27 | * @param {Object} options client options
28 | * @param {boolean} options.autoCommit automatically commits read message
29 | * @param {Object|String} options.partitioner name or class for partitioner implementation
30 | * @param {String} options.strategy name of the assignment strategy
31 | * @param {Object} clientFactory your custom kafka client adapter
32 | * @returns {ObservableFactory}
33 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options.
34 | */
35 | function KafkaObservable(options, clientFactory) {
36 | /**
37 | * Object capable of creating both the consumer and producer observables
38 | * @inner
39 | * @name ObservableFactory
40 | * @typedef {Object} ObservableFactory
41 | * @property {Function} fromTopic same as fromTopic from prototype
42 | * @property {Function} toTopic same as toTopic from prototype
43 | */
44 | return {
45 | /**
46 | * Creates a Consumer observable
47 | * @param {string} topic topic to subscribe
48 | * @param {Object} opts kafka client options
49 | * @param {Object} client kafka adapter
50 | * @returns {Observable}
51 | * @see {@link observables/Consumer}
52 | */
53 | fromTopic:
54 | (topic, opts = options, client = clientFactory) =>
55 | Consumer.create(topic, opts, client),
56 | /**
57 | * Creates a Producer observable
58 | * @param {string} topic topic to subscribe
59 | * @param {Array|Observable} messages message(s) to be sent
60 | * @param {Object} opts kafka client options
61 | * @param {Object} client kafka adapter
62 | * @returns {Observable}
63 | * @see {@link observables/Producer}
64 | */
65 | toTopic:
66 | (topic, messages, opts = options, client = clientFactory) =>
67 | Producer.create(topic, messages, opts, client),
68 |
69 | /**
70 | * Operator: formats a message as a JSON object
71 | * @function
72 | * @param {Function} mapper mapping function
73 | */
74 | JSONMessage: jsonMessage,
75 | /**
76 | * Operator: formats a message as a text message
77 | * @function
78 | * @param {Function} mapper mapping function
79 | */
80 | TextMessage: kafkaMessage
81 | }
82 | }
83 |
84 | // observables
85 | KafkaObservable.fromTopic = KafkaObservable().fromTopic;
86 | KafkaObservable.toTopic = KafkaObservable().toTopic;
87 |
88 | // operators
89 | KafkaObservable.JSONMessage = jsonMessage;
90 | KafkaObservable.TextMessage = kafkaMessage;
91 |
92 | module.exports = KafkaObservable;
--------------------------------------------------------------------------------
/lib/client/base.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Base Adapter class.
3 | *
4 | * @author ghermeto
5 | **/
6 |
7 | const EventEmitter = require('events');
8 | const bunyan = require('bunyan');
9 | const shortid = require('shortid');
10 |
11 | /**
12 | * Base pipeline class extended by both Consumer and Producer classes
13 | * @class Base
14 | */
15 | class Base extends EventEmitter {
16 |
17 | constructor(opts) {
18 | super();
19 | this.isValid(opts);
20 | this.opts = opts;
21 | this.clientId = opts.clientId || `kafka-observable-${shortid.generate()}`;
22 | this.brokers = Base.formatBrokers(opts.brokers);
23 |
24 | this.log = bunyan.createLogger({
25 | name: this.clientId,
26 | level: process.env.LOG_LEVEL
27 | });
28 | }
29 |
30 | /**
31 | * Convert to array or throw if parameter is not string or array
32 | * @param {String|Array} brokers single or multiple brokers URL
33 | * @return {Array.<*>}
34 | */
35 | static formatBrokers(brokers) {
36 | const className = brokers.constructor.name;
37 | if (className !== 'Array' && className !== 'String') {
38 | throw new TypeError('invalid brokers parameter');
39 | }
40 | return [].concat(brokers);
41 | }
42 |
43 | /**
44 | * Validate options. Throws if invalid.
45 | * @param {Object} opts user defined options
46 | */
47 | isValid(opts) {
48 | if (!opts.brokers) {
49 | throw new Error('missing vipaddress or brokers parameter');
50 | }
51 | }
52 |
53 | /**
54 | * To be overridden by consumer and producer defaults
55 | * @return {Object} default client options
56 | */
57 | defaults() {
58 | return {
59 | connectionString: this.brokers.join(','),
60 | logger: {logFunction: this.logging.bind(this)}
61 | };
62 | }
63 |
64 | /**
65 | * @return {Object} client options
66 | */
67 | clientOptions() {
68 | return Object.assign({}, this.defaults(), this.opts);
69 | }
70 |
71 | /**
72 | * Emits error event if listener error is found.
73 | * EventEmitter throws if an error event is emitted without listeners
74 | * @see https://nodejs.org/api/events.html#events_error_events
75 | * @param {Object} err error object
76 | */
77 | handleError(err) {
78 | this.log.error(err);
79 | if (this.listenerCount('error') > 0) {
80 | this.emit('error', err);
81 | }
82 | return Promise.reject(err);
83 | }
84 |
85 | /**
86 | * Pipes logs through Bunyan and fire events
87 | * @param {String} level log level
88 | * @param {Array} args
89 | */
90 | logging(level, ...args) {
91 | this.log[level.toLowerCase()](...args);
92 | this.monitoring(level, ...args);
93 | }
94 |
95 | /**
96 | * Parses through the logs to emit meaningful events
97 | * @param {String} level log level
98 | * @param {Array} args
99 | */
100 | monitoring(level, ...args) {
101 | if (level === bunyan.ERROR) {
102 | this.handleError(args[3]);
103 | }
104 | }
105 |
106 | }
107 |
108 | module.exports = Base;
--------------------------------------------------------------------------------
/lib/client/consumer.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Kafka Consumer Adapter. It adapts no-kafka group consumer to fits the
3 | * KafkaObservable interface.
4 | *
5 | * @example consumer
6 | * const kafkaClient = require('./lib/client');
7 | * const consumer = kafkaClient.consumer({
8 | * brokers: 'https://exemple.com:7123',
9 | * groupId: 'audit',
10 | * });
11 | *
12 | * consumer.on('error', err => console.error(err));
13 | * consumer.subscribe('my_topic', (messageSet, topic, partition) => {
14 | * messageSet.forEach(promise => promise.then(({offset, message}) => {
15 | * console.info(topic, partition, message.value.toString('utf8'), offset);
16 | * return consumer.client().commitOffset({t opic, partition, offset });
17 | * }));
18 | * });
19 | *
20 | * @author ghermeto
21 | **/
22 | const Kafka = require('no-kafka');
23 | const Base = require('./base');
24 |
25 | /**
26 | * Enables subscribing to a topic in the pipeline
27 | * @class Consumer
28 | */
29 | class Consumer extends Base {
30 | constructor(opts) {
31 | super(opts);
32 | }
33 |
34 | /**
35 | * Validates required options
36 | * @private
37 | * @override
38 | * @param {Object} opts user defined options
39 | */
40 | isValid(opts) {
41 | super.isValid(opts);
42 | if (!opts.groupId) { throw new Error('missing consumer groupId'); }
43 | }
44 |
45 | /**
46 | * Dynamically generates consumer default configuration properties
47 | * @private
48 | * @override
49 | * @returns {Object} Default configs
50 | */
51 | defaults() {
52 | const base = super.defaults();
53 | return Object.assign({
54 | idleTimeout: 100,
55 | heartbeatTimeout: 100,
56 | clientId: this.clientId
57 | }, base);
58 | }
59 |
60 | /**
61 | * Returns the consumer client once it has subscribed to a topic.
62 | *
63 | * @return {Kafka.GroupConsumer|GroupConsumer}
64 | */
65 | client() {
66 | return this.consumer;
67 | }
68 |
69 | /**
70 | * Convert strategy name to object.
71 | * Possible values are: WeightedRoundRobin, Consistent and Default.
72 | * @private
73 | * @param {String} name strategy name.
74 | */
75 | strategy(name = 'Default') {
76 | const className = `${name}AssignmentStrategy`;
77 | if (!Kafka[className]) {
78 | this.log.warn(`Strategy ${className} not found. Using default.`);
79 | return new Kafka.DefaultAssignmentStrategy();
80 | }
81 | return new Kafka[className]();
82 | }
83 |
84 | /**
85 | * First unsubscribe from the topic and then ends the client connection.
86 | *
87 | * @param {String} topic topic to unsubscribe
88 | * @return {Promise}
89 | */
90 | end(topic) {
91 | return this.client()
92 | .unsubscribe(topic)
93 | .then(() => this.client().end());
94 | }
95 |
96 | /**
97 | * Commits the received message
98 | *
99 | * @param {String} topic topic which had the message
100 | * @param {Number} partition partition which had the message
101 | * @param {Number} offset message offset in the topic partition
102 | * @return {Promise}
103 | */
104 | commit(topic, partition, offset) {
105 | return this.client().commitOffset({ topic, partition, offset });
106 | }
107 |
108 | /**
109 | * Subscribe to a list of channels passing a callback
110 | *
111 | * @param {Array} topics topics to subscribe
112 | * @param {Function} handler function to be executed when a message arrives
113 | */
114 | subscribe(topics, handler) {
115 | const options = this.clientOptions();
116 | this.consumer = new Kafka.GroupConsumer(options);
117 |
118 | const strategies = [{
119 | strategy: this.strategy(this.opts.strategy),
120 | metadata: this.opts.metadata,
121 | subscriptions: [].concat(topics),
122 | handler
123 | }];
124 |
125 | return this.consumer.init(strategies)
126 | .catch(err => this.handleError(err));
127 | }
128 | }
129 |
130 | module.exports = Consumer;
--------------------------------------------------------------------------------
/lib/client/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Kafka Client Adapter - adapts the no-kafka client to the expected interface
3 | *
4 | * @example subscriber
5 | * const kafkaClient = require('./lib/client');
6 | * const consumer = kafkaClient.consumer({
7 | * brokers: 'https://exemple.com:7123',
8 | * groupId: 'audit',
9 | * });
10 | *
11 | * consumer.on('error', err => console.error(err));
12 | * consumer.subscribe('my_topic', (messageSet, topic, partition) => {
13 | * messageSet.forEach(promise => promise.then(({offset, message}) => {
14 | * console.info(topic, partition, message.value.toString('utf8'), offset);
15 | * return consumer.client().commitOffset({t opic, partition, offset });
16 | * }));
17 | * });
18 | *
19 | * @example producer
20 | * const kafkaClient = require('./lib/client');
21 | * const producer = kafkaClient.producer({brokers: 'https://exemple.com:7123'});
22 | *
23 | * const message = { hello: 'world', ok: true };
24 | * producer.on('error', err => console.error(err));
25 | * producer.publish('my_topic', message);
26 | *
27 | * @author ghermeto
28 | **/
29 | 'use strict';
30 |
31 | const Producer = require('./producer');
32 | const Consumer = require('./consumer');
33 |
34 | exports.producer = opts => new Producer(opts);
35 | exports.consumer = opts => new Consumer(opts);
36 |
--------------------------------------------------------------------------------
/lib/client/producer.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Kafka Producer Adapter. It adapts no-kafka producer to the format expected
3 | * by the KafkaObservable interface
4 | *
5 | * @example producer
6 | * const kafkaClient = require('./lib/client');
7 | * const producer = kafkaClient.producer({brokers: 'https://exemple.com:7123'});
8 | *
9 | * const message = { hello: 'world', ok: true };
10 | * producer.on('error', err => console.error(err));
11 | * producer.publish('my_topic', message);
12 | *
13 | * @author ghermeto
14 | **/
15 |
16 | const Kafka = require('no-kafka');
17 | const Base = require('./base');
18 | const shortid = require('shortid');
19 |
20 | /**
21 | * @class Producer
22 | */
23 | class Producer extends Base {
24 | constructor(opts) {
25 | super(opts);
26 | const options = this.clientOptions();
27 | this.producer = new Kafka.Producer(Object.assign({}, options));
28 | }
29 |
30 | /**
31 | * @return {Object} client options
32 | */
33 | clientOptions() {
34 | const base = super.clientOptions();
35 | const fixed = { partitioner: this.partitioner(this.opts.partitioner) };
36 | return Object.assign({}, base, fixed);
37 | }
38 |
39 | /**
40 | * Dynamically generates producer default configuration properties
41 | * @private
42 | * @override
43 | * @returns {Object} Default configs
44 | */
45 | defaults() {
46 | const base = super.defaults();
47 | const defaults = { timeout: 1000 };
48 | return Object.assign({}, base, defaults);
49 | }
50 |
51 | /**
52 | * Initializes the producer if necessary or use the cached producer
53 | * @private
54 | * @return {Promise} active producer
55 | */
56 | init() {
57 | if (this.active) { return this.active; }
58 | this.active = this.producer.init();
59 | return this.active;
60 | }
61 |
62 | /**
63 | * Returns a partitioner instance from a string or from a class.
64 | * @private
65 | * @param {String|Object} nameOrClass name of the partitioner or class
66 | * @return {Object} partitioner instance
67 | */
68 | partitioner(nameOrClass = 'Default') {
69 | if (typeof nameOrClass === 'object') {
70 | return new nameOrClass();
71 | }
72 | const className = `${nameOrClass}Partitioner`;
73 | if (!Kafka[className]) {
74 | this.log.warn(`Partitioner ${className} not found. Using default.`);
75 | return new Kafka.DefaultPartitioner();
76 | }
77 | return new Kafka[className]();
78 | }
79 |
80 | /**
81 | * Serializes messages
82 | * @static
83 | * @private
84 | * @param {Object|String} message
85 | * @return {String} serialized message
86 | */
87 | static serialize(message) {
88 | return typeof message === 'string' ? message : JSON.stringify(message);
89 | }
90 |
91 | /**
92 | * Returns the producer client once it has sent at least one message.
93 | * @return {Kafka.Producer|Producer}
94 | */
95 | client() {
96 | return this.producer;
97 | }
98 |
99 | /**
100 | * Terminates the client connection.
101 | * @return {Promise}
102 | */
103 | end() {
104 | return this.client().end();
105 | }
106 |
107 | /**
108 | * Publishes a message to Kafka
109 | * @param {String} topic topic to publish on
110 | * @param {Object|String} message message to be sent
111 | * @param {String} key message key
112 | * @return {Promise}
113 | */
114 | publish(topic, message, key = `${this.clientId}-${shortid.generate()}`) {
115 | const serialized = Producer.serialize(message);
116 | const payload = {topic: topic, message: {key: key, value: serialized}};
117 |
118 | return this.init()
119 | .then(() => this.producer.send(payload))
120 | .catch(err => this.handleError(err));
121 | }
122 | }
123 |
124 | module.exports = Producer;
--------------------------------------------------------------------------------
/lib/observables/consumer.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Kafka Consumer Observable.
3 | *
4 | * @example consumer
5 | * const consumerObservable = require('./lib/consumer');
6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' };
7 | * const observable = consumerObservable.create('my_topic', opts);
8 | *
9 | * observable
10 | * .do(m => m.commit())
11 | * .map(m => m.message)
12 | * .subscribe(message => console.info(message));
13 | *
14 | * @module observables/Consumer
15 | * @author ghermeto
16 | **/
17 |
18 | const bunyan = require('bunyan');
19 | const defaultFactory = require('../client');
20 | const Observable = require('rxjs').Observable;
21 |
22 | const observerError = 'Error executing observer. Message not committed.';
23 |
24 | /**
25 | * logger
26 | * @constant
27 | * @type {Object}
28 | */
29 | const log = bunyan.createLogger({
30 | name: 'KafkaObservable',
31 | level: process.env.LOG_LEVEL
32 | });
33 |
34 | /**
35 | * default consumer options
36 | * @constant
37 | * @type {Object}
38 | */
39 | const defaults = {
40 | autoCommit: true
41 | };
42 |
43 | /**
44 | * Creates a selector function with the kafka consumer in scope
45 | *
46 | * @private
47 | * @param {Observer} observer
48 | * @param {Object | defaultFactory} kafkaConsumer
49 | */
50 | const selectorFactory =
51 | (observer, kafkaConsumer) =>
52 | /**
53 | * Handles a kafka message. Will consume the message and pass it to
54 | * the observable
55 | *
56 | * @private
57 | * @param {Array} messageSet list of messages received
58 | * @param {String} topic topic which the message was received
59 | * @param {Number} partition partion on which the message was received
60 | */
61 | (messageSet, topic, partition) =>
62 | messageSet.map(({offset, message}) => {
63 | const value = message.value;
64 | const decoded = value ? value.toString('utf8') : null;
65 | log.trace(topic, partition, offset, decoded);
66 |
67 | // pre-wired commit function as observable
68 | const commit = () =>
69 | Observable
70 | .fromPromise(kafkaConsumer
71 | .commit(topic, partition, offset));
72 |
73 | if (kafkaConsumer.opts.autoCommit) {
74 | // if we will commit here, no need to return commit function
75 | try {
76 | observer.next({ topic, partition, offset, message });
77 | // must return commit as a promise
78 | return kafkaConsumer.commit(topic, partition, offset);
79 | } catch (err) {
80 | log.error(observerError, err);
81 | }
82 | }
83 |
84 | observer.next({ topic, partition, offset, message, commit });
85 | });
86 |
87 | /**
88 | * Observable as result from messages in a kafka topic
89 | *
90 | * @private
91 | * @param {Object} consumer
92 | * @param {String} topic kafka topic
93 | * @return {Observable}
94 | */
95 | const fromTopic = (consumer, topic) => {
96 | const observable = Observable.create(observer => {
97 | consumer
98 | .subscribe(topic, selectorFactory(observer, consumer))
99 | .catch(err => observer.error(err));
100 |
101 | return () => consumer.end(topic);
102 | });
103 |
104 | observable._adapter = consumer;
105 | return observable;
106 | };
107 |
108 | /**
109 | * Consumer factory.
110 | *
111 | * @param {String} topic topic name
112 | * @param {Object} options
113 | * @param {Array|String} options.brokers comma separated list of brokers
114 | * @param {Object | defaultFactory} clientFactory
115 | * @return {Observable}
116 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options.
117 | */
118 | exports.create = function (topic, options, clientFactory = defaultFactory) {
119 | const opts = Object.assign({}, defaults, options);
120 | try {
121 | const kafkaConsumer = clientFactory.consumer(opts);
122 | return fromTopic(kafkaConsumer, topic);
123 | } catch (err) {
124 | return Observable.throw(err);
125 | }
126 | };
127 |
--------------------------------------------------------------------------------
/lib/observables/producer.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Kafka Producer Observable.
3 | *
4 | * @example producer
5 | * const producerObservable = require('./lib/producer');
6 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' };
7 | * const observable = producerObservable.create('my_topic', 'message', opts);
8 | *
9 | * observable.subscribe(result => console.info(result));
10 | *
11 | * @example producer - using arrays
12 | * const producerObservable = require('./lib/producer');
13 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' };
14 | * const messages = ['first message', 'second message'];
15 | * const observable = producerObservable.create('my_topic', messages, opts);
16 | *
17 | * observable.subscribe(result => console.info(result));
18 | *
19 | * @example producer - using observables
20 | * const producerObservable = require('./lib/producer');
21 | * const opts = { brokers: 'kafka://127.0.0.1:9092', groupId: 'me' };
22 | * const messages = Observable.from(['first message', 'second message']);
23 | * const observable = producerObservable.create('my_topic', messages, opts);
24 | *
25 | * observable.subscribe(result => console.info(result));
26 | *
27 | * @module observables/Producer
28 | * @author ghermeto
29 | **/
30 |
31 | const defaultFactory = require('../client');
32 | const Observable = require('rxjs').Observable;
33 |
34 | /**
35 | * Makes sure messages is formatted as an array or an observable
36 | *
37 | * @private
38 | * @param {*} messages
39 | * @return {Observable|Array}
40 | */
41 | const formatMessages =
42 | messages =>
43 | (messages instanceof Observable) ? messages : [].concat(messages);
44 |
45 | /**
46 | * Observable as result of publishing messages to Kafka
47 | *
48 | * @private
49 | * @param {String} topic
50 | * @param {Array|Observable} messages
51 | * @param {Object} producer
52 | */
53 | const toTopic =
54 | (topic, messages, producer) => {
55 | const observable = Observable.create(observer => {
56 | Observable.from(messages)
57 | // formats and sends the message through kafka
58 | .concatMap(message => {
59 | const {value = message, key} = message;
60 | const result = producer.publish(topic, value, key);
61 | return Observable.fromPromise(result);
62 | })
63 | // removes surrounding array and flattens
64 | .concatMap(result => Observable.from(result))
65 | .catch(err => observer.error(err))
66 | .subscribe({
67 | next: result => observer.next(result),
68 | error: err => observer.error(err),
69 | complete: () => observer.complete()
70 | });
71 | return () => {
72 | return producer.end();
73 | };
74 | });
75 |
76 | observable._adapter = producer;
77 | return observable;
78 | };
79 |
80 | /**
81 | * Producer factory.
82 | *
83 | * @param {String} topic topic name
84 | * @param {Array|Observable} messages
85 | * @param {Object} options
86 | * @param {Array|String} options.brokers comma separated list of brokers
87 | * @param {Object|defaultFactory} clientFactory client factory
88 | * @return {Observable}
89 | * @see {@link https://github.com/oleksiyk/kafka/blob/master/README.md} for default client options.
90 | */
91 | exports.create = (topic, messages, options, clientFactory = defaultFactory) => {
92 | try {
93 | const kafkaProducer = clientFactory.producer(options);
94 | return toTopic(topic, formatMessages(messages), kafkaProducer);
95 | } catch (err) {
96 | return Observable.throw(err);
97 | }
98 | };
99 |
--------------------------------------------------------------------------------
/lib/operators/json-message.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Observable operator: extracts and deserialize an object from a JSON message.
3 | *
4 | * @example
5 | * const opts = { brokers: 'kafka://127.0.0.1:9092' };
6 | * const KafkaObservable = require('kafka-observable')(opts);
7 | * KafkaObservable.fromTopic('my_topic')
8 | * .let(KafkaObservable.JSONMessage())
9 | * .subscribe(message => console.info(message));
10 | *
11 | * @module operators/JSONMessage
12 | * @author ghermeto
13 | **/
14 | 'use strict';
15 |
16 | const Observable = require('rxjs').Observable;
17 | const kafkaMessage = require('./kafka-message');
18 |
19 | const jsonMessage = (mapper = x => x) =>
20 | (source) =>
21 | Observable.create(observer =>
22 | source
23 | .let(kafkaMessage())
24 | .map(message => {
25 | try { return JSON.parse(message); }
26 | catch(err) { observer.error(err); }
27 | })
28 | .subscribe(
29 | json => {
30 | try { observer.next(mapper(json)); }
31 | catch(err) { observer.error(err); }
32 | },
33 | err => observer.error(err),
34 | () => observer.complete()));
35 |
36 | /**
37 | * @function JSONMessage
38 | * @param {Function} mapper mapping function
39 | */
40 | module.exports = jsonMessage;
--------------------------------------------------------------------------------
/lib/operators/kafka-message.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Observable operator: extracts a string value from a kafka message
3 | *
4 | * @example
5 | * const opts = { brokers: 'kafka://127.0.0.1:9092' };
6 | * const KafkaObservable = require('kafka-observable')(opts);
7 | * KafkaObservable.fromTopic('my_topic')
8 | * .let(KafkaObservable.TextMessage())
9 | * .subscribe(message => console.info(message));
10 | *
11 | * @module operators/TextMessage
12 | * @author ghermeto
13 | **/
14 | 'use strict';
15 |
16 | const Observable = require('rxjs').Observable;
17 |
18 | const kafkaMessage = (mapper = x => x) =>
19 | (source) =>
20 | Observable.create(observer =>
21 | source
22 | .map(({message}) => message.value.toString('utf8'))
23 | .subscribe(
24 | message => {
25 | try { observer.next(mapper(message)); }
26 | catch(err) { observer.error(err); }
27 | },
28 | err => observer.error(err),
29 | () => observer.complete()));
30 |
31 | /**
32 | * @function TextMessage
33 | * @param {Function} mapper mapping function
34 | */
35 | module.exports = kafkaMessage;
--------------------------------------------------------------------------------
/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "kafka-observable",
3 | "version": "0.0.3",
4 | "lockfileVersion": 1,
5 | "requires": true,
6 | "dependencies": {
7 | "@types/bluebird": {
8 | "version": "3.5.0",
9 | "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.0.tgz",
10 | "integrity": "sha1-JjNHCk6r6aR82aRf2yDtX5NAe8o="
11 | },
12 | "@types/lodash": {
13 | "version": "4.14.107",
14 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.107.tgz",
15 | "integrity": "sha512-afvjfP2rl3yvtv2qrCRN23zIQcDinF+munMJCoHEw2BXF22QJogTlVfNPTACQ6ieDyA6VnyKT4WLuN/wK368ng=="
16 | },
17 | "JSONStream": {
18 | "version": "1.3.2",
19 | "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz",
20 | "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=",
21 | "dev": true,
22 | "requires": {
23 | "jsonparse": "1.3.1",
24 | "through": "2.3.8"
25 | }
26 | },
27 | "abbrev": {
28 | "version": "1.0.9",
29 | "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz",
30 | "integrity": "sha1-kbR5JYinc4wl813W9jdSovh3YTU=",
31 | "dev": true
32 | },
33 | "align-text": {
34 | "version": "0.1.4",
35 | "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz",
36 | "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=",
37 | "dev": true,
38 | "requires": {
39 | "kind-of": "3.2.2",
40 | "longest": "1.0.1",
41 | "repeat-string": "1.6.1"
42 | }
43 | },
44 | "amdefine": {
45 | "version": "1.0.1",
46 | "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz",
47 | "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=",
48 | "dev": true
49 | },
50 | "argparse": {
51 | "version": "1.0.10",
52 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
53 | "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
54 | "dev": true,
55 | "requires": {
56 | "sprintf-js": "1.0.3"
57 | }
58 | },
59 | "async": {
60 | "version": "1.5.2",
61 | "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
62 | "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
63 | "dev": true
64 | },
65 | "babylon": {
66 | "version": "7.0.0-beta.19",
67 | "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.19.tgz",
68 | "integrity": "sha512-Vg0C9s/REX6/WIXN37UKpv5ZhRi6A4pjHlpkE34+8/a6c2W1Q692n3hmc+SZG5lKRnaExLUbxtJ1SVT+KaCQ/A==",
69 | "dev": true
70 | },
71 | "balanced-match": {
72 | "version": "1.0.0",
73 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
74 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
75 | },
76 | "bin-protocol": {
77 | "version": "3.0.4",
78 | "resolved": "https://registry.npmjs.org/bin-protocol/-/bin-protocol-3.0.4.tgz",
79 | "integrity": "sha1-RlqdNQb+sOEmtStbIWDZNuFbJ/Q=",
80 | "requires": {
81 | "lodash": "4.16.4",
82 | "long": "3.2.0",
83 | "protocol-buffers-schema": "3.3.2"
84 | }
85 | },
86 | "bl": {
87 | "version": "1.2.2",
88 | "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
89 | "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
90 | "dev": true,
91 | "requires": {
92 | "readable-stream": "2.3.6",
93 | "safe-buffer": "5.1.1"
94 | },
95 | "dependencies": {
96 | "process-nextick-args": {
97 | "version": "2.0.0",
98 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
99 | "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==",
100 | "dev": true
101 | },
102 | "readable-stream": {
103 | "version": "2.3.6",
104 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
105 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
106 | "dev": true,
107 | "requires": {
108 | "core-util-is": "1.0.2",
109 | "inherits": "2.0.3",
110 | "isarray": "1.0.0",
111 | "process-nextick-args": "2.0.0",
112 | "safe-buffer": "5.1.1",
113 | "string_decoder": "1.1.1",
114 | "util-deprecate": "1.0.2"
115 | }
116 | },
117 | "string_decoder": {
118 | "version": "1.1.1",
119 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
120 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
121 | "dev": true,
122 | "requires": {
123 | "safe-buffer": "5.1.1"
124 | }
125 | }
126 | }
127 | },
128 | "bluebird": {
129 | "version": "3.5.1",
130 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz",
131 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA=="
132 | },
133 | "brace-expansion": {
134 | "version": "1.1.11",
135 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
136 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
137 | "requires": {
138 | "balanced-match": "1.0.0",
139 | "concat-map": "0.0.1"
140 | }
141 | },
142 | "buffer-crc32": {
143 | "version": "0.2.13",
144 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
145 | "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
146 | },
147 | "bunyan": {
148 | "version": "1.8.12",
149 | "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz",
150 | "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=",
151 | "requires": {
152 | "dtrace-provider": "0.8.6",
153 | "moment": "2.22.1",
154 | "mv": "2.1.1",
155 | "safe-json-stringify": "1.1.0"
156 | }
157 | },
158 | "caller-id": {
159 | "version": "0.1.0",
160 | "resolved": "https://registry.npmjs.org/caller-id/-/caller-id-0.1.0.tgz",
161 | "integrity": "sha1-Wb2sCJPRLDhxQIJ5Ix+XRYNk8Hs=",
162 | "dev": true,
163 | "requires": {
164 | "stack-trace": "0.0.10"
165 | }
166 | },
167 | "camelcase": {
168 | "version": "1.2.1",
169 | "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
170 | "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=",
171 | "dev": true,
172 | "optional": true
173 | },
174 | "catharsis": {
175 | "version": "0.8.9",
176 | "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.9.tgz",
177 | "integrity": "sha1-mMyJDKZS3S7w5ws3klMQ/56Q/Is=",
178 | "dev": true,
179 | "requires": {
180 | "underscore-contrib": "0.3.0"
181 | }
182 | },
183 | "center-align": {
184 | "version": "0.1.3",
185 | "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz",
186 | "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=",
187 | "dev": true,
188 | "optional": true,
189 | "requires": {
190 | "align-text": "0.1.4",
191 | "lazy-cache": "1.0.4"
192 | }
193 | },
194 | "cliui": {
195 | "version": "2.1.0",
196 | "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz",
197 | "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=",
198 | "dev": true,
199 | "optional": true,
200 | "requires": {
201 | "center-align": "0.1.3",
202 | "right-align": "0.1.3",
203 | "wordwrap": "0.0.2"
204 | },
205 | "dependencies": {
206 | "wordwrap": {
207 | "version": "0.0.2",
208 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz",
209 | "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=",
210 | "dev": true,
211 | "optional": true
212 | }
213 | }
214 | },
215 | "concat-map": {
216 | "version": "0.0.1",
217 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
218 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
219 | },
220 | "concat-stream": {
221 | "version": "1.5.2",
222 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz",
223 | "integrity": "sha1-cIl4Yk2FavQaWnQd790mHadSwmY=",
224 | "dev": true,
225 | "requires": {
226 | "inherits": "2.0.3",
227 | "readable-stream": "2.0.6",
228 | "typedarray": "0.0.6"
229 | }
230 | },
231 | "connection-parse": {
232 | "version": "0.0.7",
233 | "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz",
234 | "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk="
235 | },
236 | "core-util-is": {
237 | "version": "1.0.2",
238 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
239 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
240 | "dev": true
241 | },
242 | "debug": {
243 | "version": "3.1.0",
244 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
245 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
246 | "dev": true,
247 | "requires": {
248 | "ms": "2.0.0"
249 | }
250 | },
251 | "decamelize": {
252 | "version": "1.2.0",
253 | "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
254 | "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
255 | "dev": true,
256 | "optional": true
257 | },
258 | "deep-is": {
259 | "version": "0.1.3",
260 | "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
261 | "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
262 | "dev": true
263 | },
264 | "docker-modem": {
265 | "version": "1.0.6",
266 | "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.6.tgz",
267 | "integrity": "sha512-kDwWa5QaiVMB8Orbb7nXdGdwEZHKfEm7iPwglXe1KorImMpmGNlhC7A5LG0p8rrCcz1J4kJhq/o63lFjDdj8rQ==",
268 | "dev": true,
269 | "requires": {
270 | "JSONStream": "1.3.2",
271 | "debug": "3.1.0",
272 | "readable-stream": "1.0.34",
273 | "split-ca": "1.0.1"
274 | },
275 | "dependencies": {
276 | "isarray": {
277 | "version": "0.0.1",
278 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
279 | "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
280 | "dev": true
281 | },
282 | "readable-stream": {
283 | "version": "1.0.34",
284 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
285 | "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
286 | "dev": true,
287 | "requires": {
288 | "core-util-is": "1.0.2",
289 | "inherits": "2.0.3",
290 | "isarray": "0.0.1",
291 | "string_decoder": "0.10.31"
292 | }
293 | }
294 | }
295 | },
296 | "dockerode": {
297 | "version": "2.5.5",
298 | "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.5.5.tgz",
299 | "integrity": "sha512-H3HX18xKmy51wqpPHvGDwPOotJMy9l/AWfiaVu4imrgBGr384rINEB2FwTwoYU++krkZjseVYyiVK8CnRz2tkw==",
300 | "dev": true,
301 | "requires": {
302 | "concat-stream": "1.5.2",
303 | "docker-modem": "1.0.6",
304 | "tar-fs": "1.12.0"
305 | }
306 | },
307 | "dtrace-provider": {
308 | "version": "0.8.6",
309 | "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.6.tgz",
310 | "integrity": "sha1-QooiOv4DQl0s1tY0f99AxmkDVj0=",
311 | "optional": true,
312 | "requires": {
313 | "nan": "2.10.0"
314 | }
315 | },
316 | "end-of-stream": {
317 | "version": "1.4.1",
318 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
319 | "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
320 | "dev": true,
321 | "requires": {
322 | "once": "1.4.0"
323 | }
324 | },
325 | "escape-string-regexp": {
326 | "version": "1.0.5",
327 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
328 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
329 | "dev": true
330 | },
331 | "escodegen": {
332 | "version": "1.8.1",
333 | "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.8.1.tgz",
334 | "integrity": "sha1-WltTr0aTEQvrsIZ6o0MN07cKEBg=",
335 | "dev": true,
336 | "requires": {
337 | "esprima": "2.7.3",
338 | "estraverse": "1.9.3",
339 | "esutils": "2.0.2",
340 | "optionator": "0.8.2",
341 | "source-map": "0.2.0"
342 | }
343 | },
344 | "esprima": {
345 | "version": "2.7.3",
346 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz",
347 | "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE=",
348 | "dev": true
349 | },
350 | "estraverse": {
351 | "version": "1.9.3",
352 | "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.9.3.tgz",
353 | "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=",
354 | "dev": true
355 | },
356 | "esutils": {
357 | "version": "2.0.2",
358 | "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
359 | "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=",
360 | "dev": true
361 | },
362 | "exit": {
363 | "version": "0.1.2",
364 | "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz",
365 | "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=",
366 | "dev": true
367 | },
368 | "fast-levenshtein": {
369 | "version": "2.0.6",
370 | "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
371 | "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
372 | "dev": true
373 | },
374 | "fs.realpath": {
375 | "version": "1.0.0",
376 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
377 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
378 | "dev": true
379 | },
380 | "glob": {
381 | "version": "6.0.4",
382 | "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
383 | "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
384 | "optional": true,
385 | "requires": {
386 | "inflight": "1.0.6",
387 | "inherits": "2.0.3",
388 | "minimatch": "3.0.4",
389 | "once": "1.4.0",
390 | "path-is-absolute": "1.0.1"
391 | }
392 | },
393 | "graceful-fs": {
394 | "version": "4.1.11",
395 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
396 | "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=",
397 | "dev": true
398 | },
399 | "handlebars": {
400 | "version": "4.0.11",
401 | "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz",
402 | "integrity": "sha1-Ywo13+ApS8KB7a5v/F0yn8eYLcw=",
403 | "dev": true,
404 | "requires": {
405 | "async": "1.5.2",
406 | "optimist": "0.6.1",
407 | "source-map": "0.4.4",
408 | "uglify-js": "2.8.29"
409 | },
410 | "dependencies": {
411 | "source-map": {
412 | "version": "0.4.4",
413 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
414 | "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
415 | "dev": true,
416 | "requires": {
417 | "amdefine": "1.0.1"
418 | }
419 | }
420 | }
421 | },
422 | "has-flag": {
423 | "version": "1.0.0",
424 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz",
425 | "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=",
426 | "dev": true
427 | },
428 | "hashring": {
429 | "version": "3.2.0",
430 | "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz",
431 | "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=",
432 | "requires": {
433 | "connection-parse": "0.0.7",
434 | "simple-lru-cache": "0.0.2"
435 | }
436 | },
437 | "inflight": {
438 | "version": "1.0.6",
439 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
440 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
441 | "requires": {
442 | "once": "1.4.0",
443 | "wrappy": "1.0.2"
444 | }
445 | },
446 | "inherits": {
447 | "version": "2.0.3",
448 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
449 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
450 | },
451 | "is-buffer": {
452 | "version": "1.1.6",
453 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
454 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
455 | "dev": true
456 | },
457 | "isarray": {
458 | "version": "1.0.0",
459 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
460 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
461 | "dev": true
462 | },
463 | "isexe": {
464 | "version": "2.0.0",
465 | "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
466 | "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
467 | "dev": true
468 | },
469 | "istanbul": {
470 | "version": "0.4.5",
471 | "resolved": "https://registry.npmjs.org/istanbul/-/istanbul-0.4.5.tgz",
472 | "integrity": "sha1-ZcfXPUxNqE1POsMQuRj7C4Azczs=",
473 | "dev": true,
474 | "requires": {
475 | "abbrev": "1.0.9",
476 | "async": "1.5.2",
477 | "escodegen": "1.8.1",
478 | "esprima": "2.7.3",
479 | "glob": "5.0.15",
480 | "handlebars": "4.0.11",
481 | "js-yaml": "3.11.0",
482 | "mkdirp": "0.5.1",
483 | "nopt": "3.0.6",
484 | "once": "1.4.0",
485 | "resolve": "1.1.7",
486 | "supports-color": "3.2.3",
487 | "which": "1.3.0",
488 | "wordwrap": "1.0.0"
489 | },
490 | "dependencies": {
491 | "glob": {
492 | "version": "5.0.15",
493 | "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz",
494 | "integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=",
495 | "dev": true,
496 | "requires": {
497 | "inflight": "1.0.6",
498 | "inherits": "2.0.3",
499 | "minimatch": "3.0.4",
500 | "once": "1.4.0",
501 | "path-is-absolute": "1.0.1"
502 | }
503 | }
504 | }
505 | },
506 | "jasmine": {
507 | "version": "2.99.0",
508 | "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-2.99.0.tgz",
509 | "integrity": "sha1-jKctEC5jm4Z8ZImFbg4YqceqQrc=",
510 | "dev": true,
511 | "requires": {
512 | "exit": "0.1.2",
513 | "glob": "7.1.2",
514 | "jasmine-core": "2.99.1"
515 | },
516 | "dependencies": {
517 | "glob": {
518 | "version": "7.1.2",
519 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
520 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
521 | "dev": true,
522 | "requires": {
523 | "fs.realpath": "1.0.0",
524 | "inflight": "1.0.6",
525 | "inherits": "2.0.3",
526 | "minimatch": "3.0.4",
527 | "once": "1.4.0",
528 | "path-is-absolute": "1.0.1"
529 | }
530 | }
531 | }
532 | },
533 | "jasmine-core": {
534 | "version": "2.99.1",
535 | "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-2.99.1.tgz",
536 | "integrity": "sha1-5kAN8ea1bhMLYcS80JPap/boyhU=",
537 | "dev": true
538 | },
539 | "js-yaml": {
540 | "version": "3.11.0",
541 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz",
542 | "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==",
543 | "dev": true,
544 | "requires": {
545 | "argparse": "1.0.10",
546 | "esprima": "4.0.0"
547 | },
548 | "dependencies": {
549 | "esprima": {
550 | "version": "4.0.0",
551 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
552 | "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw==",
553 | "dev": true
554 | }
555 | }
556 | },
557 | "js2xmlparser": {
558 | "version": "3.0.0",
559 | "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-3.0.0.tgz",
560 | "integrity": "sha1-P7YOqgicVED5MZ9RdgzNB+JJlzM=",
561 | "dev": true,
562 | "requires": {
563 | "xmlcreate": "1.0.2"
564 | }
565 | },
566 | "jsdoc": {
567 | "version": "3.5.5",
568 | "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-3.5.5.tgz",
569 | "integrity": "sha512-6PxB65TAU4WO0Wzyr/4/YhlGovXl0EVYfpKbpSroSj0qBxT4/xod/l40Opkm38dRHRdQgdeY836M0uVnJQG7kg==",
570 | "dev": true,
571 | "requires": {
572 | "babylon": "7.0.0-beta.19",
573 | "bluebird": "3.5.1",
574 | "catharsis": "0.8.9",
575 | "escape-string-regexp": "1.0.5",
576 | "js2xmlparser": "3.0.0",
577 | "klaw": "2.0.0",
578 | "marked": "0.3.19",
579 | "mkdirp": "0.5.1",
580 | "requizzle": "0.2.1",
581 | "strip-json-comments": "2.0.1",
582 | "taffydb": "2.6.2",
583 | "underscore": "1.8.3"
584 | }
585 | },
586 | "jsonparse": {
587 | "version": "1.3.1",
588 | "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
589 | "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=",
590 | "dev": true
591 | },
592 | "kind-of": {
593 | "version": "3.2.2",
594 | "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
595 | "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
596 | "dev": true,
597 | "requires": {
598 | "is-buffer": "1.1.6"
599 | }
600 | },
601 | "klaw": {
602 | "version": "2.0.0",
603 | "resolved": "https://registry.npmjs.org/klaw/-/klaw-2.0.0.tgz",
604 | "integrity": "sha1-WcEo4Nxc5BAgEVEZTuucv4WGUPY=",
605 | "dev": true,
606 | "requires": {
607 | "graceful-fs": "4.1.11"
608 | }
609 | },
610 | "lazy-cache": {
611 | "version": "1.0.4",
612 | "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
613 | "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=",
614 | "dev": true,
615 | "optional": true
616 | },
617 | "levn": {
618 | "version": "0.3.0",
619 | "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
620 | "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
621 | "dev": true,
622 | "requires": {
623 | "prelude-ls": "1.1.2",
624 | "type-check": "0.3.2"
625 | }
626 | },
627 | "lodash": {
628 | "version": "4.16.4",
629 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.16.4.tgz",
630 | "integrity": "sha1-Ac4wa5utExnypVKGdPiCl663ASc="
631 | },
632 | "long": {
633 | "version": "3.2.0",
634 | "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz",
635 | "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s="
636 | },
637 | "longest": {
638 | "version": "1.0.1",
639 | "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz",
640 | "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=",
641 | "dev": true
642 | },
643 | "marked": {
644 | "version": "0.3.19",
645 | "resolved": "https://registry.npmjs.org/marked/-/marked-0.3.19.tgz",
646 | "integrity": "sha512-ea2eGWOqNxPcXv8dyERdSr/6FmzvWwzjMxpfGB/sbMccXoct+xY+YukPD+QTUZwyvK7BZwcr4m21WBOW41pAkg==",
647 | "dev": true
648 | },
649 | "minimatch": {
650 | "version": "3.0.4",
651 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
652 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
653 | "requires": {
654 | "brace-expansion": "1.1.11"
655 | }
656 | },
657 | "minimist": {
658 | "version": "0.0.8",
659 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
660 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
661 | },
662 | "mkdirp": {
663 | "version": "0.5.1",
664 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
665 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
666 | "requires": {
667 | "minimist": "0.0.8"
668 | }
669 | },
670 | "mock-require": {
671 | "version": "2.0.2",
672 | "resolved": "https://registry.npmjs.org/mock-require/-/mock-require-2.0.2.tgz",
673 | "integrity": "sha1-HqpxqtIwE3c9En3H6Ro/u0g31g0=",
674 | "dev": true,
675 | "requires": {
676 | "caller-id": "0.1.0"
677 | }
678 | },
679 | "moment": {
680 | "version": "2.22.1",
681 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.1.tgz",
682 | "integrity": "sha512-shJkRTSebXvsVqk56I+lkb2latjBs8I+pc2TzWc545y2iFnSjm7Wg0QMh+ZWcdSLQyGEau5jI8ocnmkyTgr9YQ==",
683 | "optional": true
684 | },
685 | "ms": {
686 | "version": "2.0.0",
687 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
688 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
689 | "dev": true
690 | },
691 | "murmur-hash-js": {
692 | "version": "1.0.0",
693 | "resolved": "https://registry.npmjs.org/murmur-hash-js/-/murmur-hash-js-1.0.0.tgz",
694 | "integrity": "sha1-UEEEkmnJZjPIZjhpYLL0KJ515bA="
695 | },
696 | "mv": {
697 | "version": "2.1.1",
698 | "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
699 | "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
700 | "optional": true,
701 | "requires": {
702 | "mkdirp": "0.5.1",
703 | "ncp": "2.0.0",
704 | "rimraf": "2.4.5"
705 | }
706 | },
707 | "nan": {
708 | "version": "2.10.0",
709 | "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
710 | "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==",
711 | "optional": true
712 | },
713 | "ncp": {
714 | "version": "2.0.0",
715 | "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
716 | "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
717 | "optional": true
718 | },
719 | "nice-simple-logger": {
720 | "version": "1.0.1",
721 | "resolved": "https://registry.npmjs.org/nice-simple-logger/-/nice-simple-logger-1.0.1.tgz",
722 | "integrity": "sha1-D55khSe+e+PkmrdvqMjAmK+VG/Y=",
723 | "requires": {
724 | "lodash": "4.16.4"
725 | }
726 | },
727 | "no-kafka": {
728 | "version": "3.2.9",
729 | "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.2.9.tgz",
730 | "integrity": "sha1-jLSk8aDVDqYUXFvAZ6A1Dl5CmMc=",
731 | "requires": {
732 | "@types/bluebird": "3.5.0",
733 | "@types/lodash": "4.14.107",
734 | "bin-protocol": "3.0.4",
735 | "bluebird": "3.5.1",
736 | "buffer-crc32": "0.2.13",
737 | "hashring": "3.2.0",
738 | "lodash": "4.16.4",
739 | "murmur-hash-js": "1.0.0",
740 | "nice-simple-logger": "1.0.1",
741 | "wrr-pool": "1.1.3"
742 | }
743 | },
744 | "nopt": {
745 | "version": "3.0.6",
746 | "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
747 | "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
748 | "dev": true,
749 | "requires": {
750 | "abbrev": "1.0.9"
751 | }
752 | },
753 | "once": {
754 | "version": "1.4.0",
755 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
756 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
757 | "requires": {
758 | "wrappy": "1.0.2"
759 | }
760 | },
761 | "optimist": {
762 | "version": "0.6.1",
763 | "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
764 | "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
765 | "dev": true,
766 | "requires": {
767 | "minimist": "0.0.8",
768 | "wordwrap": "0.0.3"
769 | },
770 | "dependencies": {
771 | "wordwrap": {
772 | "version": "0.0.3",
773 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz",
774 | "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=",
775 | "dev": true
776 | }
777 | }
778 | },
779 | "optionator": {
780 | "version": "0.8.2",
781 | "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
782 | "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=",
783 | "dev": true,
784 | "requires": {
785 | "deep-is": "0.1.3",
786 | "fast-levenshtein": "2.0.6",
787 | "levn": "0.3.0",
788 | "prelude-ls": "1.1.2",
789 | "type-check": "0.3.2",
790 | "wordwrap": "1.0.0"
791 | }
792 | },
793 | "path-is-absolute": {
794 | "version": "1.0.1",
795 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
796 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
797 | },
798 | "prelude-ls": {
799 | "version": "1.1.2",
800 | "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
801 | "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
802 | "dev": true
803 | },
804 | "process-nextick-args": {
805 | "version": "1.0.7",
806 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
807 | "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=",
808 | "dev": true
809 | },
810 | "protocol-buffers-schema": {
811 | "version": "3.3.2",
812 | "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz",
813 | "integrity": "sha512-Xdayp8sB/mU+sUV4G7ws8xtYMGdQnxbeIfLjyO9TZZRJdztBGhlmbI5x1qcY4TG5hBkIKGnc28i7nXxaugu88w=="
814 | },
815 | "pump": {
816 | "version": "1.0.3",
817 | "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz",
818 | "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==",
819 | "dev": true,
820 | "requires": {
821 | "end-of-stream": "1.4.1",
822 | "once": "1.4.0"
823 | }
824 | },
825 | "readable-stream": {
826 | "version": "2.0.6",
827 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
828 | "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=",
829 | "dev": true,
830 | "requires": {
831 | "core-util-is": "1.0.2",
832 | "inherits": "2.0.3",
833 | "isarray": "1.0.0",
834 | "process-nextick-args": "1.0.7",
835 | "string_decoder": "0.10.31",
836 | "util-deprecate": "1.0.2"
837 | }
838 | },
839 | "repeat-string": {
840 | "version": "1.6.1",
841 | "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
842 | "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=",
843 | "dev": true
844 | },
845 | "requizzle": {
846 | "version": "0.2.1",
847 | "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.1.tgz",
848 | "integrity": "sha1-aUPDUwxNmn5G8c3dUcFY/GcM294=",
849 | "dev": true,
850 | "requires": {
851 | "underscore": "1.6.0"
852 | },
853 | "dependencies": {
854 | "underscore": {
855 | "version": "1.6.0",
856 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz",
857 | "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=",
858 | "dev": true
859 | }
860 | }
861 | },
862 | "resolve": {
863 | "version": "1.1.7",
864 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz",
865 | "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=",
866 | "dev": true
867 | },
868 | "right-align": {
869 | "version": "0.1.3",
870 | "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
871 | "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=",
872 | "dev": true,
873 | "optional": true,
874 | "requires": {
875 | "align-text": "0.1.4"
876 | }
877 | },
878 | "rimraf": {
879 | "version": "2.4.5",
880 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
881 | "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
882 | "optional": true,
883 | "requires": {
884 | "glob": "6.0.4"
885 | }
886 | },
887 | "rxjs": {
888 | "version": "5.5.10",
889 | "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.10.tgz",
890 | "integrity": "sha512-SRjimIDUHJkon+2hFo7xnvNC4ZEHGzCRwh9P7nzX3zPkCGFEg/tuElrNR7L/rZMagnK2JeH2jQwPRpmyXyLB6A==",
891 | "requires": {
892 | "symbol-observable": "1.0.1"
893 | }
894 | },
895 | "safe-buffer": {
896 | "version": "5.1.1",
897 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
898 | "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==",
899 | "dev": true
900 | },
901 | "safe-json-stringify": {
902 | "version": "1.1.0",
903 | "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.1.0.tgz",
904 | "integrity": "sha512-EzBtUaFH9bHYPc69wqjp0efJI/DPNHdFbGE3uIMn4sVbO0zx8vZ8cG4WKxQfOpUOKsQyGBiT2mTqnCw+6nLswA==",
905 | "optional": true
906 | },
907 | "shortid": {
908 | "version": "2.2.8",
909 | "resolved": "https://registry.npmjs.org/shortid/-/shortid-2.2.8.tgz",
910 | "integrity": "sha1-AzsRfWoul1gE9vCWnb59PQs1UTE="
911 | },
912 | "simple-lru-cache": {
913 | "version": "0.0.2",
914 | "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz",
915 | "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0="
916 | },
917 | "source-map": {
918 | "version": "0.2.0",
919 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.2.0.tgz",
920 | "integrity": "sha1-2rc/vPwrqBm03gO9b26qSBZLP50=",
921 | "dev": true,
922 | "optional": true,
923 | "requires": {
924 | "amdefine": "1.0.1"
925 | }
926 | },
927 | "split-ca": {
928 | "version": "1.0.1",
929 | "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz",
930 | "integrity": "sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=",
931 | "dev": true
932 | },
933 | "sprintf-js": {
934 | "version": "1.0.3",
935 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
936 | "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
937 | "dev": true
938 | },
939 | "stack-trace": {
940 | "version": "0.0.10",
941 | "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
942 | "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=",
943 | "dev": true
944 | },
945 | "string_decoder": {
946 | "version": "0.10.31",
947 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
948 | "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
949 | "dev": true
950 | },
951 | "strip-json-comments": {
952 | "version": "2.0.1",
953 | "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
954 | "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
955 | "dev": true
956 | },
957 | "supports-color": {
958 | "version": "3.2.3",
959 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
960 | "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
961 | "dev": true,
962 | "requires": {
963 | "has-flag": "1.0.0"
964 | }
965 | },
966 | "symbol-observable": {
967 | "version": "1.0.1",
968 | "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.1.tgz",
969 | "integrity": "sha1-g0D8RwLDEi310iKI+IKD9RPT/dQ="
970 | },
971 | "taffydb": {
972 | "version": "2.6.2",
973 | "resolved": "https://registry.npmjs.org/taffydb/-/taffydb-2.6.2.tgz",
974 | "integrity": "sha1-fLy2S1oUG2ou/CxdLGe04VCyomg=",
975 | "dev": true
976 | },
977 | "tar-fs": {
978 | "version": "1.12.0",
979 | "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.12.0.tgz",
980 | "integrity": "sha1-pqgFU9ilTHPeHQrg553ncDVgXh0=",
981 | "dev": true,
982 | "requires": {
983 | "mkdirp": "0.5.1",
984 | "pump": "1.0.3",
985 | "tar-stream": "1.5.5"
986 | }
987 | },
988 | "tar-stream": {
989 | "version": "1.5.5",
990 | "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.5.5.tgz",
991 | "integrity": "sha512-mQdgLPc/Vjfr3VWqWbfxW8yQNiJCbAZ+Gf6GDu1Cy0bdb33ofyiNGBtAY96jHFhDuivCwgW1H9DgTON+INiXgg==",
992 | "dev": true,
993 | "requires": {
994 | "bl": "1.2.2",
995 | "end-of-stream": "1.4.1",
996 | "readable-stream": "2.0.6",
997 | "xtend": "4.0.1"
998 | }
999 | },
1000 | "through": {
1001 | "version": "2.3.8",
1002 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
1003 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
1004 | "dev": true
1005 | },
1006 | "type-check": {
1007 | "version": "0.3.2",
1008 | "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
1009 | "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
1010 | "dev": true,
1011 | "requires": {
1012 | "prelude-ls": "1.1.2"
1013 | }
1014 | },
1015 | "typedarray": {
1016 | "version": "0.0.6",
1017 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
1018 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=",
1019 | "dev": true
1020 | },
1021 | "uglify-js": {
1022 | "version": "2.8.29",
1023 | "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz",
1024 | "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=",
1025 | "dev": true,
1026 | "optional": true,
1027 | "requires": {
1028 | "source-map": "0.5.7",
1029 | "uglify-to-browserify": "1.0.2",
1030 | "yargs": "3.10.0"
1031 | },
1032 | "dependencies": {
1033 | "source-map": {
1034 | "version": "0.5.7",
1035 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
1036 | "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=",
1037 | "dev": true,
1038 | "optional": true
1039 | }
1040 | }
1041 | },
1042 | "uglify-to-browserify": {
1043 | "version": "1.0.2",
1044 | "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz",
1045 | "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=",
1046 | "dev": true,
1047 | "optional": true
1048 | },
1049 | "underscore": {
1050 | "version": "1.8.3",
1051 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
1052 | "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=",
1053 | "dev": true
1054 | },
1055 | "underscore-contrib": {
1056 | "version": "0.3.0",
1057 | "resolved": "https://registry.npmjs.org/underscore-contrib/-/underscore-contrib-0.3.0.tgz",
1058 | "integrity": "sha1-ZltmwkeD+PorGMn4y7Dix9SMJsc=",
1059 | "dev": true,
1060 | "requires": {
1061 | "underscore": "1.6.0"
1062 | },
1063 | "dependencies": {
1064 | "underscore": {
1065 | "version": "1.6.0",
1066 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz",
1067 | "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=",
1068 | "dev": true
1069 | }
1070 | }
1071 | },
1072 | "util-deprecate": {
1073 | "version": "1.0.2",
1074 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
1075 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
1076 | "dev": true
1077 | },
1078 | "which": {
1079 | "version": "1.3.0",
1080 | "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz",
1081 | "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==",
1082 | "dev": true,
1083 | "requires": {
1084 | "isexe": "2.0.0"
1085 | }
1086 | },
1087 | "window-size": {
1088 | "version": "0.1.0",
1089 | "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz",
1090 | "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=",
1091 | "dev": true,
1092 | "optional": true
1093 | },
1094 | "wordwrap": {
1095 | "version": "1.0.0",
1096 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
1097 | "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=",
1098 | "dev": true
1099 | },
1100 | "wrappy": {
1101 | "version": "1.0.2",
1102 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
1103 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
1104 | },
1105 | "wrr-pool": {
1106 | "version": "1.1.3",
1107 | "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.3.tgz",
1108 | "integrity": "sha1-/a0i8uofMDY//l14HPeUl6d/8H4=",
1109 | "requires": {
1110 | "lodash": "4.16.4"
1111 | }
1112 | },
1113 | "xmlcreate": {
1114 | "version": "1.0.2",
1115 | "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-1.0.2.tgz",
1116 | "integrity": "sha1-+mv3YqYKQT+z3Y9LA8WyaSONMI8=",
1117 | "dev": true
1118 | },
1119 | "xtend": {
1120 | "version": "4.0.1",
1121 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
1122 | "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=",
1123 | "dev": true
1124 | },
1125 | "yargs": {
1126 | "version": "3.10.0",
1127 | "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
1128 | "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
1129 | "dev": true,
1130 | "optional": true,
1131 | "requires": {
1132 | "camelcase": "1.2.1",
1133 | "cliui": "2.1.0",
1134 | "decamelize": "1.2.0",
1135 | "window-size": "0.1.0"
1136 | }
1137 | }
1138 | }
1139 | }
1140 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "kafka-observable",
3 | "version": "0.0.3",
4 | "description": "kafka consumer and producer as observables",
5 | "main": "index.js",
6 | "scripts": {
7 | "prepublish": "npm test",
8 | "test": "npm run unit-test",
9 | "clean": "rm -rf ./node_modules ./out ./coverage",
10 | "gen-docs": "node_modules/.bin/jsdoc -r -R README.md -P package.json index.js lib/observables/* lib/operators/*",
11 | "coverage": "JASMINE_CONFIG_PATH=test/support/unit.json node_modules/.bin/istanbul cover --include-all-sources node_modules/.bin/jasmine",
12 | "unit-test": "JASMINE_CONFIG_PATH=test/support/unit.json node_modules/.bin/jasmine",
13 | "integration-test": "LOG_LEVEL=trace JASMINE_CONFIG_PATH=test/support/integration.json node_modules/.bin/jasmine"
14 | },
15 | "engines": {
16 | "node": ">=6.4.0"
17 | },
18 | "keywords": [
19 | "kafka",
20 | "rx",
21 | "observable"
22 | ],
23 | "author": "gui.hermeto@gmail.com",
24 | "license": "MIT",
25 | "repository": {
26 | "type": "git",
27 | "url": "https://github.com/ghermeto/kafka-observable.git"
28 | },
29 | "publishConfig": {
30 | "registry": "https://registry.npmjs.org/"
31 | },
32 | "dependencies": {
33 | "bunyan": "^1.8.12",
34 | "no-kafka": "^3.2.6",
35 | "rxjs": "^5.5.2",
36 | "shortid": "^2.2.8"
37 | },
38 | "devDependencies": {
39 | "dockerode": "^2.5.3",
40 | "istanbul": "^0.4.5",
41 | "jasmine": "^2.8.0",
42 | "jsdoc": "^3.5.5",
43 | "mock-require": "^2.0.2"
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/test/integration/helpers/helper.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Docker = require('dockerode');
4 | jasmine.DEFAULT_TIMEOUT_INTERVAL = 15000;
5 |
6 | const topic = 'test_kafka';
7 | const createTopic = () => ([
8 | '/opt/kafka_2.11-0.10.1.0/bin/kafka-topics.sh',
9 | '--create',
10 | '--zookeeper', 'localhost:2181',
11 | '--replication-factor', '1',
12 | '--partitions', '1',
13 | '--topic', topic
14 | ]);
15 |
16 | /**
17 | * @type {Docker}
18 | */
19 | global.docker = new Docker();
20 |
21 | /**
22 | * end handler for jasmine
23 | * @param {Function} done jasmine done function
24 | * @return {Function} handler that calls done the expected way
25 | **/
26 | global.finish_test = function finish_test(done) {
27 | return function (err) {
28 | if (err) { done.fail(err); }
29 | done();
30 | };
31 | };
32 |
33 | /**
34 | * delay func
35 | * @param {Number} millis time in milliseconds
36 | * @return {Promise} resolves after given time
37 | */
38 | global.delay = millis =>
39 | new Promise((resolve) => setTimeout(()=> resolve(), millis));
40 |
41 | /**
42 | * Before integration tests starts, create the container, start it and create
43 | * the topic in kafka required for testing.
44 | */
45 | beforeAll(function (done) {
46 | docker.createContainer({
47 | Image: 'spotify/kafka',
48 | AttachStdout: true,
49 | AttachStderr: true,
50 | Hostname: 'kafka',
51 | StopTimeout: 5,
52 | ExposedPorts: { '9092/tcp':{} },
53 | HostConfig: { PortBindings: { '9092/tcp': [{ HostPort: '9092' }] } }
54 | })
55 | .then(container => container.start())
56 | .then(container => global.container = container)
57 | .then(() => console.info('waiting for kafka to start...') || delay(6000))
58 | .then(() => container.exec({ Cmd: createTopic() }))
59 | .then(exec => exec.start())
60 | .then(() => delay(3000))
61 | .then(() => console.info(`${topic} topic created...`) || done());
62 | });
63 |
64 | /**
65 | * test to see if container was successfully started
66 | */
67 | it('should have a valid container', done => {
68 | container.inspect(function (err, data) {
69 | if (err || !data) { done.fail(err || 'no container data'); }
70 | done();
71 | });
72 | });
73 |
74 | /**
75 | * shutdown procedure. We need to stop and remove the container not only on
76 | * afterAll, but anytime we stop test tests.
77 | */
78 | const gracefulShutdown = () => global.container
79 | .stop()
80 | .then(container => container.remove());
81 |
82 | process.on('SIGTERM', gracefulShutdown);
83 | process.on('SIGINT', gracefulShutdown);
84 |
85 | afterAll(done => gracefulShutdown().then(done));
86 |
--------------------------------------------------------------------------------
/test/integration/observables/consumer.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const kafkaClient = require('../../../lib/client');
5 | const consumerObservable = require('../../../lib/observables/consumer');
6 |
7 | /**
8 | * brokerRedirection is required because no-kafka client will try to use
9 | * container given hostname and not the ip specified in brokers
10 | */
11 | const opts = {
12 | groupId: 'test',
13 | brokers: '127.0.0.1:9092',
14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' }
15 | };
16 |
17 | const producer = kafkaClient.producer(opts);
18 |
19 | describe('consumer', () => {
20 | let subscription;
21 |
22 | describe('without autoCommit', () => {
23 | let options;
24 |
25 | beforeEach(()=> {
26 | options = Object.assign({ autoCommit: false }, opts);
27 | });
28 |
29 | it('should get a message as an observable', done => {
30 | const observable = consumerObservable.create('test_kafka', options);
31 | subscription = observable
32 | .subscribe(
33 | ({ message, commit }) => {
34 | const msg = message.value.toString('utf8');
35 | expect(msg).toEqual('my message');
36 | commit().subscribe(() => done());
37 | },
38 | err => done.fail(err)
39 | );
40 |
41 | delay(4000)
42 | .then(() => producer.publish('test_kafka', 'my message'));
43 | });
44 |
45 | it('should be able to map messages', done => {
46 | const observable = consumerObservable.create('test_kafka', options);
47 | subscription = observable
48 | .map( ({ message, commit }) => ({
49 | commit,
50 | message: message.value.toString('utf8'),
51 | mapped: true
52 | }) )
53 | .subscribe(
54 | ({ message, commit, mapped }) => {
55 | expect(mapped).toBe(true);
56 | expect(message).toEqual('my message');
57 | commit().subscribe(() => done());
58 | },
59 | err => done.fail(err)
60 | );
61 |
62 | delay(5000)
63 | .then(() => producer.publish('test_kafka', 'my message'));
64 | });
65 |
66 | it('should be able to reduce messages', done => {
67 | const observable = consumerObservable.create('test_kafka', options);
68 | subscription = observable
69 | .do(m => m.commit().subscribe())
70 | .takeUntil(Observable.interval(6000))
71 | .reduce((acc, m) => acc + 1, 0)
72 | .subscribe(number => {
73 | expect(number).toBe(3);
74 | done();
75 | });
76 |
77 | delay(5000)
78 | .then(() => producer.publish('test_kafka', 'first message'))
79 | .then(() => producer.publish('test_kafka', 'second message'))
80 | .then(() => producer.publish('test_kafka', 'third message'));
81 | });
82 |
83 | it('should filter messages', done => {
84 | const observable = consumerObservable.create('test_kafka', options);
85 | subscription = observable
86 | /**
87 | * here are have to concatMap through the commit because ".take"
88 | * will immediately call unsubscribe and close the connection,
89 | * so the last commit will fail if done after ".take"
90 | */
91 | .concatMap(m => m.commit().map(() => m))
92 | .take(4)
93 | .map(({message}) => message.value.toString('utf8'))
94 | .filter(message => message.startsWith('a'))
95 | .reduce((acc, message) => acc + 1, 0)
96 | .subscribe(number => {
97 | expect(number).toBe(2);
98 | done();
99 | });
100 |
101 | delay(5000)
102 | .then(() => producer.publish('test_kafka', 'banana'))
103 | .then(() => producer.publish('test_kafka', 'avocado'))
104 | .then(() => delay(500))
105 | .then(() => producer.publish('test_kafka', 'coconut'))
106 | .then(() => producer.publish('test_kafka', 'apple'));
107 | });
108 | });
109 |
110 | describe('with autoCommit', () => {
111 | const getOffset = observable =>
112 | observable
113 | ._adapter
114 | .client()
115 | .offset('test_kafka');
116 |
117 | it('should get a message as an observable', done => {
118 | const observable = consumerObservable.create('test_kafka', opts);
119 | subscription = observable
120 | .subscribe(
121 | ({ message }) => {
122 | const msg = message.value.toString('utf8');
123 | expect(msg).toEqual('my message');
124 | done();
125 | },
126 | err => done.fail(err)
127 | );
128 |
129 | delay(5000)
130 | .then(() => producer.publish('test_kafka', 'my message'));
131 | });
132 |
133 | it('should commit the offset', done => {
134 | const observable = consumerObservable.create('test_kafka', opts);
135 | subscription = observable
136 | .take(1)
137 | .map(({ message }) => message.value.toString('utf8'))
138 | .subscribe(
139 | msg => expect(msg).toEqual('my message'),
140 | err => done.fail(err)
141 | );
142 |
143 | getOffset(observable)
144 | .then(startOffset => {
145 | subscription.add(() => {
146 | getOffset(observable)
147 | .then(offset => expect(offset).toBe(startOffset + 1))
148 | .then(done);
149 | });
150 | });
151 |
152 | delay(5000)
153 | .then(() => producer.publish('test_kafka', 'my message'));
154 | });
155 |
156 | it('should filter messages', done => {
157 | const observable = consumerObservable.create('test_kafka', opts);
158 | subscription = observable
159 | .take(4)
160 | .map(({message}) => message.value.toString('utf8'))
161 | .filter(message => message.startsWith('a'))
162 | .reduce((acc, message) => acc + 1, 0)
163 | .subscribe(number => {
164 | expect(number).toBe(2);
165 | done();
166 | });
167 |
168 | delay(5000)
169 | .then(() => producer.publish('test_kafka', 'banana'))
170 | .then(() => producer.publish('test_kafka', 'avocado'))
171 | .then(() => producer.publish('test_kafka', 'coconut'))
172 | .then(() => producer.publish('test_kafka', 'apple'));
173 | });
174 |
175 | it('should not commit message if observer throws', done => {
176 | const observable = consumerObservable.create('test_kafka', opts);
177 | const adapter = observable._adapter;
178 | spyOn(adapter, 'commit').and.callThrough();
179 |
180 | subscription = observable
181 | .take(1)
182 | .map(({ message }) => message.value.toString('utf8'))
183 | .catch(err => done.fail(err))
184 | .subscribe({
185 | next: () => { throw "Ops!" },
186 | error: err => done.fail(err)
187 | });
188 |
189 | delay(5000)
190 | .then(() => producer.publish('test_kafka', 'my message'))
191 | .then(() => subscription.add(() => {
192 | expect(adapter.commit).not.toHaveBeenCalled();
193 | done();
194 | }));
195 | });
196 |
197 | });
198 |
199 | afterEach(() => {
200 | subscription.unsubscribe();
201 | });
202 |
203 | });
--------------------------------------------------------------------------------
/test/integration/observables/producer.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const consumerObservable = require('../../../lib/observables/consumer');
5 | const producerObservable = require('../../../lib/observables/producer');
6 |
7 | /**
8 | * brokerRedirection is required because no-kafka client will try to use
9 | * container given hostname and not the ip specified in brokers
10 | */
11 | const opts = {
12 | groupId: 'test',
13 | brokers: '127.0.0.1:9092',
14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' }
15 | };
16 |
17 | describe('producer', () => {
18 | let consumerSubscription, producerSubscription;
19 |
20 | it('should send a single message', done => {
21 | const msg = 'producer1';
22 | const consumer = consumerObservable.create('test_kafka', opts);
23 | const producer = producerObservable.create('test_kafka', msg, opts);
24 |
25 | consumerSubscription = consumer
26 | .takeUntil(Observable.interval(6000))
27 | .subscribe(() => {}, err => done.fail(err), () => done());
28 |
29 | delay(5000)
30 | .then(() => {
31 | producerSubscription = producer
32 | .subscribe(
33 | ({topic, error}) => {
34 | expect(topic).toEqual('test_kafka');
35 | expect(error).toBeNull();
36 | },
37 | err => done.fail(err)
38 | );
39 | });
40 | });
41 |
42 | it('should send many messages', done => {
43 | const msgs = ['first', 'second'];
44 | const consumer = consumerObservable.create('test_kafka', opts);
45 | const producer = producerObservable.create('test_kafka', msgs, opts);
46 |
47 | consumerSubscription = consumer
48 | .take(2)
49 | .subscribe(() => {}, err => done.fail(err), () => done());
50 |
51 | delay(5000)
52 | .then(() => {
53 | producerSubscription = producer
54 | .subscribe(
55 | ({topic, error}) => {
56 | expect(topic).toEqual('test_kafka');
57 | expect(error).toBeNull();
58 | },
59 | err => done.fail(err)
60 | );
61 | });
62 | });
63 |
64 | it('should send messages from an observable', done => {
65 | const msgs = Observable.from(['first', 'second', 'third']);
66 | const consumer = consumerObservable.create('test_kafka', opts);
67 | const producer = producerObservable.create('test_kafka', msgs, opts);
68 |
69 | consumerSubscription = consumer
70 | .take(3)
71 | .subscribe(() => {}, err => done.fail(err), () => done());
72 |
73 | delay(5000)
74 | .then(() => {
75 | producerSubscription = producer
76 | .subscribe(
77 | ({topic, error}) => {
78 | expect(topic).toEqual('test_kafka');
79 | expect(error).toBeNull();
80 | },
81 | err => done.fail(err)
82 | );
83 | });
84 | });
85 |
86 | afterEach(() => {
87 | consumerSubscription.unsubscribe();
88 | producerSubscription.unsubscribe();
89 | });
90 |
91 | });
92 |
--------------------------------------------------------------------------------
/test/integration/operators/json-message.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const kafkaClient = require('../../../lib/client');
5 | const KafkaObservable = require('../../../index');
6 |
7 | /**
8 | * brokerRedirection is required because no-kafka client will try to use
9 | * container given hostname and not the ip specified in brokers
10 | */
11 | const opts = {
12 | groupId: 'test',
13 | brokers: '127.0.0.1:9092',
14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' }
15 | };
16 |
17 | const producer = kafkaClient.producer(opts);
18 |
19 | describe('JSONMessage', () => {
20 | let subscription;
21 |
22 | it('should get a message value as JSON', done => {
23 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
24 | subscription = observable
25 | .take(1)
26 | .let(KafkaObservable.JSONMessage())
27 | .subscribe(
28 | json => {
29 | expect(json.key).toBeDefined();
30 | expect(json.key).toEqual('test-value');
31 | },
32 | err => done.fail(err),
33 | () => done()
34 | );
35 |
36 | delay(5000)
37 | .then(() => producer.publish('test_kafka', {key: 'test-value'}));
38 | });
39 |
40 | it('should be available when called in the instance', done => {
41 | const kafka = KafkaObservable(opts);
42 | const observable = kafka.fromTopic('test_kafka');
43 | subscription = observable
44 | .take(1)
45 | .let(kafka.JSONMessage())
46 | .subscribe(
47 | json => {
48 | expect(json.key).toBeDefined();
49 | expect(json.key).toEqual('test-value');
50 | },
51 | err => done.fail(err),
52 | () => done()
53 | );
54 |
55 | delay(5000)
56 | .then(() => producer.publish('test_kafka', {key: 'test-value'}));
57 | });
58 |
59 | afterEach(() => {
60 | subscription.unsubscribe();
61 | });
62 |
63 | });
--------------------------------------------------------------------------------
/test/integration/operators/kafka-message.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const kafkaClient = require('../../../lib/client');
5 | const KafkaObservable = require('../../../index');
6 |
7 | /**
8 | * brokerRedirection is required because no-kafka client will try to use
9 | * container given hostname and not the ip specified in brokers
10 | */
11 | const opts = {
12 | groupId: 'test',
13 | brokers: '127.0.0.1:9092',
14 | brokerRedirection: { 'kafka:9092' : 'localhost:9092' }
15 | };
16 |
17 | const producer = kafkaClient.producer(opts);
18 |
19 | describe('TextMessage', () => {
20 | let subscription;
21 |
22 | it('should get a message value as text', done => {
23 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
24 | subscription = observable
25 | .take(1)
26 | .let(KafkaObservable.TextMessage())
27 | .subscribe(
28 | msg => expect(msg).toEqual('my message'),
29 | err => done.fail(err),
30 | () => done()
31 | );
32 |
33 | delay(4000)
34 | .then(() => producer.publish('test_kafka', 'my message'));
35 | });
36 |
37 | afterEach(() => {
38 | subscription.unsubscribe();
39 | });
40 |
41 | });
--------------------------------------------------------------------------------
/test/support/integration.json:
--------------------------------------------------------------------------------
1 | {
2 | "spec_dir": "test/integration",
3 | "spec_files": [
4 | "**/*.spec.js"
5 | ],
6 | "helpers": [
7 | "helpers/**/*.js"
8 | ],
9 | "stopSpecOnExpectationFailure": true,
10 | "random": false
11 | }
12 |
--------------------------------------------------------------------------------
/test/support/unit.json:
--------------------------------------------------------------------------------
1 | {
2 | "spec_dir": "test/unit",
3 | "spec_files": [
4 | "**/*.spec.js"
5 | ],
6 | "helpers": [
7 | "mock/**/*.js",
8 | "helpers/**/*.js"
9 | ],
10 | "stopSpecOnExpectationFailure": true,
11 | "random": false
12 | }
13 |
--------------------------------------------------------------------------------
/test/unit/consumer.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 |
5 | const consumerObservable = require('../../lib/observables/consumer');
6 | const opts = {
7 | groupId: 'test',
8 | brokers: '127.0.0.1:9092'
9 | };
10 |
11 | describe('consumer', () => {
12 | let subscription, observable, producer;
13 | beforeEach(() => {
14 | observable = consumerObservable.create('test_kafka', opts);
15 | producer = mockProducer();
16 | });
17 |
18 | it('should get a message as an observable', done => {
19 | subscription = observable
20 | .subscribe(
21 | ({ message }) => {
22 | const msg = message.value.toString('utf8');
23 | expect(msg).toEqual('my message');
24 | done();
25 | },
26 | err => done.fail(err)
27 | );
28 |
29 | delay(100)
30 | .then(() => producer.publish('test_kafka', 'my message'));
31 | });
32 |
33 | it('should automatically commit message', done => {
34 | const adapter = observable._adapter;
35 | spyOn(adapter, 'commit').and.callThrough();
36 | subscription = observable
37 | .take(1)
38 | .subscribe();
39 |
40 | delay(100)
41 | .then(() => producer.publish('test_kafka', 'my message'))
42 | .then(() => subscription.add(() => {
43 | expect(adapter.commit).toHaveBeenCalled();
44 | done();
45 | }));
46 | });
47 |
48 | it('should be able to map messages', done => {
49 | subscription = observable
50 | .map( ({ message }) => ({
51 | message: message.value.toString('utf8'),
52 | mapped: true
53 | }) )
54 | .subscribe(
55 | ({ message, mapped }) => {
56 | expect(mapped).toBe(true);
57 | expect(message).toEqual('my message');
58 | done()
59 | },
60 | err => done.fail(err)
61 | );
62 |
63 | delay(100)
64 | .then(() => producer.publish('test_kafka', 'my message'));
65 | });
66 |
67 | it('should be able to reduce messages', done => {
68 | subscription = observable
69 | .takeUntil(Observable.interval(500))
70 | .reduce((acc, m) => acc + 1, 0)
71 | .subscribe(number => {
72 | expect(number).toBe(3);
73 | done();
74 | });
75 |
76 | delay(100)
77 | .then(() => producer.publish('test_kafka', 'first message'))
78 | .then(() => producer.publish('test_kafka', 'second message'))
79 | .then(() => producer.publish('test_kafka', 'third message'));
80 | });
81 |
82 | it('should filter messages', done => {
83 | subscription = observable
84 | .take(4)
85 | .map(({message}) => message.value.toString('utf8'))
86 | .filter(message => message.startsWith('a'))
87 | .reduce((acc, message) => acc + 1, 0)
88 | .subscribe(number => {
89 | expect(number).toBe(2);
90 | done();
91 | });
92 |
93 | delay(100)
94 | .then(() => producer.publish('test_kafka', 'abacate'))
95 | .then(() => producer.publish('test_kafka', 'banana'))
96 | .then(() => producer.publish('test_kafka', 'coconut'))
97 | .then(() => producer.publish('test_kafka', 'apple'));
98 | });
99 |
100 | it('should catch if an error is thrown before subscribe', done => {
101 | subscription = observable
102 | .map(({message}) => {
103 | const msg = message.value.toString('utf8');
104 | if (msg ==='throw') { throw 'Ops!' }
105 | return msg;
106 | })
107 | .catch(err => Observable.of('pass'))
108 | .subscribe(message => message !== 'throw' ? done() : done.fail());
109 |
110 | delay(100)
111 | .then(() => producer.publish('test_kafka', 'throw'));
112 | });
113 |
114 | it('should log error observer throw', done => {
115 | spyOn(bunyan, 'error');
116 |
117 | subscription = observable
118 | .subscribe(() => { throw 'Ops!'; });
119 |
120 | delay(100)
121 | .then(() => producer.publish('test_kafka', 'throw'))
122 | .then(() => subscription.add(()=> {
123 | expect(bunyan.error).toHaveBeenCalled();
124 | done();
125 | }));
126 | });
127 |
128 | it('observable has a non standard _adapter attached to it', () => {
129 | expect(observable._adapter).toBeDefined();
130 | });
131 |
132 | it('should use DefaultAssignmentStrategy if strategy is not found', done => {
133 | const options = Object.assign({strategy: 'ops'}, opts);
134 | observable = consumerObservable.create('test_kafka', options);
135 | subscription = observable
136 | .subscribe(() => {
137 | const client = observable._adapter.client();
138 | const strategy = client.strategies[0].strategy.constructor.name;
139 | expect(strategy).toEqual('DefaultAssignmentStrategy');
140 | done();
141 | });
142 |
143 | delay(100)
144 | .then(() => producer.publish('test_kafka', 'message'));
145 | });
146 |
147 | it('should error if strategy is invalid', done => {
148 | const options = Object.assign({strategy: 'ops'}, opts);
149 | observable = consumerObservable.create('test_kafka', options);
150 | subscription = observable
151 | .subscribe(() => {
152 | const client = observable._adapter.client();
153 | const strategy = client.strategies[0].strategy.constructor.name;
154 | expect(strategy).toEqual('DefaultAssignmentStrategy');
155 | done();
156 | });
157 |
158 | delay(100)
159 | .then(() => producer.publish('test_kafka', 'message'));
160 | });
161 |
162 | it('should error if groupId is not defined', done => {
163 | const options = Object.assign({}, opts, { groupId: undefined });
164 | observable = consumerObservable.create('test_kafka', options);
165 | subscription = observable
166 | .subscribe({ next: () => done.fail(), error: err => done() });
167 | });
168 |
169 | it('should error if brokers is not defined', done => {
170 | const options = Object.assign({}, opts, { brokers: undefined });
171 | observable = consumerObservable.create('test_kafka', options);
172 | subscription = observable
173 | .subscribe({ next: () => done.fail(), error: err => done() });
174 | });
175 |
176 | it('should error if brokers is not a string or an array', done => {
177 | const options = Object.assign({}, opts, { brokers: {} });
178 | observable = consumerObservable.create('test_kafka', options);
179 | subscription = observable
180 | .subscribe({ next: () => done.fail(), error: err => done() });
181 | });
182 |
183 | it('should error if initial client subscription fails', done => {
184 | const options = Object.assign({metadata: 1}, opts);
185 | observable = consumerObservable.create('test_kafka', options);
186 | subscription = observable
187 | .subscribe({ next: () => done.fail(), error: err => done() });
188 | });
189 |
190 | it('should work for null message', done => {
191 | observable = consumerObservable.create('test_kafka', opts);
192 | subscription = observable
193 | .subscribe(({message}) => {
194 | expect(message.value).toBeNull();
195 | done();
196 | });
197 |
198 | delay(100)
199 | .then(() => producer.publish('test_kafka', null));
200 | });
201 |
202 | describe('without auto-commit', () => {
203 | let options;
204 |
205 | beforeEach(() => {
206 | options = Object.assign({ autoCommit: false }, opts) ;
207 | observable = consumerObservable.create('test_kafka', options);
208 | });
209 |
210 | it('should have a commit function', done => {
211 | subscription = observable
212 | .subscribe(
213 | ({ commit }) => {
214 | expect(commit).toBeDefined();
215 | expect(typeof commit).toEqual('function');
216 | expect(commit() instanceof Observable).toBe(true);
217 | done();
218 | },
219 | err => done.fail(err)
220 | );
221 |
222 | delay(100)
223 | .then(() => producer.publish('test_kafka', 'my message'));
224 | });
225 |
226 | it('should not commit message', done => {
227 | const adapter = observable._adapter;
228 | spyOn(adapter, 'commit').and.callThrough();
229 | subscription = observable
230 | .take(1)
231 | .subscribe();
232 |
233 | delay(100)
234 | .then(() => producer.publish('test_kafka', 'my message'))
235 | .then(() => subscription.add(() => {
236 | expect(adapter.commit).not.toHaveBeenCalled();
237 | done();
238 | }));
239 | });
240 | });
241 |
242 |
243 | afterEach(() => {
244 | subscription.unsubscribe();
245 | });
246 |
247 | });
--------------------------------------------------------------------------------
/test/unit/helpers/helper.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | /**
4 | * end handler for jasmine
5 | * @param {Function} done jasmine done function
6 | * @return {Function} handler that calls done the expected way
7 | **/
8 | global.finish_test = function finish_test(done) {
9 | return function (err) {
10 | if (err) { done.fail(err); }
11 | done();
12 | };
13 | };
14 |
15 | /**
16 | * delay func
17 | * @param {Number} millis time in milliseconds
18 | * @return {Promise} resolves after given time
19 | */
20 | global.delay = millis =>
21 | new Promise((resolve) => setTimeout(()=> resolve(), millis));
22 |
--------------------------------------------------------------------------------
/test/unit/kafka-observable.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const KafkaObservable = require('../../index');
5 |
6 | const opts = {
7 | groupId: 'test',
8 | brokers: '127.0.0.1:9092'
9 | };
10 |
11 | describe('fromTopic', () => {
12 | let subscription;
13 | beforeEach(() => {
14 |
15 | });
16 |
17 | it('should inherit options', done => {
18 | const producer = mockProducer();
19 | const consumer = KafkaObservable(opts).fromTopic('test_kafka');
20 |
21 | subscription = consumer
22 | .take(1)
23 | .map(({message}) => message.value.toString('utf8'))
24 | .subscribe(
25 | message => expect(message).toEqual('my message'),
26 | err => done.fail(err),
27 | () => done()
28 | );
29 |
30 | delay(100)
31 | .then(() => producer.publish('test_kafka', 'my message'));
32 | });
33 |
34 | it('should accept options', done => {
35 | const producer = mockProducer();
36 | const consumer = KafkaObservable.fromTopic('test_kafka', opts);
37 |
38 | subscription = consumer
39 | .take(1)
40 | .map(({message}) => message.value.toString('utf8'))
41 | .subscribe(
42 | message => expect(message).toEqual('my message'),
43 | err => done.fail(err),
44 | () => done()
45 | );
46 |
47 | delay(100)
48 | .then(() => producer.publish('test_kafka', 'my message'));
49 | });
50 |
51 | afterEach(() => {
52 | subscription.unsubscribe();
53 | });
54 |
55 | });
56 |
57 | describe('toTopic', () => {
58 | let consumerSubscription, producerSubscription;
59 |
60 | it('should send a message', done => {
61 | const msg = 'message';
62 | const observable = KafkaObservable(opts);
63 | const consumer = observable.fromTopic('test_kafka');
64 | const producer = observable.toTopic('test_kafka', msg);
65 |
66 | consumerSubscription = consumer
67 | .take(1)
68 | .subscribe(() => {}, err => done.fail(err), () => done());
69 |
70 | delay(100)
71 | .then(() => {
72 | producerSubscription = producer
73 | .subscribe(
74 | ({topic, error}) => {
75 | expect(topic).toEqual('test_kafka');
76 | expect(error).toBeNull();
77 | },
78 | err => done.fail(err)
79 | );
80 | });
81 | });
82 |
83 | afterEach(() => {
84 | consumerSubscription.unsubscribe();
85 | producerSubscription.unsubscribe();
86 | });
87 | });
--------------------------------------------------------------------------------
/test/unit/mock/mock-bunyan.js:
--------------------------------------------------------------------------------
1 | /**
2 | * mocks no-kafka module
3 | * @author ghermeto
4 | * @version 0.0.1
5 | **/
6 | const mock = require('mock-require');
7 |
8 | const bunyan = {
9 | log: () => {},
10 | trace: () => {},
11 | debug: () => {},
12 | info: () => {},
13 | warn: () => {},
14 | error: () => {}
15 | };
16 |
17 | mock('bunyan', { createLogger: () => bunyan });
18 | mock.reRequire('bunyan');
19 |
20 | module.exports = { bunyan };
21 | beforeAll(() => {
22 | global.bunyan = bunyan;
23 | });
24 |
--------------------------------------------------------------------------------
/test/unit/mock/mock-no-kafka.js:
--------------------------------------------------------------------------------
1 | /**
2 | * mocks no-kafka module
3 | * @author ghermeto
4 | * @version 0.0.1
5 | **/
6 | const mock = require('mock-require');
7 | const shortid = require('shortid');
8 |
9 | const mockKafka = {
10 | offset: 0,
11 | partition: 0,
12 | registry: {}
13 | };
14 |
15 | class GroupConsumer {
16 | constructor(options) {
17 | this.options = options;
18 | }
19 |
20 | init(strategies) {
21 | const metadata = strategies[0].metadata;
22 | if (metadata && metadata !== 'object') {
23 | return Promise.reject();
24 | }
25 |
26 | this.strategies = strategies;
27 | strategies.forEach(strategy => {
28 |
29 | mockKafka.registry = strategy.subscriptions
30 | .reduce((registry, topic) => {
31 | const current = registry[topic] || [];
32 | const updatedList = current.concat(strategy.handler);
33 | return Object.assign({}, registry, {[topic]: updatedList});
34 | }, mockKafka.registry);
35 | });
36 | return Promise.resolve(mockKafka.offset);
37 | }
38 |
39 | commitOffset() {
40 | return Promise.resolve(mockKafka.offset++);
41 | }
42 |
43 | unsubscribe(topic) {
44 | delete mockKafka.registry[topic];
45 | return Promise.resolve();
46 | }
47 |
48 | end() {
49 | this.strategies = null;
50 | mockKafka.registry = {};
51 | }
52 | }
53 |
54 | class Producer {
55 | constructor(options) {
56 | this.options = options;
57 | options.logger.logFunction('trace', ['mock']);
58 | }
59 |
60 | init() {
61 | return Promise.resolve();
62 | }
63 |
64 | end() {
65 | return Promise.resolve();
66 | }
67 |
68 | send({ topic, message }) {
69 | const error = this.options.metadata ? new Error('test') : null;
70 | mockProducer().publish(topic, message.value, message.key);
71 | return Promise.resolve([{ topic, error }]);
72 | }
73 | }
74 |
75 | const mockProducer = () => {
76 | return {
77 | publish(topic, message, key = shortid.generate()) {
78 | const handlerList = mockKafka.registry[topic];
79 | if (!mockKafka.registry[topic]) {
80 | throw 'No consumer in such topic';
81 | }
82 |
83 | const msg = this.createMessage(message, key);
84 |
85 | handlerList.forEach(handler => {
86 | handler([msg], topic, mockKafka.partition);
87 | });
88 | },
89 |
90 | createMessage(message, key) {
91 | return {
92 | offset: mockKafka.offset,
93 | messageSize: message ? message.length : 0,
94 | message: {
95 | key: key ? Buffer.from(key, 'utf8') : null,
96 | value: message
97 | ? typeof message === 'string'
98 | ? Buffer.from(message, 'utf8')
99 | : message
100 | : null
101 | }
102 | }
103 | }
104 |
105 | }
106 | };
107 |
108 | const mockKafkaClient = {
109 | DefaultAssignmentStrategy: class DefaultAssignmentStrategy { },
110 | DefaultPartitioner: class DefaultPartitioner { },
111 | GroupConsumer: GroupConsumer,
112 | Producer: Producer
113 | };
114 |
115 | mock('no-kafka', mockKafkaClient);
116 | mock.reRequire('no-kafka');
117 |
118 | module.exports = { mockKafkaClient, mockProducer };
119 | beforeAll(() => {
120 | global.kafkaClient = mockKafkaClient;
121 | global.mockProducer = mockProducer;
122 | });
123 |
--------------------------------------------------------------------------------
/test/unit/operators.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const kafkaClient = require('../../lib/client');
5 | const KafkaObservable = require('../../index');
6 |
7 | const opts = {
8 | groupId: 'test',
9 | brokers: '127.0.0.1:9092'
10 | };
11 |
12 | const producer = kafkaClient.producer(opts);
13 |
14 | describe('JSONMessage operator', () => {
15 | let subscription;
16 |
17 | it('should get a message value as JSON', done => {
18 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
19 | subscription = observable
20 | .take(1)
21 | .let(KafkaObservable.JSONMessage())
22 | .subscribe(
23 | json => {
24 | expect(json.key).toBeDefined();
25 | expect(json.key).toEqual('test-value');
26 | },
27 | err => done.fail(err),
28 | () => done()
29 | );
30 |
31 | delay(100)
32 | .then(() => producer.publish('test_kafka', {key: 'test-value'}));
33 | });
34 |
35 | it('should error if not JSON', done => {
36 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
37 | subscription = observable
38 | .take(1)
39 | .let(KafkaObservable.JSONMessage())
40 | .subscribe(() => done.fail(), err => done());
41 |
42 | delay(100)
43 | .then(() => producer.publish('test_kafka', 'test'));
44 | });
45 |
46 | it('should error if mapper throws', done => {
47 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
48 | subscription = observable
49 | .take(1)
50 | .let(KafkaObservable.JSONMessage(x => { throw 'Ops!'; }))
51 | .subscribe(() => done.fail(), err => done());
52 |
53 | delay(100)
54 | .then(() => producer.publish('test_kafka', {key: 'test-value'}));
55 | });
56 |
57 | it('should be available when called in the instance', done => {
58 | const kafka = KafkaObservable(opts);
59 | const observable = kafka.fromTopic('test_kafka');
60 | subscription = observable
61 | .take(1)
62 | .let(kafka.JSONMessage())
63 | .subscribe(
64 | json => {
65 | expect(json.key).toBeDefined();
66 | expect(json.key).toEqual('test-value');
67 | },
68 | err => done.fail(err),
69 | () => done()
70 | );
71 |
72 | delay(100)
73 | .then(() => producer.publish('test_kafka', {key: 'test-value'}));
74 | });
75 |
76 | afterEach(() => {
77 | subscription.unsubscribe();
78 | });
79 |
80 | });
81 |
82 | describe('TextMessage operator', () => {
83 | let subscription;
84 |
85 | it('should get a message value as text', done => {
86 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
87 | subscription = observable
88 | .take(1)
89 | .let(KafkaObservable.TextMessage())
90 | .subscribe(
91 | msg => expect(msg).toEqual('my message'),
92 | err => done.fail(err),
93 | () => done()
94 | );
95 |
96 | delay(100)
97 | .then(() => producer.publish('test_kafka', 'my message'));
98 | });
99 |
100 | it('should error if mapper throws', done => {
101 | const observable = KafkaObservable.fromTopic('test_kafka', opts);
102 | subscription = observable
103 | .take(1)
104 | .let(KafkaObservable.TextMessage(x => { throw 'Ops!'; }))
105 | .subscribe(() => done.fail(), err => done());
106 |
107 | delay(100)
108 | .then(() => producer.publish('test_kafka', 'my message'));
109 | });
110 |
111 | afterEach(() => {
112 | subscription.unsubscribe();
113 | });
114 |
115 | });
--------------------------------------------------------------------------------
/test/unit/producer.spec.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const Observable = require('rxjs').Observable;
4 | const consumerObservable = require('../../lib/observables/consumer');
5 | const producerObservable = require('../../lib/observables/producer');
6 |
7 | const opts = {
8 | groupId: 'test',
9 | brokers: '127.0.0.1:9092'
10 | };
11 |
12 | describe('producer', () => {
13 | let consumerSubscription, producerSubscription;
14 |
15 | it('should send a single message', done => {
16 | const msg = 'message';
17 | const consumer = consumerObservable.create('test_kafka', opts);
18 | const producer = producerObservable.create('test_kafka', msg, opts);
19 |
20 | consumerSubscription = consumer
21 | .take(1)
22 | .subscribe(() => {}, err => done.fail(err), () => done());
23 |
24 | delay(100)
25 | .then(() => {
26 | producerSubscription = producer
27 | .subscribe(
28 | ({topic, error}) => {
29 | expect(topic).toEqual('test_kafka');
30 | expect(error).toBeNull();
31 | },
32 | err => done.fail(err)
33 | );
34 | });
35 | });
36 |
37 | it('should send many messages', done => {
38 | const msgs = ['first', 'second'];
39 | const consumer = consumerObservable.create('test_kafka', opts);
40 | const producer = producerObservable.create('test_kafka', msgs, opts);
41 |
42 | consumerSubscription = consumer
43 | .take(2)
44 | .subscribe(() => {}, err => done.fail(err), () => done());
45 |
46 | delay(100)
47 | .then(() => {
48 | producerSubscription = producer
49 | .subscribe(
50 | ({topic, error}) => {
51 | expect(topic).toEqual('test_kafka');
52 | expect(error).toBeNull();
53 | },
54 | err => done.fail(err)
55 | );
56 | });
57 | });
58 |
59 | it('should send messages from an observable', done => {
60 | const msgs = Observable.from(['first', 'second', 'third']);
61 | const consumer = consumerObservable.create('test_kafka', opts);
62 | const producer = producerObservable.create('test_kafka', msgs, opts);
63 |
64 | consumerSubscription = consumer
65 | .take(3)
66 | .subscribe(() => {}, err => done.fail(err), () => done());
67 |
68 | delay(100)
69 | .then(() => {
70 | producerSubscription = producer
71 | .subscribe(
72 | ({topic, error}) => {
73 | expect(topic).toEqual('test_kafka');
74 | expect(error).toBeNull();
75 | },
76 | err => done.fail(err)
77 | );
78 | });
79 | });
80 |
81 | it('should error if brokers is not defined', done => {
82 | const msgs = ['first', 'second'];
83 | const options = Object.assign({}, opts, { brokers: undefined });
84 | const producer = producerObservable.create('test_kafka', msgs, options);
85 | producerSubscription = producer
86 | .subscribe({ next: () => done.fail(), error: err => done() });
87 | });
88 |
89 | it('should error if brokers is not a string or an array', done => {
90 | const msgs = ['first', 'second'];
91 | const options = Object.assign({}, opts, { brokers: {} });
92 | const producer = producerObservable.create('test_kafka', msgs, options);
93 | producerSubscription = producer
94 | .subscribe({ next: () => done.fail(), error: err => done() });
95 | });
96 |
97 | it('should propagate error if observable gives an error', done => {
98 | const msgs = Observable.throw('test');
99 | const producer = producerObservable.create('test_kafka', msgs, opts);
100 | producerSubscription = producer
101 | .subscribe({
102 | next: () => done.fail(),
103 | error: err => done(),
104 | complete: () => done()
105 | });
106 | });
107 |
108 | it('should default to DefaultPartitioner', done => {
109 | const msgs = ['first', 'second'];
110 | const options = Object.assign({ partitioner: 'NotFound'}, opts);
111 | const consumer = consumerObservable.create('test_kafka', opts);
112 | const producer = producerObservable.create('test_kafka', msgs, options);
113 | consumerSubscription = consumer
114 | .take(2)
115 | .subscribe(() => {}, err => done.fail(err), () => done());
116 |
117 | delay(100)
118 | .then(() => {
119 | producerSubscription = producer
120 | .subscribe(() => {
121 | const client = producer._adapter.client();
122 | const partitioner = client.options.partitioner;
123 |
124 | expect(partitioner.constructor.name)
125 | .toEqual('DefaultPartitioner');
126 |
127 | done();
128 | },
129 | err => done.fail(err)
130 | );
131 | });
132 |
133 | });
134 |
135 | it('should serialize an object as message', done => {
136 | const msgs = [{ object: true }];
137 | const consumer = consumerObservable.create('test_kafka', opts);
138 | const producer = producerObservable.create('test_kafka', msgs, opts);
139 | consumerSubscription = consumer
140 | .take(1)
141 | .map(({message}) => message.value.toString('utf8'))
142 | .map(message => JSON.parse(message))
143 | .subscribe(
144 | json => expect(json.object).toBe(true),
145 | err => done.fail(err),
146 | () => done()
147 | );
148 |
149 | delay(100)
150 | .then(() => {
151 | producerSubscription = producer
152 | .subscribe(() => {}, err => done.fail(err));
153 | });
154 |
155 | });
156 |
157 | afterEach(() => {
158 | consumerSubscription.unsubscribe();
159 | producerSubscription.unsubscribe();
160 | });
161 |
162 | });
163 |
--------------------------------------------------------------------------------