├── post.txt ├── .gitignore ├── Procfile ├── .env ├── start-kafka-shell.sh ├── package.json ├── fig.yml ├── app.js ├── worker.js └── README.md /post.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node ./app.js 2 | worker: node ./worker.js 3 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | MONGO_URL=192.168.59.103:27017/kafka 2 | KAFKA_URL=192.168.59.103:2181 3 | PORT=3001 4 | -------------------------------------------------------------------------------- /start-kafka-shell.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka:0.8.2.0 /bin/bash 3 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "express-kafka-db", 3 | "version": "0.0.0", 4 | "description": "", 5 | "main": "app.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "kafka-node": "0.2.19", 11 | "express": "4.0.0", 12 | "moment-timezone": "0.3.0", 13 | "mongojs": "0.18.0" 14 | }, 15 | "author": "", 16 | "license": "MIT" 17 | } 18 | -------------------------------------------------------------------------------- /fig.yml: -------------------------------------------------------------------------------- 1 | zookeeper: 2 | image: wurstmeister/zookeeper 3 | ports: 4 | - "2181:2181" 5 | kafka: 6 | image: wurstmeister/kafka:0.8.2.0 7 | ports: 8 | - "9092:9092" 9 | links: 10 | - zookeeper:zk 11 | environment: 12 | KAFKA_ADVERTISED_HOST_NAME: 192.168.59.103 13 | volumes: 14 | - /var/run/docker.sock:/var/run/docker.sock 15 | mongodb: 16 | image: mongo:2.6.7 17 | ports: 18 | - "27017:27017" 19 | - "28017:28017" 20 | command: mongod --rest 21 | environment: 22 | - MONGO_URL=mongodb://mongodb:27017 23 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var express = require('express'), 2 | moment = require('moment-timezone'), 3 | app = express(), 4 | 5 | // database 6 | mongojs = require('mongojs'), 7 | db = mongojs(process.env.MONGO_URL || 'localhost:27017/kafka'), 8 | 9 | // kafka 10 | kafka = require('kafka-node'), 11 | client = new kafka.Client(process.env.KAFKA_URL || '192.168.59.103:2181/', process.env.KAFKA_PRODUCER_ID || 'kafka-node-producer', {}), 12 | HighLevelProducer = kafka.HighLevelProducer, 13 | producer = new HighLevelProducer(client); 14 | 15 | // Express setup 16 | app.listen(process.env.PORT || 3001); 17 | 18 | app.post('/', function(req, res) { 19 | 20 | var timestamp = moment().unix(); 21 | 22 | // sends value to kafka 23 | var topicMessage = { topic: 'my-node-topic', messages: [ 24 | // all messages must be string :S 25 | JSON.stringify({ timestamp: timestamp, rnd: Math.random() }) 26 | ] }; 27 | 28 | var payload = [ topicMessage ]; 29 | 30 | producer.send(payload, function (err, data) { 31 | if (err) { 32 | res.send(500, err); 33 | } else { 34 | res.json(200, {timestamp: timestamp}); 35 | } 36 | }); 37 | 38 | }); 39 | 40 | app.get('/', function(req, res) { 41 | db.collection('kafka').find({}, function(err, data) { 42 | if(err) { 43 | res.send(500, err); 44 | } else if (!data) { 45 | res.send(404, 'nothing found...'); 46 | } else { 47 | res.json(200, data); 48 | } 49 | }); 50 | }) 51 | 52 | 53 | // Kafka events 54 | producer.on('ready', function () { 55 | console.log('KAFKA producer ready'); 56 | }); 57 | 58 | producer.on('error', function (err) { 59 | console.log('KAFKA producer error:' + err); 60 | }) 61 | -------------------------------------------------------------------------------- /worker.js: -------------------------------------------------------------------------------- 1 | var moment = require('moment-timezone'), 2 | 3 | // Kafka setup 4 | kafka = require('kafka-node'), 5 | client = new kafka.Client(process.env.KAFKA_URL || '192.168.59.103:2181/', process.env.KAFKA_CONSUMER_ID || 'kafka-node-consumer', { 6 | sessionTimeout: 1000 // this is just to enable multiple restarts in debug and avoid https://github.com/SOHU-Co/kafka-node/issues/90 - should be removed in PRD 7 | }), 8 | HighLevelConsumer = kafka.HighLevelConsumer, 9 | 10 | // database 11 | mongojs = require('mongojs'), 12 | db = mongojs(process.env.MONGO_URL || 'localhost:27017/kafka'); 13 | 14 | 15 | var consumer = new HighLevelConsumer( 16 | client, 17 | [ 18 | { topic: 'my-node-topic' } 19 | ], 20 | { 21 | groupId: 'worker.js' // this identifies consumer and make the offset consumption scoped to this key 22 | } 23 | ); 24 | 25 | consumer.on('ready', function() { 26 | console.log('KAFKA consumer ready'); 27 | }); 28 | 29 | consumer.on('message', function (message) { 30 | 31 | // example message: {"topic":"my-node-topic","value":"{\"timestamp\":1425599538}","offset":0,"partition":0,"key":{"type":"Buffer","data":[]}} 32 | db.collection('kafka').update( 33 | { 34 | _id: 123456 //always overrite the same doc just for illustration 35 | }, 36 | { 37 | '$set': { 38 | message: message, 39 | received_at: moment().unix() 40 | }, 41 | '$inc': { 42 | messagesCount: 1 43 | } 44 | }, 45 | { 46 | upsert: true 47 | }, function(err, data) { 48 | if(err) { 49 | console.log('MONGO error updating document: ' + err); 50 | } else { 51 | console.log('MONGO updating document OK:' + message.value); 52 | } 53 | }); 54 | }); 55 | 56 | consumer.on('error', function (err) { 57 | console.log('KAFKA consumer error:' + err); 58 | }); 59 | 60 | process.on('beforeExit', function(code) { 61 | //force offset commit 62 | consumer.close(true, function(err, data) { 63 | console.log('KAFKA consumer commit ok :)'); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Node.js + Express + Kafka + Mongodb 2 | 3 | ## Environment setup with [Docker](https://www.docker.io/) 4 | 5 | ### Node 6 | 7 | - Install [node v0.12](http://nodejs.org/download/) 8 | 9 | - Install node-foreman `$ npm install -g foreman` 10 | 11 | - run `$ npm install` 12 | 13 | 14 | ### Kafka 15 | 16 | If you are using a Mac follow the instructions [here](https://docs.docker.com/installation/mac/) to setup a docker environment. 17 | 18 | - Install [fig](http://orchardup.github.io/fig/install.html) 19 | 20 | - Start the test environment 21 | - `fig up` 22 | - Start a kafka shell 23 | - `./start-kafka-shell.sh `, example `./start-kafka-shell.sh 192.168.59.103:9092 192.168.59.103:2181` 24 | - From within the shell, create a topic 25 | - `$KAFKA_HOME/bin/kafka-topics.sh --create --topic my-node-topic --partitions 2 --zookeeper $ZK --replication-factor 1` 26 | 27 | - For more details and troubleshooting see [https://github.com/wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) 28 | 29 | 30 | ## Running 31 | 32 | - `$ nf start` 33 | 34 | caveat: between restart, wait for a similar message before starting to avoid consumer connection error described in https://github.com/SOHU-Co/kafka-node/issues/90 35 | 36 | (we set up `sessionTimeout: 1000` in the consumer ti make it less painful to test, this value shoued be increased in production to avoid zookeeper timeouts due to network glitches) 37 | 38 | ``` 39 | zookeeper_1 | 2015-02-19 11:28:58,000 [myid:] - INFO [SessionTracker:ZooKeeperServer@347] - Expiring session 0x14ba15e9c0f0024, timeout of 4000ms exceeded 40 | zookeeper_1 | 2015-02-19 11:28:58,002 [myid:] - INFO [ProcessThread(sid:0 cport:-1)::PrepRequestProcessor@494] - Processed session termination for sessionid: 0x14ba15e9c0f0024 41 | ``` 42 | 43 | ## Test Requests 44 | 45 | - to send messages: `$ curl -X POST http://localhost:3001/` 46 | 47 | - to read last received message by worker: `$ curl http://localhost:3001/` 48 | 49 | - to perform apache benchmark `$ ab -n 1000 -c 5 -T 'application/x-www-form-urlencoded' -p post.txt http://localhost:3001/` 50 | 51 | 52 | ## References 53 | 54 | - [wurstmeister/storm-kafka-0.8-plus-test](https://github.com/wurstmeister/storm-kafka-0.8-plus-test) 55 | - [SOHU-Co/kafka-node](https://github.com/SOHU-Co/kafka-node/) 56 | --------------------------------------------------------------------------------