├── .gitignore ├── README.md ├── consume.js ├── index.js ├── package-lock.json ├── package.json └── produce.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Node.js Kafka Example 2 | 3 | This is the example repository for my [blog post](https://www.sohamkamani.com/nodejs/working-with-kafka/) on working with Kafka with Node.js. 4 | 5 | To run the example: 6 | 7 | 1. Clone this repo 8 | 1. Configure the `brokers`, `clientId` and `topic` in the `producer.js` and `consumer.js` files according to where kafka is running on your system. 9 | 1. Install dependencies by running `npm install` 10 | 1. Run `node index.js` 11 | 12 | > You can see how to install and run a Kafka cluster in my [other tutorial](https://www.sohamkamani.com/blog/2017/11/22/how-to-install-and-run-kafka/) 13 | -------------------------------------------------------------------------------- /consume.js: -------------------------------------------------------------------------------- 1 | const { Kafka, logLevel } = require("kafkajs") 2 | 3 | const clientId = "my-app" 4 | const brokers = ["localhost:9092"] 5 | const topic = "message-log" 6 | 7 | const kafka = new Kafka({ 8 | clientId, 9 | brokers, 10 | // logCreator: customLogger, 11 | logLevel: logLevel.DEBUG, 12 | }) 13 | 14 | // the kafka instance and configuration variables are the same as before 15 | 16 | // create a new consumer from the kafka client, and set its group ID 17 | // the group ID helps Kafka keep track of the messages that this client 18 | // is yet to receive 19 | const consumer = kafka.consumer({ 20 | groupId: clientId, 21 | minBytes: 5, 22 | maxBytes: 1e6, 23 | // wait for at most 3 seconds before receiving new data 24 | maxWaitTimeInMs: 3000, 25 | }) 26 | 27 | const consume = async () => { 28 | // first, we wait for the client to connect and subscribe to the given topic 29 | await consumer.connect() 30 | await consumer.subscribe({ topic, fromBeginning: true }) 31 | await consumer.run({ 32 | // this function is called every time the consumer gets a new message 33 | eachMessage: ({ message }) => { 34 | // here, we just log the message to the standard output 35 | console.log(`received message: ${message.value}`) 36 | }, 37 | }) 38 | } 39 | 40 | module.exports = consume 41 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const produce = require("./produce") 2 | const consume = require("./consume") 3 | 4 | // call the `produce` function and log an error if it occurs 5 | produce().catch((err) => { 6 | console.error("error in producer: ", err) 7 | }) 8 | 9 | // start the consumer, and log any errors 10 | consume().catch((err) => { 11 | console.error("error in consumer: ", err) 12 | }) 13 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs-kafka-example", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "kafkajs": { 8 | "version": "1.15.0", 9 | "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.15.0.tgz", 10 | "integrity": "sha512-yjPyEnQCkPxAuQLIJnY5dI+xnmmgXmhuOQ1GVxClG5KTOV/rJcW1qA3UfvyEJKTp/RTSqQnUR3HJsKFvHyTpNg==" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs-kafka-example", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "kafkajs": "^1.15.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /produce.js: -------------------------------------------------------------------------------- 1 | // import the `Kafka` instance from the kafkajs library 2 | const { Kafka, logLevel } = require("kafkajs") 3 | 4 | // the client ID lets kafka know who's producing the messages 5 | const clientId = "my-app" 6 | // we can define the list of brokers in the cluster 7 | const brokers = ["localhost:9092"] 8 | // this is the topic to which we want to write messages 9 | const topic = "message-log" 10 | 11 | // initialize a new kafka client and initialize a producer from it 12 | const kafka = new Kafka({ clientId, brokers, logLevel: logLevel.DEBUG }) 13 | const producer = kafka.producer({}) 14 | 15 | // we define an async function that writes a new message each second 16 | const produce = async () => { 17 | await producer.connect() 18 | let i = 0 19 | 20 | // after the produce has connected, we start an interval timer 21 | setInterval(async () => { 22 | try { 23 | // send a message to the configured topic with 24 | // the key and value formed from the current value of `i` 25 | await producer.send({ 26 | topic, 27 | acks: 1, 28 | messages: [ 29 | { 30 | key: String(i), 31 | value: "this is message " + i, 32 | }, 33 | ], 34 | }) 35 | 36 | // if the message is written successfully, log it and increment `i` 37 | console.log("writes: ", i) 38 | i++ 39 | } catch (err) { 40 | console.error("could not write message " + err) 41 | } 42 | }, 1000) 43 | } 44 | 45 | module.exports = produce 46 | --------------------------------------------------------------------------------