├── start.sh
├── .gitignore
├── sample.env
├── config.js
├── package.json
├── createTopic.js
├── readme.md
├── app_json.js
├── consumer.js
└── producer.js


/start.sh:
--------------------------------------------------------------------------------
1 | npm start
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | .env
3 | 


--------------------------------------------------------------------------------
/sample.env:
--------------------------------------------------------------------------------
1 | KAFKA_HOST= 'localhost:9092'
2 | KAFKA_TOPIC= 'kafka-example-topic'
3 | 


--------------------------------------------------------------------------------
/config.js:
--------------------------------------------------------------------------------
 1 | require('dotenv').config();
 2 | 
 3 | const config = {
 4 |   KafkaHost:process.env.KAFKA_HOST,
 5 |   KafkaTopic: process.env.KAFKA_TOPIC
 6 | };
 7 | 
 8 | module.exports = config;
 9 | 
10 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "kafka-producer_consumer_tutorial",
 3 |   "version": "1.0.0",
 4 |   "description": "Building a real-time data streaming application pipeline with Apache Kafka",
 5 |   "main": "app.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1",
 8 |     "start": "node producer.js"
 9 |   },
10 |   "author": "Alexander Nnakwue",
11 |   "license": "MIT",
12 |   "dependencies": {
13 |     "dotenv": "^8.2.0",
14 |     "kafka-node": "^4.1.3"
15 |   }
16 | }
17 | 


--------------------------------------------------------------------------------
/createTopic.js:
--------------------------------------------------------------------------------
 1 | const kafka = require('kafka-node');
 2 | const config  = require('./config');
 3 | 
 4 | const client = new kafka.KafkaClient({kafkaHost: config.KafkaHost});
 5 | 
 6 | 
 7 | 
 8 | const topicToCreate = [{
 9 |   topic: config.KafkaTopic,
10 |   partitions: 1,
11 |   replicationFactor: 1
12 | }
13 | ];
14 | 
15 | client.createTopics(topicToCreate, (error, result) => {
16 |   // result is an array of any errors if a given topic could not be created
17 |   console.log(result, 'topic created successfully');
18 | });
19 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | # Kakfa streaming app
 2 | 
 3 | ### Producer script
 4 | 
 5 | Check the ```producer.js``` file for the client
 6 | 
 7 | 
 8 | ### Consumer script
 9 | 
10 | Check the ```consumer.js``` file
11 | 
12 | 
13 | ### create topic
14 | 
15 | Check the ```createTopic.js``` file
16 | 
17 | 
18 | To start the application(producer), run the ```start.sh``` shell script
19 | If there is a permissions issue, run ```chmod +x start.sh```
20 | 
21 | To run the consumer script, run  ```node ./consumer.js```
22 | 
23 | 
24 | 
25 | 
26 | 
27 | 
28 | 
29 | 


--------------------------------------------------------------------------------
/app_json.js:
--------------------------------------------------------------------------------
 1 | 
 2 | const jsonData = [{
 3 |   "name":"Alexander Nnakwue",
 4 |   "age": 24,
 5 |   "occupation": "Software Engineer",
 6 |   "religion": "Islam",
 7 |   "sports":"football"
 8 | },
 9 | {
10 | "name":"Alexander Nnakwue Chinedu",
11 |   "age": 25,
12 |   "occupation": "Software Tester",
13 |   "religion": "Christain",
14 |   "sports":"Soccer champ"
15 | },
16 | {
17 | "name":"Alexander Nnakwue Ebuka",
18 |   "age": 26,
19 |   "occupation": "Technical Author & Reviewer",
20 |   "religion": "Christain",
21 |   "sports":"Hand-ball"
22 | }];
23 | 
24 | module.exports = jsonData;
25 | 


--------------------------------------------------------------------------------
/consumer.js:
--------------------------------------------------------------------------------
 1 | const kafka = require('kafka-node');
 2 | const config = require('./config');
 3 | 
 4 | try {
 5 |   const Consumer = kafka.Consumer;
 6 |  const client = new kafka.KafkaClient({idleConnection: 24 * 60 * 60 * 1000,  kafkaHost: config.KafkaHost});
 7 | 
 8 |  let consumer = new Consumer(
 9 |     client,
10 |     [{ topic: config.KafkaTopic, partition: 0 }],
11 |     {
12 |       autoCommit: true,
13 |       fetchMaxWaitMs: 1000,
14 |       fetchMaxBytes: 1024 * 1024,
15 |       encoding: 'utf8',
16 |       // fromOffset: false
17 |     }
18 |   );
19 |   consumer.on('message', async function(message) {
20 |     console.log(
21 |       'kafka ',
22 |       JSON.parse(message.value)
23 |     );
24 |   })
25 |   consumer.on('error', function(error) {
26 |     //  handle error 
27 |     console.log('error', error);
28 |   });
29 | }
30 | catch(error) {
31 |   // catch error trace
32 |   console.log(error);
33 | }
34 | 


--------------------------------------------------------------------------------
/producer.js:
--------------------------------------------------------------------------------
 1 | const Kafka = require('kafka-node');
 2 | const config  = require('./config');
 3 | 
 4 | const Producer = Kafka.Producer;
 5 | const client = new Kafka.KafkaClient({kafkaHost: config.KafkaHost});
 6 | const producer = new Producer(client,  {requireAcks: 0, partitionerType: 2});
 7 | 
 8 | 
 9 | 
10 | const pushDataToKafka =(dataToPush) => {
11 | 
12 |   try {
13 |   let payloadToKafkaTopic = [{topic: config.KafkaTopic, messages: JSON.stringify(dataToPush) }];
14 |   console.log(payloadToKafkaTopic);
15 |   producer.on('ready', async function() {
16 |     producer.send(payloadToKafkaTopic, (err, data) => {
17 |           console.log('data: ', data);
18 |   });
19 | 
20 |   producer.on('error', function(err) {
21 |     //  handle error cases here
22 |   })
23 |   })
24 |   }
25 | catch(error) {
26 |   console.log(error);
27 | }
28 | 
29 | };
30 | 
31 | 
32 | const jsonData = require('./app_json.js');
33 | 
34 | pushDataToKafka(jsonData);
35 | 


--------------------------------------------------------------------------------