├── .gitignore ├── assets ├── events.gif ├── throughput.gif ├── kafkaConnection_js_—_KafkaMirror.png └── kafkaConnection_js_—_KafkaMirror2.png ├── utilities.js ├── LICENSE ├── package.json ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json -------------------------------------------------------------------------------- /assets/events.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-mirror-connect/HEAD/assets/events.gif -------------------------------------------------------------------------------- /assets/throughput.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-mirror-connect/HEAD/assets/throughput.gif -------------------------------------------------------------------------------- /assets/kafkaConnection_js_—_KafkaMirror.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-mirror-connect/HEAD/assets/kafkaConnection_js_—_KafkaMirror.png -------------------------------------------------------------------------------- /assets/kafkaConnection_js_—_KafkaMirror2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-mirror-connect/HEAD/assets/kafkaConnection_js_—_KafkaMirror2.png -------------------------------------------------------------------------------- /utilities.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transformLogData: (log) => { 3 | const data = {}; 4 | data.type = 'producer'; 5 | data.level = log.level; 6 | data.timestamp = log.extra.timestamp; 7 | data.broker = log.extra.broker; 8 | data.clientId = log.extra.clientId; 9 | data.correlationId = log.extra.correlationId; 10 | data.size = log.extra.size; 11 | data.topics = log.extra.data.topics; 12 | return data; 13 | }, 14 | }; 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 OSLabs Beta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-mirror-connect", 3 | "version": "1.0.2", 4 | "description": "Connect Kafka application to KafkaMirror. Built on top of KafkaJS.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/oslabs-beta/KafkaMirror.git" 12 | }, 13 | "keywords": [ 14 | "['kafka'", 15 | "'kafkajs'", 16 | "'apache", 17 | "kafka'", 18 | "'apache'", 19 | "'kafka", 20 | "monitoring'", 21 | "'monitoring'", 22 | "'kafka", 23 | "alert'", 24 | "'kafka", 25 | "gui'", 26 | "'gui'", 27 | "'alert'", 28 | "'bitcoin'", 29 | "'cryptocurrency'", 30 | "'crypto'", 31 | "'codesmith'", 32 | "'fintech'", 33 | "'blockchain'", 34 | "'javascript']" 35 | ], 36 | "author": "Jonah Stewart (https://github.com/oslabs-beta/KafkaMirror)", 37 | "license": "ISC", 38 | "bugs": { 39 | "url": "https://github.com/oslabs-beta/KafkaMirror/issues" 40 | }, 41 | "homepage": "https://github.com/oslabs-beta/KafkaMirror#readme", 42 | "dependencies": { 43 | "kafkajs": "^1.15.0", 44 | "winston": "^3.3.3" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KafkaMirror 2 | 3 | Kafka monitoring tool 4 | 5 | ## Table of Contents 6 | 7 | #### Overview 8 | 9 | #### Demo 10 | 11 | #### Quick Start 12 | 13 | #### Viewing your metrics 14 | 15 | #### Open-Source Kafka simulator tool 16 | 17 | ## Overview 18 | 19 | - Monitor your Kafka application in real-time 20 | - Track throughput, events per second 21 | - Easily integrate KafkaMirror to your existing application 22 | - Compatible with Kafka applications utilizing KafkaJS 23 | 24 | KafkaMirror allows for easy, highly accurate monitoring of your kafka cluster in real-time. By directly accessing the log data for your kafka cluster, KafkaMirror delivers precise, reliable feedback which can be used to improve system health and identify irregularities. 25 | 26 | ## Demo 27 | 28 | ![kafka-mirror-events](./assets/events.gif 'KafkaMirror Event Metrics') 29 | ![kafka-mirror-throughput](./assets/throughput.gif 'KafkaMirror Throughput Metrics')
30 | 31 | ## Quick Start 32 | 33 | A few simple steps to initialize KafkaMirror: 34 | 35 | 1. `npm install kafka-mirror-connect` 36 | 2. In the file where you initialize your KafkaJS to connect with your Kafka Cluster, require in kafka-mirror connect.
37 | 38 | ```js 39 | const KafkaMirror = require('kafka-mirror-connect'); 40 | ``` 41 | 42 | 3. Replace "new Kafka" with a call to KafkaMirror.
43 | 44 | ```js 45 | const kafka = KafkaMirror({ 46 | clientId, 47 | brokers: ['localhost:9092'], 48 | }); 49 | ``` 50 | 51 |
52 | 53 | ## Viewing your metrics 54 | 55 | The KafkaMirror GUI can be used to view your metrics. 56 | 57 | 1. Clone this repo (https://github.com/oslabs-beta/KafkaMirror.git) 58 | 2. cd into KafkaMirror 59 | 3. Start the application with npm start 60 | 4. Navigate to localhost:3000 61 | 5. Within the GUI, navigate to the setting page and enter the location (e.g. port 9092) of your Kafka cluster. 62 | 63 | ## Open source Kafka simulator 64 | 65 | - https://github.com/oslabs-beta/kafka-simulator 66 | - preconfigured to work with KafkaMirror 67 | 68 | The KafkaMirror GUI can be used to view your metrics. 69 | 70 | ## Authors 71 | 72 | KafkaMirror Engineers:
73 | [Joe Kinney](https://github.com/joekinney-png)
74 | [Jonah Stewart](https://github.com/jonahlstewart)
75 | [Keon Kim](https://github.com/Keon-Kim-0)
76 | [Mark Miller](https://github.com/markmanuelmiller)
77 | [Tobi-wan Rodriguez](https://github.com/rtobiwan)
78 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // Node package which wraps KafkaJS and facilitates communication between applications that use Kafka and KafkaMirror 2 | // monitoring tool 3 | 4 | const { Kafka, logLevel } = require('kafkajs'); 5 | const winston = require('winston'); 6 | const { transformLogData } = require('./utilities'); 7 | 8 | // Create websocket that will send data about the Kafka cluster that Kafka simulator is running to KafkaMirror application 9 | // This is the socket.io "server" 10 | const app = require('express')(); 11 | const http = require('http').createServer(app); 12 | const io = require('socket.io')(http, { 13 | cors: { 14 | origin: 'http://localhost:8080', 15 | methods: ['GET', 'POST'], 16 | }, 17 | }); 18 | 19 | // Function that is invoked to create a connection between kafka-simulator or other Kafka application and KafkaMirror 20 | // at the specified port 21 | const KafkaMirror = (props, port = 3030) => { 22 | let socket = null; 23 | 24 | io.on('connection', (socketConnection) => { 25 | socket = socketConnection; 26 | console.log(`Client is connected [id=${socket.id}]`); 27 | socket.on('disconnect', () => { 28 | console.log(`Client disconnected [id=${socket.id}]`); 29 | }); 30 | }); 31 | 32 | http.listen(port, () => { 33 | console.log(`listening on port ${port}`); 34 | }); 35 | 36 | // specify log levels for winston logger 37 | const toWinstonLogLevel = (level) => { 38 | switch (level) { 39 | case logLevel.ERROR: 40 | case logLevel.NOTHING: 41 | return 'error'; 42 | case logLevel.WARN: 43 | return 'warn'; 44 | case logLevel.INFO: 45 | return 'info'; 46 | case logLevel.DEBUG: 47 | return 'debug'; 48 | } 49 | }; 50 | 51 | // Use chunking to send new data through socket io on a periodic basis every second as opposed to real-time to 52 | // accomodate large amounts of streaming data generated by node.js when parsing the yelp and other large datasets 53 | let chunk = []; 54 | setInterval(() => { 55 | io.sockets.emit('log', JSON.stringify(chunk)); 56 | console.log('chunk length is', chunk.length); 57 | chunk = []; 58 | }, 1000); 59 | 60 | // Initialize and configure Winston logger to be incorporated into the establishment of connection with Kafka application 61 | const WinstonLogCreator = (logLevel) => { 62 | let size; 63 | let newID; 64 | let lastSentID; 65 | 66 | // Winston needs at least one transport in order to register that a log has been recieved 67 | const logger = winston.createLogger({ 68 | level: toWinstonLogLevel(logLevel), 69 | transports: [new winston.transports.File({ filename: 'myapp.log' })], 70 | }); 71 | 72 | logger.stream({ start: -1 }).on('log', function (log) { 73 | // When Kafka produces a log of type Request Produce we pull the size and correlation ID 74 | // Size of message (e.g., size of yelp review text) is not included in the corresponding Response Produce 75 | // KafkaMirror pulls the size included in each log message and displays in the "throughput" chart of KafkaMirror 76 | // correlation ID stored to prevent duplicate produce logs 77 | if (log.message.indexOf('Request Produce') > -1) { 78 | size = log.extra.size; 79 | newID = log.extra.correlationId; 80 | } 81 | 82 | // When Kafka produces a log of type response produce, we will pull information from the previous 83 | // request produce and add data to our chuck array 84 | if (log.message.indexOf('Response Produce') > -1) { 85 | const data = transformLogData(log); 86 | data.requestSize = size; 87 | if (lastSentID !== newID) { 88 | chunk.push(data); 89 | lastSentID = newID; 90 | } 91 | } 92 | }); 93 | 94 | return ({ namespace, level, label, log }) => { 95 | const { message, ...extra } = log; 96 | logger.log({ 97 | level: toWinstonLogLevel(level), 98 | message, 99 | extra, 100 | }); 101 | }; 102 | }; 103 | 104 | // Invoked to create a new connection to a Kafka instance using KafkaJS node package 105 | const kafka = new Kafka({ 106 | ...props, 107 | logLevel: logLevel.DEBUG, 108 | logCreator: WinstonLogCreator, 109 | }); 110 | 111 | return kafka; 112 | }; 113 | 114 | module.exports = KafkaMirror; 115 | --------------------------------------------------------------------------------