├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── consumerExample.ts ├── example ├── .env ├── App.js ├── App.tsx ├── admin.js ├── admin.ts ├── chart.html ├── client │ ├── client.js │ └── client.ts ├── kafka.js ├── kafka.ts ├── main.js ├── main.ts ├── package-lock.json ├── package.json ├── producer.js ├── producer.ts ├── producer2.js ├── producer2.ts ├── test.js ├── test.ts ├── truck_engine_sensors.json └── tsconfig.json ├── kafka-socks ├── Confluent.ts ├── Consumer.ts ├── LICENSE ├── README.md ├── Subject.ts ├── __tests__ │ ├── Confluent.test.js │ ├── Confluent.test.ts │ ├── Consumer.test.ts │ └── Subject.test.ts ├── dist │ ├── Confluent.d.ts │ ├── Consumer.d.ts │ ├── Subject.d.ts │ ├── index.d.ts │ ├── index.js │ ├── jest-setup.d.ts │ ├── jest-setup.js │ ├── jest-teardown.d.ts │ ├── jest-teardown.js │ ├── jest.config.d.ts │ └── jest.config.js ├── index.ts ├── jest-setup.js ├── jest-teardown.js ├── jest.config.js ├── package-lock.json ├── package.json └── tsconfig.json ├── package-lock.json ├── playground ├── .babelrc.json ├── Dockerfile ├── __tests__ │ ├── endToEnd.js │ ├── enzymeTests.js │ ├── reducerTest.js │ ├── routeTest.js │ └── testToDo.md ├── client │ ├── App.tsx │ ├── BarDisplay.tsx │ ├── DataDisplay.tsx │ ├── assets │ │ ├── KSminiLogo.png │ │ ├── KSminiLogo.svg │ │ ├── allison.jpg │ │ ├── allison.svg │ │ ├── code.svg │ │ ├── data-collection.svg │ │ ├── jason.jpg │ │ ├── jenessa.jpg │ │ ├── logo.svg │ │ ├── miniLogo.svg │ │ ├── npm.svg │ │ ├── programing.svg │ │ └── vinit.jpeg │ ├── components │ │ ├── Features.js │ │ ├── Features.tsx │ │ ├── Footer.js │ │ ├── Footer.tsx │ │ ├── GettingStarted.js │ │ ├── GettingStarted.tsx │ │ ├── NavBar.js │ │ ├── NavBar.tsx │ │ ├── TeamMembers.js │ │ └── TeamMembers.tsx │ ├── containers │ │ ├── FeaturesContainer.tsx │ │ └── TeamContainer.tsx │ ├── custom.d.ts │ ├── index.html │ ├── index.tsx │ └── theme.ts ├── dist │ ├── 4442f5598cdd09dd61daa0acdbd5973e.jpg │ ├── 47e71068503c68fe50c6d69309a53c58.svg │ ├── 6f3d3a46540b585405dafab2ab839c0a.jpg │ ├── a129a10f1cab60807e5dbdd62ce025d5.jpg │ ├── a8b1e12c0f5f369bcc49e2a1cae8ff82.svg │ ├── bundle.js │ └── e444e1007793f23b14b6633daaa40408.jpeg ├── jest-setup.ts ├── jest-teardown.ts ├── package-lock.json ├── package.json ├── server │ ├── kafka.ts │ ├── producer.ts │ ├── server.js │ ├── server.ts │ └── truck_engine_sensors.json ├── tsconfig.json └── webpack.config.js ├── websocketExample.ts └── www ├── .docusaurus ├── client-modules.js └── docusaurus.config.js ├── .gitignore ├── README.md ├── babel.config.js ├── docs ├── demo.md ├── intro.md ├── start.md └── team.md ├── docusaurus.config.js ├── package-lock.json ├── package.json ├── sidebars.js ├── src ├── components │ ├── GettingStarted.js │ ├── GettingStarted.module.css │ ├── HomepageFeatures.js │ ├── HomepageFeatures.module.css │ └── Team.js ├── css │ └── custom.css └── pages │ ├── index.js │ ├── index.module.css │ └── markdown-page.md └── static ├── .nojekyll └── img ├── KS-logo-spin.png ├── KS-logo.png ├── Kafkasocks-full-logo.png ├── Kafkasocks-mini-logo.png ├── allison.svg ├── code.svg ├── consumer-subject.svg ├── custom-coding.svg ├── data-collection.svg ├── docusaurus.png ├── favicon.ico ├── jason.svg ├── jenessa.svg ├── layers.svg ├── logo.svg ├── producer.svg ├── programing.svg ├── socket-on.svg ├── undraw_docusaurus_mountain.svg ├── undraw_docusaurus_react.svg └── vinit.svg /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | *.bak 4 | Subject.js 5 | Consumer.js 6 | Confluent.js 7 | *.zip 8 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "pwa-chrome", 9 | "request": "launch", 10 | "name": "Launch Chrome against localhost", 11 | "url": "http://localhost:8080", 12 | "webRoot": "${workspaceFolder}" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "liveServer.settings.port": 5501 3 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | . 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with the owners of this repository before making a change. 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | ## Pull Request Process 9 | 10 | 1. Ensure any install or build dependencies are removed before the end of the layer when doing a 11 | build. 12 | 2. Update the README.md with details of changes to the interface, this includes new environment 13 | variables, exposed ports, useful file locations and container parameters. 14 | 3. Increase the version numbers in any examples files and the README.md to the new version that this 15 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 16 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 17 | do not have permission to do that, you may request the second reviewer to merge it for you. 18 | 19 | ## Code of Conduct 20 | 21 | ### Our Pledge 22 | 23 | In the interest of fostering an open and welcoming environment, we as 24 | contributors and maintainers pledge to making participation in our project and 25 | our community a harassment-free experience for everyone, regardless of age, body 26 | size, disability, ethnicity, gender identity and expression, level of experience, 27 | nationality, personal appearance, race, religion, or sexual identity and 28 | orientation. 29 | 30 | ### Our Standards 31 | 32 | Examples of behavior that contributes to creating a positive environment 33 | include: 34 | 35 | * Using welcoming and inclusive language 36 | * Being respectful of differing viewpoints and experiences 37 | * Gracefully accepting constructive criticism 38 | * Focusing on what is best for the community 39 | * Showing empathy towards other community members 40 | 41 | Examples of unacceptable behavior by participants include: 42 | 43 | * The use of sexualized language or imagery and unwelcome sexual attention or 44 | advances 45 | * Trolling, insulting/derogatory comments, and personal or political attacks 46 | * Public or private harassment 47 | * Publishing others' private information, such as a physical or electronic 48 | address, without explicit permission 49 | * Other conduct which could reasonably be considered inappropriate in a 50 | professional setting 51 | 52 | ### Our Responsibilities 53 | 54 | Project maintainers are responsible for clarifying the standards of acceptable 55 | behavior and are expected to take appropriate and fair corrective action in 56 | response to any instances of unacceptable behavior. 57 | 58 | Project maintainers have the right and responsibility to remove, edit, or 59 | reject comments, commits, code, wiki edits, issues, and other contributions 60 | that are not aligned to this Code of Conduct, or to ban temporarily or 61 | permanently any contributor for other behaviors that they deem inappropriate, 62 | threatening, offensive, or harmful. 63 | 64 | ### Scope 65 | 66 | This Code of Conduct applies both within project spaces and in public spaces 67 | when an individual is representing the project or its community. Examples of 68 | representing a project or community include using an official project e-mail 69 | address, posting via an official social media account, or acting as an appointed 70 | representative at an online or offline event. Representation of a project may be 71 | further defined and clarified by project maintainers. 72 | 73 | ### Enforcement 74 | 75 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 76 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 77 | complaints will be reviewed and investigated and will result in a response that 78 | is deemed necessary and appropriate to the circumstances. The project team is 79 | obligated to maintain confidentiality with regard to the reporter of an incident. 80 | Further details of specific enforcement policies may be posted separately. 81 | 82 | Project maintainers who do not follow or enforce the Code of Conduct in good 83 | faith may face temporary or permanent repercussions as determined by other 84 | members of the project's leadership. 85 | 86 | ### Attribution 87 | 88 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 89 | available at [http://contributor-covenant.org/version/1/4][version] 90 | 91 | [homepage]: http://contributor-covenant.org 92 | [version]: http://contributor-covenant.org/version/1/4/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kafka-Penguin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /consumerExample.ts: -------------------------------------------------------------------------------- 1 | import exampleKafka from './confluentClassExample'; 2 | //import Kafka Socks as needed 3 | import { Consumer, Subject } from 'kafka-socks'; 4 | 5 | //Create a new Consumer, passing in a KafkaJS consumer, a topic, and the name of your Websocket event 6 | const kafkaSocksConsumer = new Consumer(kafkaConsumer, 'kafka-topic', 'websocket-event-ID'); 7 | 8 | //Create a new Subject, passing in the Socket.io instance and the name of your subject 9 | const kafkaSocksSubject = new Subject(io, 'subject-name'); 10 | -------------------------------------------------------------------------------- /example/.env: -------------------------------------------------------------------------------- 1 | API_KEY=YA46J4FQIQRYNZY3 2 | API_SECRET=sbzVVNfzXWR5o296sd9MlNvoZoYFGIBa4dJMj/RJiRa7awkH2lD9IluziRdjQR8S 3 | KAFKA_BOOTSTRAP_SERVER=pkc-lzvrd.us-west4.gcp.confluent.cloud:9092 4 | TOPIC=trucks-topic 5 | PORT=3001 6 | -------------------------------------------------------------------------------- /example/App.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // import React from 'react'; 3 | // import ReactDOM from 'react-dom'; 4 | // import Button from '@material-ui/core/Button'; 5 | // function App() { 6 | // return ( 7 | // 10 | // ); 11 | // } 12 | // ReactDOM.render(, document.querySelector('#app')) 13 | // export default App; 14 | -------------------------------------------------------------------------------- /example/App.tsx: -------------------------------------------------------------------------------- 1 | // import React from 'react'; 2 | // import ReactDOM from 'react-dom'; 3 | // import Button from '@material-ui/core/Button'; 4 | 5 | // function App() { 6 | // return ( 7 | // 10 | // ); 11 | // } 12 | 13 | // ReactDOM.render(, document.querySelector('#app')) 14 | 15 | // export default App; -------------------------------------------------------------------------------- /example/admin.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | var kafka_1 = require("./kafka"); 40 | var topic = process.env.TOPIC; 41 | var admin = kafka_1.kafka.admin(); 42 | var main = function () { return __awaiter(void 0, void 0, void 0, function () { 43 | return __generator(this, function (_a) { 44 | switch (_a.label) { 45 | case 0: return [4 /*yield*/, admin.connect()]; 46 | case 1: 47 | _a.sent(); 48 | return [4 /*yield*/, admin.createTopics({ 49 | topics: [{ topic: topic, replicationFactor: 3 }], 50 | waitForLeaders: true 51 | })]; 52 | case 2: 53 | _a.sent(); 54 | return [2 /*return*/]; 55 | } 56 | }); 57 | }); }; 58 | main()["catch"](function (error) { 59 | console.error(error); 60 | process.exit(1); 61 | }); 62 | -------------------------------------------------------------------------------- /example/admin.ts: -------------------------------------------------------------------------------- 1 | import { kafka } from './kafka' 2 | 3 | const topic = process.env.TOPIC; 4 | const admin = kafka.admin() 5 | 6 | const main = async () => { 7 | await admin.connect() 8 | await admin.createTopics({ 9 | topics: [{ topic ,replicationFactor: 3}], 10 | waitForLeaders: true, 11 | }) 12 | } 13 | 14 | main().catch(error => { 15 | console.error(error) 16 | process.exit(1) 17 | }) -------------------------------------------------------------------------------- /example/client/client.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | exports.__esModule = true; 3 | var socket_io_client_1 = require("socket.io-client"); 4 | var head = document.getElementById('head'); 5 | head.innerHTML = 'new Head'; 6 | var messages = document.getElementById('messages'); 7 | // const item = document.createElement('li'); 8 | // item.textContent = 'hello'; 9 | // messages!.appendChild(item); 10 | var truckSocket = socket_io_client_1.io('/trucks'); 11 | truckSocket.on('truck message', function (msg) { 12 | var item = document.createElement('li'); 13 | item.textContent = msg; 14 | messages.appendChild(item); 15 | window.scrollTo(0, document.body.scrollHeight); 16 | }); 17 | -------------------------------------------------------------------------------- /example/client/client.ts: -------------------------------------------------------------------------------- 1 | import { io } from "socket.io-client"; 2 | 3 | const head = document.getElementById('head') 4 | head!.innerHTML = 'new Head'; 5 | const messages = document.getElementById('messages'); 6 | // const item = document.createElement('li'); 7 | // item.textContent = 'hello'; 8 | // messages!.appendChild(item); 9 | const truckSocket = io('/trucks') 10 | 11 | truckSocket.on('truck message', function(msg) { 12 | const item = document.createElement('li'); 13 | item.textContent = msg; 14 | messages!.appendChild(item); 15 | window.scrollTo(0, document.body.scrollHeight); 16 | }); -------------------------------------------------------------------------------- /example/kafka.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | exports.__esModule = true; 3 | exports.kafka = void 0; 4 | var Confluent_1 = require("../kafka-socks/Confluent"); 5 | require('dotenv').config(); 6 | var _a = process.env, API_KEY = _a.API_KEY, API_SECRET = _a.API_SECRET, KAFKA_BOOTSTRAP_SERVER = _a.KAFKA_BOOTSTRAP_SERVER; 7 | var broker = new Confluent_1["default"](API_KEY, API_SECRET, KAFKA_BOOTSTRAP_SERVER); 8 | var kafka = broker.create('new-client'); 9 | exports.kafka = kafka; 10 | -------------------------------------------------------------------------------- /example/kafka.ts: -------------------------------------------------------------------------------- 1 | import Confluent from '../kafka-socks/Confluent'; 2 | require('dotenv').config(); 3 | 4 | const { API_KEY, API_SECRET, KAFKA_BOOTSTRAP_SERVER } = process.env; 5 | const broker = new Confluent(API_KEY!, API_SECRET!, KAFKA_BOOTSTRAP_SERVER!); 6 | const kafka = broker.create('new-client'); 7 | 8 | 9 | export { kafka }; -------------------------------------------------------------------------------- /example/main.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | exports.__esModule = true; 3 | var producer_1 = require("./producer"); 4 | var kafka_1 = require("./kafka"); 5 | var Consumer_1 = require("../kafka-socks/Consumer"); 6 | var Subject_1 = require("../kafka-socks/Subject"); 7 | require('dotenv').config(); 8 | var express = require('express'); 9 | var port = process.env.PORT; 10 | var app = express(); 11 | var http = require('http'); 12 | var server = http.createServer(app); 13 | var Server = require('socket.io').Server; 14 | var io = new Server(server); 15 | var path = require('path'); 16 | app.get('/', function (req, res) { 17 | res.sendFile(__dirname + '/chart.html'); 18 | }); 19 | // in the html we do this 20 | //socket.emit('consumer', consumer) 21 | //io.on('consumer' (socket, consumer)) 22 | producer_1.produce()["catch"](function (error) { 23 | console.log(error); 24 | process.exit(1); 25 | }); 26 | var kafkaconsumer_1 = kafka_1.kafka.consumer({ 27 | groupId: 'truck-group-1' 28 | }); 29 | var kafkaconsumer_2 = kafka_1.kafka.consumer({ 30 | groupId: 'truck-group-2' 31 | }); 32 | //const consumer_1 = new Consumer(kafkaconsumer_1, 'trucks-topic-1'<--related to Kafka (what topic should the consumer subscribe to), `truck message-1`<-- socketio (what event should it emit)) // 33 | var consumer_1 = new Consumer_1["default"](kafkaconsumer_1, 'trucks-topic-1', "truck message-1"); // 34 | var consumer_2 = new Consumer_1["default"](kafkaconsumer_2, 'trucks-topic-2', 'truck message-2'); 35 | var trucks_subject = new Subject_1["default"](io, 'trucks'); 36 | trucks_subject.add(consumer_1); 37 | trucks_subject.add(consumer_2); 38 | //wrap connect in an event listener of sorts 39 | app.get('/consume', function (req, res) { 40 | trucks_subject.connect(); 41 | return res.send({ message: 'works!' }); 42 | }); 43 | app.get('/pause', function (req, res) { 44 | console.log('in the middleware for pause'); 45 | trucks_subject.pause(); 46 | }); 47 | app.get('/resume', function (req, res) { 48 | console.log('in the middleware for resume'); 49 | trucks_subject.resume(); 50 | }); 51 | // io.on('connection', socket => { 52 | // produce().catch(error => { 53 | // console.log(error); 54 | // process.exit(1); 55 | // }) 56 | // const consumer = kafka.consumer({ 57 | // groupId: 'truck-group' 58 | // }) 59 | // const consumer_run = new Consumer(consumer, process.env.TOPIC, 'truck message', io) 60 | // consumer_run.run() 61 | // }) 62 | // .catch(async error => { 63 | // console.error(error) 64 | // try { 65 | // await consumer.disconnect() 66 | // } catch (e) { 67 | // console.error('Failed to gracefully disconnect consumer', e) 68 | // } 69 | // process.exit(1) 70 | // }) 71 | // io.on('connection', socket => { 72 | // produce().catch(error => { 73 | // console.log(error); 74 | // process.exit(1); 75 | // }) 76 | // const consumer = kafka.consumer({ 77 | // groupId: 'truck-group' 78 | // }) 79 | // const consume = async () => { 80 | // // console.log('in consume()') 81 | // await consumer.connect() 82 | // await consumer.subscribe({ 83 | // topic: process.env.TOPIC, 84 | // // topic: process.env.TOPIC, 85 | // fromBeginning: true 86 | // }) 87 | // // let output; 88 | // await consumer.run({ 89 | // eachMessage: async ({ topic, partition, message }) => { 90 | // // output = message.value.toString(); 91 | // io.emit('truck message', message.value.toString()) 92 | // console.log('Received Message', 93 | // JSON.parse(message.value.toString())) 94 | // // console.log('Received message', { 95 | // // topic, 96 | // // partition, 97 | // // key: message.key.toString(), 98 | // // value: JSON.parse(message.value.toString()), 99 | // // valueString: message.value.toString() 100 | // // }) 101 | // } 102 | // }) 103 | // } 104 | // consume() 105 | // .catch(async error => { 106 | // console.error(error) 107 | // try { 108 | // await consumer.disconnect() 109 | // } catch (e) { 110 | // console.error('Failed to gracefully disconnect consumer', e) 111 | // } 112 | // process.exit(1) 113 | // }) 114 | // }) 115 | // consume method that connects, subscribes to a topic and consumes messages from that topic 116 | server.listen(port, function () { 117 | console.log("Listening on port " + server.address().port); 118 | }); 119 | require('dotenv').config(); 120 | // // running the server 121 | // server.listen(port, () => { 122 | // console.log(`Listening on port ${server.address().port}`); 123 | // }); 124 | // running the producer 125 | // produce().catch(error => { 126 | // console.log(error); 127 | // process.exit(1); 128 | // }) 129 | // Running the consumer 130 | // consume().then(message => { 131 | // // socket io stuff 132 | // }).catch(async error => { 133 | // console.error(error) 134 | // try { 135 | // await consumer.disconnect() 136 | // } catch (e) { 137 | // console.error('Failed to gracefully disconnect consumer', e) 138 | // } 139 | // process.exit(1) 140 | // }) 141 | -------------------------------------------------------------------------------- /example/main.ts: -------------------------------------------------------------------------------- 1 | 2 | import { produce } from './producer' 3 | import { kafka } from './kafka' 4 | import Consumer from '../kafka-socks/Consumer'; 5 | import Subject from '../kafka-socks/Subject' 6 | require('dotenv').config(); 7 | const express = require('express'); 8 | const port = process.env.PORT 9 | const app = express(); 10 | const http = require('http'); 11 | const server = http.createServer(app); 12 | const { Server } = require('socket.io'); 13 | const io = new Server(server); 14 | const path = require('path'); 15 | 16 | app.get('/', (req: any, res: any) => { 17 | res.sendFile(__dirname + '/chart.html') 18 | }); 19 | 20 | 21 | 22 | produce().catch((error: any) => { 23 | console.log(error); 24 | process.exit(1); 25 | }) 26 | 27 | const kafkaconsumer_1 = kafka.consumer({ 28 | groupId: 'truck-group-1' 29 | }) 30 | const kafkaconsumer_2 = kafka.consumer({ 31 | groupId: 'truck-group-2' 32 | }) 33 | 34 | //const consumer_1 = new Consumer(kafkaconsumer_1, 'trucks-topic-1'<--related to Kafka (what topic should the consumer subscribe to), `truck message-1`<-- socketio (what event should it emit)) // 35 | const consumer_1 = new Consumer(kafkaconsumer_1, 'trucks-topic-1', `truck message-1`) // 36 | const consumer_2 = new Consumer(kafkaconsumer_2, 'trucks-topic-2', 'truck message-2') 37 | const trucks_subject = new Subject(io, 'trucks') 38 | trucks_subject.add(consumer_1) 39 | trucks_subject.add(consumer_2) 40 | 41 | //wrap connect in an event listener of sorts 42 | app.get('/consume', (req: any, res : any) => { 43 | trucks_subject.connect() 44 | return res.send({message : 'works!'}) 45 | }) 46 | 47 | app.get('/pause', (req : any, res : any ) => { 48 | console.log('in the middleware for pause') 49 | trucks_subject.pause(); 50 | }) 51 | 52 | app.get('/resume', (req: any, res: any) => { 53 | console.log('in the middleware for resume') 54 | trucks_subject.resume(); 55 | }) 56 | // io.on('connection', socket => { 57 | 58 | 59 | // produce().catch(error => { 60 | // console.log(error); 61 | // process.exit(1); 62 | // }) 63 | 64 | // const consumer = kafka.consumer({ 65 | // groupId: 'truck-group' 66 | // }) 67 | 68 | // const consumer_run = new Consumer(consumer, process.env.TOPIC, 'truck message', io) 69 | 70 | // consumer_run.run() 71 | 72 | 73 | // }) 74 | 75 | // .catch(async error => { 76 | // console.error(error) 77 | // try { 78 | // await consumer.disconnect() 79 | // } catch (e) { 80 | // console.error('Failed to gracefully disconnect consumer', e) 81 | // } 82 | // process.exit(1) 83 | // }) 84 | // io.on('connection', socket => { 85 | 86 | 87 | // produce().catch(error => { 88 | // console.log(error); 89 | // process.exit(1); 90 | // }) 91 | 92 | // const consumer = kafka.consumer({ 93 | // groupId: 'truck-group' 94 | // }) 95 | 96 | // const consume = async () => { 97 | // // console.log('in consume()') 98 | // await consumer.connect() 99 | 100 | // await consumer.subscribe({ 101 | // topic: process.env.TOPIC, 102 | // // topic: process.env.TOPIC, 103 | // fromBeginning: true 104 | // }) 105 | 106 | // // let output; 107 | // await consumer.run({ 108 | // eachMessage: async ({ topic, partition, message }) => { 109 | // // output = message.value.toString(); 110 | // io.emit('truck message', message.value.toString()) 111 | // console.log('Received Message', 112 | // JSON.parse(message.value.toString())) 113 | // // console.log('Received message', { 114 | // // topic, 115 | // // partition, 116 | // // key: message.key.toString(), 117 | // // value: JSON.parse(message.value.toString()), 118 | // // valueString: message.value.toString() 119 | // // }) 120 | // } 121 | // }) 122 | // } 123 | 124 | // consume() 125 | // .catch(async error => { 126 | // console.error(error) 127 | // try { 128 | // await consumer.disconnect() 129 | // } catch (e) { 130 | // console.error('Failed to gracefully disconnect consumer', e) 131 | // } 132 | // process.exit(1) 133 | // }) 134 | 135 | // }) 136 | 137 | 138 | // consume method that connects, subscribes to a topic and consumes messages from that topic 139 | 140 | server.listen(port, () => { 141 | console.log(`Listening on port ${server.address().port}`); 142 | }); 143 | require('dotenv').config(); 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | // // running the server 152 | // server.listen(port, () => { 153 | // console.log(`Listening on port ${server.address().port}`); 154 | // }); 155 | 156 | // running the producer 157 | // produce().catch(error => { 158 | // console.log(error); 159 | // process.exit(1); 160 | // }) 161 | 162 | // Running the consumer 163 | 164 | // consume().then(message => { 165 | // // socket io stuff 166 | // }).catch(async error => { 167 | // console.error(error) 168 | // try { 169 | // await consumer.disconnect() 170 | // } catch (e) { 171 | // console.error('Failed to gracefully disconnect consumer', e) 172 | // } 173 | // process.exit(1) 174 | // }) -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "example", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@material-ui/core": "^4.11.3", 14 | "@material-ui/icons": "^4.11.2", 15 | "dotenv": "^9.0.2", 16 | "express": "^4.17.1", 17 | "kafkajs": "^1.15.0", 18 | "react": "^16.14.0", 19 | "react-dom": "^16.14.0", 20 | "react-router-dom": "^5.2.0", 21 | "react-scroll": "^1.8.2", 22 | "react-showdown": "^2.3.0", 23 | "react-syntax-highlighter": "^15.4.3", 24 | "react-tsparticles": "^1.26.3", 25 | "regenerator-runtime": "^0.13.7", 26 | "request": "^2.88.2", 27 | "set-interval": "^2.1.2", 28 | "socket.io": "^4.1.1", 29 | "ts-loader": "^9.1.2", 30 | "ts-node": "^9.1.1", 31 | "typescript": "^4.2.4" 32 | }, 33 | "devDependencies": { 34 | "@types/node": "^15.3.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /example/producer.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __generator = (this && this.__generator) || function (thisArg, body) { 12 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 13 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 14 | function verb(n) { return function (v) { return step([n, v]); }; } 15 | function step(op) { 16 | if (f) throw new TypeError("Generator is already executing."); 17 | while (_) try { 18 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 19 | if (y = 0, t) op = [op[0] & 2, t.value]; 20 | switch (op[0]) { 21 | case 0: case 1: t = op; break; 22 | case 4: _.label++; return { value: op[1], done: false }; 23 | case 5: _.label++; y = op[1]; op = [0]; continue; 24 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 25 | default: 26 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 27 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 28 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 29 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 30 | if (t[2]) _.ops.pop(); 31 | _.trys.pop(); continue; 32 | } 33 | op = body.call(thisArg, _); 34 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 35 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 36 | } 37 | }; 38 | exports.__esModule = true; 39 | exports.produce = void 0; 40 | var kafka_1 = require("./kafka"); 41 | var fs = require('fs'); 42 | var trucks = []; 43 | try { 44 | // read contents of the file 45 | var data = fs.readFileSync('truck_engine_sensors.json', 'UTF-8'); 46 | // split the contents by new line 47 | var lines = data.split(/\r?\n/); 48 | // print all lines 49 | lines.pop(); 50 | lines.forEach(function (line) { 51 | // trucks.push(JSON.parse(line)) 52 | trucks.push(line); 53 | }); 54 | } 55 | catch (err) { 56 | console.error(err); 57 | } 58 | // PRODUCER 59 | var producer = kafka_1.kafka.producer(); 60 | var produce = function () { return __awaiter(void 0, void 0, void 0, function () { 61 | var idx, interval; 62 | return __generator(this, function (_a) { 63 | switch (_a.label) { 64 | case 0: return [4 /*yield*/, producer.connect()]; 65 | case 1: 66 | _a.sent(); 67 | idx = 0; 68 | interval = setInterval(function () { return __awaiter(void 0, void 0, void 0, function () { 69 | var truck_num, responses, err_1; 70 | return __generator(this, function (_a) { 71 | switch (_a.label) { 72 | case 0: 73 | if (idx >= trucks.length - 1) { 74 | console.log('in interval inside produce'); 75 | console.log(idx); 76 | clearInterval(interval); 77 | } 78 | _a.label = 1; 79 | case 1: 80 | _a.trys.push([1, 3, , 4]); 81 | truck_num = JSON.parse(trucks[idx])["truck_id"]; 82 | return [4 /*yield*/, producer.send({ 83 | topic: "trucks-topic-" + truck_num, 84 | messages: [ 85 | { 86 | key: String(idx), 87 | // value : String(trucks[idx].engine_temperature) 88 | value: String(trucks[idx]) 89 | } 90 | ] 91 | })]; 92 | case 2: 93 | responses = _a.sent(); 94 | console.log("Published message ", trucks[idx]); 95 | // console.log('Published message, engine_temperature', trucks[idx].engine_temperature ) 96 | idx++; 97 | return [3 /*break*/, 4]; 98 | case 3: 99 | err_1 = _a.sent(); 100 | console.log("Error with producing: ", err_1); 101 | return [3 /*break*/, 4]; 102 | case 4: return [2 /*return*/]; 103 | } 104 | }); 105 | }); }, 1000); 106 | return [2 /*return*/]; 107 | } 108 | }); 109 | }); }; 110 | exports.produce = produce; 111 | -------------------------------------------------------------------------------- /example/producer.ts: -------------------------------------------------------------------------------- 1 | import { kafka } from './kafka' 2 | 3 | const fs = require('fs'); 4 | const trucks: any = [] 5 | 6 | try { 7 | // read contents of the file 8 | const data = fs.readFileSync('truck_engine_sensors.json', 'UTF-8'); 9 | // split the contents by new line 10 | const lines = data.split(/\r?\n/); 11 | // print all lines 12 | lines.pop(); 13 | lines.forEach((line : any) => { 14 | // trucks.push(JSON.parse(line)) 15 | trucks.push(line) 16 | }); 17 | } catch (err) { 18 | console.error(err); 19 | } 20 | 21 | // PRODUCER 22 | 23 | const producer = kafka.producer() 24 | 25 | const produce = async () => { 26 | await producer.connect(); 27 | let idx = 0; 28 | 29 | const interval = setInterval( async () => { 30 | if(idx>=trucks.length-1) { 31 | console.log('in interval inside produce') 32 | console.log(idx) 33 | clearInterval(interval); 34 | } 35 | try { 36 | const truck_num = JSON.parse(trucks[idx])["truck_id"] 37 | // console.log(truck_num) 38 | const responses = await producer.send({ 39 | topic : `trucks-topic-${truck_num}`, 40 | messages : [ 41 | { 42 | key: String(idx), 43 | // value : String(trucks[idx].engine_temperature) 44 | value: String(trucks[idx]) 45 | } 46 | ] 47 | }) 48 | console.log("Published message ", trucks[idx]); 49 | // console.log('Published message, engine_temperature', trucks[idx].engine_temperature ) 50 | idx++; 51 | } 52 | catch (err) { 53 | console.log("Error with producing: ", err); 54 | } 55 | 56 | }, 1000) 57 | } 58 | 59 | export { produce } -------------------------------------------------------------------------------- /example/producer2.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | const producer_1 = require("./producer"); 4 | producer_1.produce().catch((error) => { 5 | console.log(error); 6 | process.exit(1); 7 | }); 8 | -------------------------------------------------------------------------------- /example/producer2.ts: -------------------------------------------------------------------------------- 1 | import { produce } from './producer' 2 | 3 | produce().catch((error: any) => { 4 | console.log(error); 5 | process.exit(1); 6 | }) -------------------------------------------------------------------------------- /example/test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __importDefault = (this && this.__importDefault) || function (mod) { 3 | return (mod && mod.__esModule) ? mod : { "default": mod }; 4 | }; 5 | Object.defineProperty(exports, "__esModule", { value: true }); 6 | const Consumer_1 = __importDefault(require("../kafka-socks/Consumer")); 7 | const Confluent_1 = __importDefault(require("../kafka-socks/Confluent")); 8 | const Subject_1 = __importDefault(require("../kafka-socks/Subject")); 9 | console.log('hi'); 10 | console.log(Consumer_1.default); 11 | console.log(Confluent_1.default); 12 | console.log(Subject_1.default); 13 | -------------------------------------------------------------------------------- /example/test.ts: -------------------------------------------------------------------------------- 1 | import Consumer from '../kafka-socks/Consumer'; 2 | import Confluent from '../kafka-socks/Confluent'; 3 | import Subject from '../kafka-socks/Subject'; 4 | 5 | console.log('hi') 6 | console.log(Consumer); 7 | console.log(Confluent); 8 | console.log(Subject); 9 | -------------------------------------------------------------------------------- /kafka-socks/Confluent.ts: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | // Confluent class instantiates the connection Confluent Kafka cluster 4 | export class Confluent { 5 | key: string; 6 | secret: string; 7 | server: string; 8 | 9 | /** 10 | * Constructs a wrapper around a Confluent Kafka cluster 11 | * @param key the key or username for the Confluent Kafka cluster 12 | * @param secret the secret / password for the Confluent Kafka cluster 13 | * @param server the server URL for the Confluent Kafka cluster 14 | */ 15 | constructor(key: string, secret: string, server: string) { 16 | this.key = key; 17 | this.secret = secret; 18 | this.server = server; 19 | } 20 | 21 | /** 22 | * Instantiates a kafkaJS object from the Confluent Kafka cluster 23 | * @param client identifies the client ID for the Confluent cluster 24 | * @returns a KafkaJS Object 25 | */ 26 | create(client: string) { 27 | const sasl = 28 | this.key && this.secret 29 | ? { username: this.key, password: this.secret, mechanism: "plain" } 30 | : null; 31 | const ssl = !!sasl; 32 | 33 | return new Kafka({ 34 | clientId: client, 35 | brokers: [this.server], 36 | ssl, 37 | sasl, 38 | }); 39 | } 40 | } 41 | 42 | export default Confluent; 43 | -------------------------------------------------------------------------------- /kafka-socks/Consumer.ts: -------------------------------------------------------------------------------- 1 | const { Kafka } = require("kafkajs"); 2 | 3 | type ConsumerInterface = { 4 | connect: Function; 5 | subscribe: Function; 6 | run: Function; 7 | pause: Function; 8 | resume: Function; 9 | isConsuming: Boolean; 10 | }; 11 | 12 | type ioInterface = { 13 | emit: Function; 14 | }; 15 | 16 | type EventInterface = { 17 | topic: string; 18 | partition: string; 19 | message: Message; 20 | pause: boolean; 21 | resume: boolean; 22 | }; 23 | 24 | type Message = { 25 | value: string; 26 | }; 27 | 28 | export class Consumer { 29 | consumer: ConsumerInterface; 30 | topic: string; 31 | event: string; 32 | pause: boolean; 33 | resume: boolean; 34 | isConsuming: boolean; 35 | 36 | /** 37 | * Constructs the Kafka Socks consumer object, which wraps around a KafkaJS consumer object, 38 | * and associates the consumer object with a particular websocket event 39 | * @param consumer a Kafka Socks consumer 40 | * @param topic the topic to associate with the Kafka Socks consumer 41 | * @param event the websocket event ID 42 | */ 43 | constructor( 44 | consumer: ConsumerInterface, 45 | topic: string, 46 | event: string, 47 | pause: boolean = false, 48 | resume: boolean = false, 49 | isConsuming: boolean = false 50 | ) { 51 | this.consumer = consumer; // 52 | this.topic = topic; 53 | this.event = event; 54 | this.pause = pause; 55 | this.resume = resume; 56 | this.isConsuming = isConsuming; 57 | } 58 | 59 | /** 60 | * pauses the consumption of the KafkaSocks consumer 61 | */ 62 | pauser() { 63 | console.log("in pauser method"); 64 | this.resume = false; 65 | this.pause = true; 66 | } 67 | 68 | /** 69 | * Resumes a paused KafkaSocks consumer 70 | * @param namespace not used 71 | */ 72 | resumer(namespace: any) { 73 | console.log("in resume method"); 74 | // this.pause = false; 75 | this.resume = true; 76 | this.consumer.resume([{ topic: this.topic }]); 77 | // this.runAfterResume(namespace); 78 | console.log( 79 | "this.pause should be false here inside resumer(): ", 80 | this.pause 81 | ); 82 | console.log( 83 | "this.resume should be true for here inside resumer(): ", 84 | this.resume 85 | ); 86 | } 87 | 88 | // instantiate the Kafka consumer on the passed topic and subscribe with that consumer 89 | 90 | // async runAfterResume(namespace : any) { 91 | // await this.consumer.run({ 92 | 93 | // eachMessage: async (eventInfo: EventInterface) => { 94 | // //listening for a pause event 95 | // if(this.pause) { 96 | // this.consumer.pause([{topic: this.topic}]) 97 | // //setTimeout(() => this.consumer.resume([{topic: this.topic}]), 10000) 98 | // } 99 | 100 | // // else if(this.resume) { 101 | // // console.log('in Consumer.ts after resume is set to true :', this.resume) 102 | // // this.consumer.resume([{topic: this.topic}]); 103 | // // this.resume = false; 104 | // // } 105 | // namespace.emit(this.event, eventInfo.message.value.toString()); 106 | // console.log( 107 | // "received Message from kafka", 108 | // JSON.parse(eventInfo.message.value.toString()) 109 | // ); 110 | // }, 111 | // }); 112 | // } 113 | 114 | /** 115 | * Starts the KakfaJS consumers wrapped by the Kafka Socks consumer class 116 | * @param namespace a socket.io namespace 117 | */ 118 | async run(namespace: any) { 119 | console.log("Consumer in run()"); 120 | await this.consumer.connect(); 121 | console.log("consumer has connected"); 122 | 123 | await this.consumer.subscribe({ 124 | topic: this.topic, 125 | fromBeginning: true, 126 | }); 127 | console.log("consumer has subscribed to topic: ", this.topic); 128 | 129 | this.isConsuming = true; 130 | 131 | await this.consumer.run({ 132 | eachMessage: async (eventInfo: EventInterface) => { 133 | //listening for a pause event 134 | if (this.pause) { 135 | console.log("PAUSE IS TRUE"); 136 | if (this.resume) { 137 | console.log("inside nested resume in if(this.pause)"); 138 | console.log("this.resume: ", this.resume); 139 | } else { 140 | this.consumer.pause([{ topic: this.topic }]); 141 | } 142 | //setTimeout(() => this.consumer.resume([{topic: this.topic}]), 10000) 143 | } 144 | 145 | // else if(this.resume) { 146 | // console.log('in Consumer.ts after resume is set to true :', this.resume) 147 | // this.consumer.resume([{topic: this.topic}]); 148 | // this.resume = false; 149 | // } 150 | namespace.emit(this.event, eventInfo.message.value.toString()); 151 | console.log( 152 | "received Message from kafka", 153 | JSON.parse(eventInfo.message.value.toString()) 154 | ); 155 | }, 156 | }); 157 | 158 | // namespace.on('pause', () => { 159 | // console.log('disconnected...') 160 | // // this.consumer.pause(/** */) 161 | // }); 162 | console.log("consumer has run"); 163 | } 164 | } 165 | 166 | export default Consumer; 167 | -------------------------------------------------------------------------------- /kafka-socks/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 OSLabs Beta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /kafka-socks/README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

Kafka Socks

5 |

An easy-to-use, lightweight KafkaJS-to-Socket.io library for connecting and visualizing data in realtime.

6 |

7 | 8 |

9 | 10 |

About

11 | 12 | Kafka Socks is an easy-to-use and lightweight framework that combines Kafka consumer functionality with WebSockets to pipe the Kafka messages directly to the frontend client, in realtime. Kafka Socks abstracts away much of the boilerplate and setup of this oft-used Kafka-Websocket architecture, providing developers with a simple and intuitive set of classes to achieve a powerful result on the client-side. 13 | 14 | The typical use case for the Kafka Socks library is rendering realtime data on a frontend client, but Kafka Socks framework is unopinionated and flexible enough to process realtime data in whatever way the developer may see fit. 15 | 16 | Without a WebSocket, the only way a web client could access data consumed by the Kafka consumer on the server side would be fetch requests. Not only are fetch requests notoriously slow, browsers also limit the number of responded fetch requests a client may have at any given time (most browers set this limit below 10). In short, fetch requests could get the data to the frontend, but doing so would mean that the frontend would lose the ability to access this data in realtime; in applications where frontend rendering in realtime is necessary, fetch requests simply won't work. 17 | 18 | Using the observer design pattern, WebSockets permit the server to pipe data in time because there is always an established and open link between the server and client. Kafka Socks did not invent this system design. In fact, it is a relatively common pattern to achieve realtime data processing on the frontend. Instead, Kafka Socks abstracts away the details of implementing this kafka-websocket design pattern, providing developers with an easy way to implement this pattern in a few lines of code. 19 | 20 |

Features

21 | 22 | - Confluent : A singleton class used to instantiate a Kafka Cluster object using cluster hosted by Confluent.io

23 | - Consumer: A wrapper around a kafkaJS Consumer object, instantiate as many (or as few) consumers as needed

24 | - Subject: Used to create a new Kafka Socks Subject, which pipes the messages consumed by the Kafka consumers to the specified websocket namespace 25 | 26 |

Getting Started

27 | 28 | Install Kafka Socks as an npm module and save it to your package.json as a dependency. 29 | 30 | `npm install kafka-socks` 31 | 32 | Once installed, you can now require the modules necessary to implement Kafka Socks: 33 | 34 | `import { Confluent, Consumer, Subject } from 'kafka-socks';` 35 | 36 |

How to Use

37 | 38 | 1. Import the library classes needed: 39 | 40 | ```javascript 41 | import { Confluent, Consumer, Subject } from 'kafka-socks'; 42 | ``` 43 | 44 | 2. Instantiate a websocket server. (Done here using socket.io to wrap around an express server): 45 | 46 | ```javascript 47 | const express = require('express'); 48 | const http = require('http'); 49 | const { Server } = require('socket.io'); 50 | 51 | const app = express(); 52 | const server = http.createServer(app); 53 | const io = new Server(server); 54 | 55 | ``` 56 | 57 | 3. Instantiate the Kafka Cluster object using the Kafka Socks Confluent class: 58 | 59 | ```javascript 60 | const kafka = new Confluent( 61 | API_KEY, 62 | API_SECRET, 63 | KAFKA_BOOTSTRAP_SERVER 64 | ) 65 | .create("client-id"); 66 | 67 | ``` 68 | 69 | 4. Instantiate Kafka Socks Consumer object - you can create as many as you need: 70 | ```javascript 71 | const kafkaConsumer = kafka.consumer({ groupId: 'your-groupId-here' }); 72 | const kafkaSocksConsumer = new Consumer(kafkaConsumer, 'kafka-topic', 'websocket-event-ID') 73 | ``` 74 | 75 | 5. Link the Kafkasocks Consumers with websocket namespaces for the front end: 76 | ```javascript 77 | const kafkaSocksSubject = new Subject(io, 'websocket-namespace-ID') 78 | ``` 79 | 80 | 6. Then simply set up your WebSocket listener on the front end using your favorite WebSockets framework! 81 | 82 |

Contributors

83 | 84 | Kafka Socks is an open-source community project on Github. While the project is maintained by a small group of dedicated engineers (below), we are grateful to the community for bug fixes, feature development and other contributions. 85 | 86 | [Allison Jacobs @allisonIsCoding](https://github.com/allisonIsCoding) 87 | 88 | [Jason Fricano @jfricano](https://github.com/jfricano) 89 | 90 | [Jenessa Chapalamadugu @jenessachap](https://github.com/jenessachap) 91 | 92 | [Vinit Patel @v-za](https://github.com/v-za) 93 | 94 | We welcome contributions to Kafka Socks, but we also would love to see a thriving third-party ecosystem. If you are interest in creating an open-source project that builds on top of Kafka Socks, please don't hesitate to reach out, and we'd be happy to provide feedback and support. 95 | 96 |

License

97 | 98 | This product is licensed under the MIT License - see the LICENSE.md file for details. 99 | 100 | This is an open source product. We are not affiliated nor endorsed by either the Apache Software Foundation or KafkaJS. 101 | 102 | This product is accelerated by [OS Labs](https://opensourcelabs.io/). 103 | -------------------------------------------------------------------------------- /kafka-socks/Subject.ts: -------------------------------------------------------------------------------- 1 | export class Subject { 2 | io: any; 3 | namespace: any; 4 | consumerArr: any[]; 5 | 6 | // io is socket server instance 7 | // consumerArr is an array of KafkaSocks consumerArr (from the Consumer class we created) 8 | 9 | /** 10 | * Constructs the entire set of the Kafka Socks Consumers 11 | * @param io the socket.io server object 12 | * @param namespace the websocket namespace id 13 | * @param consumerArr an array of Kafka Socks consumer objects 14 | */ 15 | constructor(io: any, namespace: string, consumerArr: any[] = []) { 16 | this.io = io; 17 | this.namespace = this.io.of("/" + namespace); 18 | this.consumerArr = consumerArr; 19 | console.log("subject made"); 20 | } 21 | 22 | /** 23 | * Adds the Kafka Socks consumer object to consumer array 24 | * @param consumer a Kafka socks consumer object 25 | */ 26 | add(consumer: any) { 27 | // instantiate the Kakfa Consumer using the Kafa-provided class + add that consumer to our array 28 | console.log("in Subject.add"); 29 | this.consumerArr.push(consumer); 30 | } 31 | 32 | /** 33 | * pauses all of the Kafka Socks consumers in the consumer array 34 | */ 35 | pause() { 36 | this.consumerArr.forEach((consumer: any) => { 37 | consumer.pauser(); 38 | 39 | // socket.on('disconnect', () => { 40 | // console.log('disconnecting'); 41 | // consumer.disconnect().then(() => console.log('disconnected')); 42 | // }); 43 | }); 44 | } 45 | 46 | /** 47 | * resumes all of the Kafka Socks consumers in the consumer array 48 | */ 49 | resume() { 50 | console.log("in resume inside Subject.ts"); 51 | this.consumerArr.forEach((consumer: any) => { 52 | consumer.resumer(this.namespace); 53 | }); 54 | } 55 | 56 | /** 57 | * intializes the listener for the websocket namespaces 58 | */ 59 | connect() { 60 | console.log("in Subject.connect()"); 61 | this.namespace.on("connection", (socket: any) => { 62 | console.log("in the namespace connection"); 63 | // we need the socket to be listening to the event here 64 | // there is no socket.on('this.event') 65 | 66 | // socket.on('disconnect', consumer => { 67 | // consumer. 68 | // }); 69 | 70 | console.log("in namespace.on cb"); 71 | this.consumerArr.forEach((consumer: any) => { 72 | consumer.run(this.namespace); 73 | 74 | // socket.on('disconnect', () => { 75 | // console.log('disconnecting'); 76 | // consumer.disconnect().then(() => console.log('disconnected')); 77 | // }); 78 | }); 79 | }); 80 | } 81 | 82 | // disconnect(event: any) { 83 | // this.namespace.on(event, (socket : object) => { 84 | // console.log('disconnected...') 85 | // }) 86 | // } 87 | } 88 | 89 | export default Subject; 90 | -------------------------------------------------------------------------------- /kafka-socks/__tests__/Confluent.test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // test confluent object shape: 3 | // key property 4 | // sercret property 5 | // server property 6 | exports.__esModule = true; 7 | var Confluent_1 = require("../Confluent"); 8 | var Kafka = require("kafkajs").Kafka; 9 | var Consumer = require("../Consumer"); 10 | require("dotenv").config(); 11 | // type ConfluentType = { 12 | // create: Function; 13 | // }; 14 | // test consumer object shape: 15 | // consumer property: is a ks consumer object 16 | // topic property: is a string 17 | // event property: is a string 18 | describe("Consumer Class", function () { 19 | var API_KEY = "PS5UR5WJMR3M4IUK"; 20 | var API_SECRET = "sViLnhxYPSZzirnBznMVHxRoQbvltcpmOJjlvnuv0f+SW138XyA1ZmO/kp7K87sg"; 21 | var KAFKA_BOOTSTRAP_SERVER = "pkc-lzvrd.us-west4.gcp.confluent.cloud:9092"; 22 | var confluent; 23 | beforeAll(function () { 24 | confluent = new Confluent_1["default"](API_KEY, API_SECRET, KAFKA_BOOTSTRAP_SERVER); 25 | }); 26 | describe("Test class properties", function () { 27 | it("should have the same consumer property as input", function () { 28 | expect(confluent.key).toBe(API_KEY); 29 | }); 30 | it("should have a topic property equal to input string", function () { 31 | expect(confluent.secret).toBe(API_SECRET); 32 | }); 33 | it("should have an event property equal to input string", function () { 34 | expect(confluent.server).toBe(KAFKA_BOOTSTRAP_SERVER); 35 | }); 36 | }); 37 | describe("Test create() method", function () { 38 | var kafka; 39 | beforeAll(function () { return (kafka = confluent.create("test-client")); }); 40 | it("should return a kafkajs Kafka object", function () { 41 | // expect().toBe(); 42 | }); 43 | it("kafkajs object should have clientId of 'test-client'", function () { 44 | // expect().toBe(); 45 | }); 46 | it("kafkajs object should be connected to proper broker", function () { 47 | // expect().toBe(); 48 | }); 49 | it("connection should be to proper broker", function () { 50 | // expect().toBe(); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /kafka-socks/__tests__/Confluent.test.ts: -------------------------------------------------------------------------------- 1 | // test confluent object shape: 2 | // key property 3 | // sercret property 4 | // server property 5 | 6 | import Confluent from "../Confluent"; 7 | 8 | const { Kafka } = require("kafkajs"); 9 | const Consumer = require("../Consumer"); 10 | require("dotenv").config(); 11 | 12 | // type ConfluentType = { 13 | // create: Function; 14 | // }; 15 | // test consumer object shape: 16 | // consumer property: is a ks consumer object 17 | // topic property: is a string 18 | // event property: is a string 19 | describe("Consumer Class", () => { 20 | const API_KEY = "PS5UR5WJMR3M4IUK"; 21 | const API_SECRET = `sViLnhxYPSZzirnBznMVHxRoQbvltcpmOJjlvnuv0f+SW138XyA1ZmO/kp7K87sg`; 22 | const KAFKA_BOOTSTRAP_SERVER = `pkc-lzvrd.us-west4.gcp.confluent.cloud:9092`; 23 | 24 | let confluent: { 25 | key: string, 26 | secret: string, 27 | server: string, 28 | create: Function 29 | }; 30 | 31 | beforeAll(() => { 32 | confluent = new Confluent(API_KEY, API_SECRET, KAFKA_BOOTSTRAP_SERVER); 33 | }); 34 | 35 | describe("Test class properties", () => { 36 | it("should have the same consumer property as input", () => { 37 | expect(confluent.key).toBe(API_KEY); 38 | }); 39 | 40 | it("should have a topic property equal to input string", () => { 41 | expect(confluent.secret).toBe(API_SECRET); 42 | }); 43 | 44 | it("should have an event property equal to input string", () => { 45 | expect(confluent.server).toBe(KAFKA_BOOTSTRAP_SERVER); 46 | }); 47 | }); 48 | 49 | describe("Test create() method", () => { 50 | let kafka; 51 | 52 | beforeAll(() => (kafka = confluent.create("test-client"))); 53 | 54 | it("should return a kafkajs Kafka object", () => { 55 | // expect().toBe(); 56 | }); 57 | 58 | it("kafkajs object should have clientId of 'test-client'", () => { 59 | // expect().toBe(); 60 | }); 61 | 62 | it("kafkajs object should be connected to proper broker", () => { 63 | // expect().toBe(); 64 | }); 65 | 66 | it("connection should be to proper broker", () => { 67 | // expect().toBe(); 68 | }); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /kafka-socks/__tests__/Consumer.test.ts: -------------------------------------------------------------------------------- 1 | require('dotenv').config(); 2 | const { Kafka } = require('kafkajs'); 3 | // const Consumer = require('../Consumer'); 4 | // import { Kafka } from 'kafkajs'; 5 | import Consumer from '../Consumer'; 6 | 7 | // test consumer object shape: 8 | // consumer property: is a ks consumer object 9 | // topic property: is a string 10 | // event property: is a string 11 | describe('Consumer Class', () => { 12 | let ksConsumer: { consumer: object, topic: string, event: string }; 13 | 14 | beforeAll(() => { 15 | const API_KEY = 'PS5UR5WJMR3M4IUK'; 16 | const API_SECRET = 'sViLnhxYPSZzirnBznMVHxRoQbvltcpmOJjlvnuv0f+SW138XyA1ZmO/kp7K87sg'; 17 | const KAFKA_BOOTSTRAP_SERVER = 'pkc-lzvrd.us-west4.gcp.confluent.cloud:9092'; 18 | const sasl = 19 | API_KEY && API_SECRET 20 | ? { username: API_KEY, password: API_SECRET, mechanism: 'plain' } 21 | : null; 22 | const ssl = !!sasl; 23 | 24 | const kafka = new Kafka({ 25 | clientId: 'client-id', 26 | brokers: [KAFKA_BOOTSTRAP_SERVER], 27 | ssl, 28 | sasl, 29 | }); 30 | 31 | ksConsumer = new Consumer(kafka.consumer(), 'test-topic', 'test-event'); 32 | }); 33 | 34 | describe('Test class properties', () => { 35 | it('should have the same consumer property as input', () => { 36 | expect(ksConsumer.consumer).toBe(ksConsumer); 37 | }); 38 | 39 | it('should have a topic property equal to input string', () => { 40 | expect(ksConsumer.topic).toBe('test-topic'); 41 | }); 42 | 43 | it('should have an event property equal to input string', () => { 44 | expect(ksConsumer.event).toBe('test-event'); 45 | }); 46 | }); 47 | 48 | // how to test the run() function??? 49 | }); 50 | 51 | // run method: how do we test this?? async.... 52 | -------------------------------------------------------------------------------- /kafka-socks/__tests__/Subject.test.ts: -------------------------------------------------------------------------------- 1 | // test object properties: 2 | // io is an http server?? 3 | // namespace is a ???? 4 | // consumerArr is array of ks consumer objects 5 | 6 | // add property: test by looking at the resulting array 7 | 8 | // connect() property: how to test this?? -------------------------------------------------------------------------------- /kafka-socks/dist/Confluent.d.ts: -------------------------------------------------------------------------------- 1 | export declare class Confluent { 2 | key: string; 3 | secret: string; 4 | server: string; 5 | constructor(key: string, secret: string, server: string); 6 | create(client: string): any; 7 | } 8 | export default Confluent; 9 | -------------------------------------------------------------------------------- /kafka-socks/dist/Consumer.d.ts: -------------------------------------------------------------------------------- 1 | declare type ConsumerInterface = { 2 | connect: Function; 3 | subscribe: Function; 4 | run: Function; 5 | pause: Function; 6 | resume: Function; 7 | isConsuming: Boolean; 8 | }; 9 | export declare class Consumer { 10 | consumer: ConsumerInterface; 11 | topic: string; 12 | event: string; 13 | pause: boolean; 14 | resume: boolean; 15 | isConsuming: boolean; 16 | constructor(consumer: ConsumerInterface, topic: string, event: string, pause?: boolean, resume?: boolean, isConsuming?: boolean); 17 | pauser(): void; 18 | resumer(namespace: any): void; 19 | runAfterResume(namespace: any): Promise; 20 | run(namespace: any): Promise; 21 | } 22 | export default Consumer; 23 | -------------------------------------------------------------------------------- /kafka-socks/dist/Subject.d.ts: -------------------------------------------------------------------------------- 1 | export declare class Subject { 2 | io: any; 3 | namespace: any; 4 | consumerArr: any[]; 5 | constructor(io: any, namespace: string, consumerArr?: any[]); 6 | add(consumer: any): void; 7 | pause(): void; 8 | resume(): void; 9 | connect(): void; 10 | } 11 | export default Subject; 12 | -------------------------------------------------------------------------------- /kafka-socks/dist/index.d.ts: -------------------------------------------------------------------------------- 1 | export { Consumer } from './Consumer'; 2 | export { Subject } from './Subject'; 3 | export { Confluent } from './Confluent'; 4 | -------------------------------------------------------------------------------- /kafka-socks/dist/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Confluent = exports.Subject = exports.Consumer = void 0; 4 | var Consumer_1 = require("./Consumer"); 5 | Object.defineProperty(exports, "Consumer", { enumerable: true, get: function () { return Consumer_1.Consumer; } }); 6 | var Subject_1 = require("./Subject"); 7 | Object.defineProperty(exports, "Subject", { enumerable: true, get: function () { return Subject_1.Subject; } }); 8 | var Confluent_1 = require("./Confluent"); 9 | Object.defineProperty(exports, "Confluent", { enumerable: true, get: function () { return Confluent_1.Confluent; } }); 10 | -------------------------------------------------------------------------------- /kafka-socks/dist/jest-setup.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/Kafkasocks/ba4a9f83134614d01da041f7f92b317198319cbd/kafka-socks/dist/jest-setup.d.ts -------------------------------------------------------------------------------- /kafka-socks/dist/jest-setup.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // module.exports = async () => { 3 | // global.testServer = await require('./server/server.js'); 4 | // }; 5 | -------------------------------------------------------------------------------- /kafka-socks/dist/jest-teardown.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/Kafkasocks/ba4a9f83134614d01da041f7f92b317198319cbd/kafka-socks/dist/jest-teardown.d.ts -------------------------------------------------------------------------------- /kafka-socks/dist/jest-teardown.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | // module.exports = async (globalConfig) => { 3 | // testServer.close(); 4 | // }; 5 | -------------------------------------------------------------------------------- /kafka-socks/dist/jest.config.d.ts: -------------------------------------------------------------------------------- 1 | export const preset: string; 2 | export const testEnvironment: string; 3 | -------------------------------------------------------------------------------- /kafka-socks/dist/jest.config.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node', 5 | }; 6 | -------------------------------------------------------------------------------- /kafka-socks/index.ts: -------------------------------------------------------------------------------- 1 | 2 | export { Consumer } from './Consumer' 3 | export { Subject } from './Subject' 4 | export { Confluent } from './Confluent' -------------------------------------------------------------------------------- /kafka-socks/jest-setup.js: -------------------------------------------------------------------------------- 1 | // module.exports = async () => { 2 | // global.testServer = await require('./server/server.js'); 3 | // }; 4 | -------------------------------------------------------------------------------- /kafka-socks/jest-teardown.js: -------------------------------------------------------------------------------- 1 | // module.exports = async (globalConfig) => { 2 | // testServer.close(); 3 | // }; 4 | -------------------------------------------------------------------------------- /kafka-socks/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /kafka-socks/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-socks", 3 | "version": "0.1.9", 4 | "description": "An easy-to-use, lightweight KafkaJS-to-Socket.io library for connecting and visualizing data in realtime.", 5 | "main": "dist/index.js", 6 | "types": "index.d.ts", 7 | "files": [ 8 | "/dist" 9 | ], 10 | "scripts": { 11 | "test": "ts-node Confluent Consumer Subject && npx jest --verbose true", 12 | "__run__": "tsc Kafkasocks.ts && node Kafkasocks.js" 13 | }, 14 | "keywords": [ 15 | "kafka", 16 | "websocket", 17 | "socket.io", 18 | "kafkajs" 19 | ], 20 | "author": "Allison Jacobs, Jason Fricano, Jenessa Chapalamadugu, Vinit Patel", 21 | "license": "MIT", 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/oslabs-beta/Kafkasocks" 25 | }, 26 | "dependencies": { 27 | "dotenv": "^10.0.0", 28 | "kafkajs": "^1.15.0", 29 | "socket.io": "^4.1.2" 30 | }, 31 | "devDependencies": { 32 | "@types/jest": "^26.0.23", 33 | "@types/node": "^15.6.1", 34 | "jest": "^27.0.1", 35 | "ts-jest": "^27.0.1", 36 | "ts-node": "^9.1.1", 37 | "typescript": "^4.2.4" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /kafka-socks/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, 4 | "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, 5 | "allowJs": true /* Allow javascript files to be compiled. */, 6 | "strict": true /* Enable all strict type-checking options. */, 7 | "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, 8 | "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */, 9 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 10 | "skipLibCheck": true /* Skip type checking of declaration files. */, 11 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */, 12 | "watch": true, 13 | "declaration": true, 14 | "outDir": "./dist", 15 | 16 | }, 17 | // "include": ["kafka-socks/**/*"], 18 | "exclude": ["../playground", "../example", "../www", "node_modules", "./dist"] 19 | } 20 | -------------------------------------------------------------------------------- /playground/.babelrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env", 4 | "@babel/preset-react", 5 | "@babel/preset-typescript" 6 | ] 7 | } -------------------------------------------------------------------------------- /playground/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16.1.0 2 | RUN npm i -g webpack 3 | RUN npm i -g ts-node 4 | WORKDIR /usr/src/app 5 | COPY . /usr/src/app 6 | RUN npm install 7 | RUN npm run build 8 | EXPOSE 3001 9 | CMD ["ts-node", "./server/server.ts"] 10 | -------------------------------------------------------------------------------- /playground/__tests__/endToEnd.js: -------------------------------------------------------------------------------- 1 | // - Expect log in page to render 2 | describe('', () => { 3 | xit('', () => { 4 | expect(); 5 | }); 6 | }); 7 | 8 | 9 | -------------------------------------------------------------------------------- /playground/__tests__/enzymeTests.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { configure, shallow } from 'enzyme'; 3 | import Adapter from 'enzyme-adapter-react-16'; 4 | 5 | configure({ adapter: new Adapter() }); 6 | 7 | // import components here ------------------------------ 8 | // import SignUp from '../testComponents/SignUp'; 9 | // import Home from '../testComponents/Home'; 10 | // import UserPage from '../testComponents/UserPage'; 11 | // import NewMessage from '../testComponents/NewMessage'; 12 | // import MessageBox from '../testComponents/MessageBox'; 13 | // import Login from '../testComponents/Login'; 14 | 15 | // describe('React Tests', () => { 16 | // describe('SignUp', () => { 17 | // let wrapper; 18 | // const props = { 19 | // testProp: 0, 20 | // signup: () => (props.testProp += 1), 21 | // }; 22 | 23 | // beforeAll(() => { 24 | // wrapper = shallow(); 25 | // }); 26 | 27 | // it('renders main login div', () => { 28 | // expect(wrapper.find('.loginsignuppage')).toHaveLength(1); 29 | // }); 30 | 31 | // it('renders three login inputs', () => { 32 | // expect(wrapper.find('.loginInput')).toHaveLength(3); 33 | // }); 34 | 35 | // it('renders one signup button', () => { 36 | // expect(wrapper.find('button')).toHaveLength(1); 37 | // }); 38 | 39 | // it('renders on click function when button is pressed', () => { 40 | // wrapper.find('button').simulate('click'); 41 | // expect(props.testProp).toEqual(1); 42 | // }); 43 | // }); 44 | 45 | // describe('Home', () => { 46 | // let wrapper; 47 | // const props = {}; 48 | 49 | // beforeAll(() => { 50 | // wrapper = shallow(); 51 | // }); 52 | 53 | // it('contains one UserPage element', () => { 54 | // expect(wrapper.find(UserPage)).toHaveLength(1); 55 | // }); 56 | // }); 57 | 58 | // describe('UserPage', () => { 59 | // let wrapper; 60 | 61 | // beforeAll(() => { 62 | // wrapper = shallow(); 63 | // }); 64 | 65 | // it('should render 4 buttons', () => { 66 | // expect(wrapper.find('button')).toHaveLength(4); 67 | // }); 68 | // }); 69 | 70 | // describe('NewMessage', () => { 71 | // let wrapper; 72 | 73 | // const props = { 74 | // testProp: 0, 75 | // send: () => (props.testProp += 1), 76 | // }; 77 | 78 | // beforeAll(() => { 79 | // wrapper = shallow(); 80 | // }); 81 | 82 | // it('should have an input box for username', () => { 83 | // expect(wrapper.find('input')).toHaveLength(1); 84 | // }); 85 | 86 | // it('should have a textarea for message', () => { 87 | // expect(wrapper.find('textarea')).toHaveLength(1); 88 | // }); 89 | 90 | // it('should have a send button', () => { 91 | // expect(wrapper.find('button')).toHaveLength(1); 92 | // }); 93 | 94 | // it('should invoke on click function when button pressed', () => { 95 | // wrapper.find('button').simulate('click'); 96 | // expect(props.testProp).toEqual(1); 97 | // }); 98 | // }); 99 | 100 | // describe('MessageBox', () => { 101 | // let wrapper; 102 | // const props = { 103 | // username: 'joe', 104 | // message: { 105 | // input: 'hi', 106 | // }, 107 | // }; 108 | 109 | // beforeAll(() => { 110 | // wrapper = shallow(); 111 | // }); 112 | 113 | // it('should render username from props', () => { 114 | // expect(wrapper.find('.senderName').text()).toEqual('joe'); 115 | // }); 116 | 117 | // it('should render input from props', () => { 118 | // expect(wrapper.find('.message').text()).toEqual('hi'); 119 | // }); 120 | // }); 121 | 122 | // describe('Login', () => { 123 | // let wrapper; 124 | // const props = { 125 | // prop1: 0, 126 | // prop2: 0, 127 | // submitLogin: () => (props.prop1 += 1), 128 | // onSignUpClick: () => (props.prop2 += 1), 129 | // }; 130 | 131 | // beforeAll(() => { 132 | // wrapper = shallow(); 133 | // }); 134 | 135 | // it('should render two login inputs', () => { 136 | // expect(wrapper.find('input')).toHaveLength(2); 137 | // }); 138 | 139 | // it('should render two buttons', () => { 140 | // expect(wrapper.find('button')).toHaveLength(2); 141 | // }); 142 | 143 | // it('should invoke submitLogin when button is clicked', () => { 144 | // wrapper.find('button').at(0).simulate('click'); 145 | // expect(props.prop1).toEqual(1); 146 | // }); 147 | 148 | // it('should invoke onSignUpClick when button is clicked', () => { 149 | // wrapper.find('button').at(1).simulate('click'); 150 | // expect(props.prop2).toEqual(1); 151 | // }); 152 | // }); 153 | 154 | // describe('AddFriend', () => { 155 | // // not complete yet 156 | // }); 157 | // }); 158 | -------------------------------------------------------------------------------- /playground/__tests__/reducerTest.js: -------------------------------------------------------------------------------- 1 | // import subject from '../client/state/reducers/mainReducer.js'; 2 | 3 | // describe('Reducer Tests', () => { 4 | // let state; 5 | 6 | // beforeEach(() => { 7 | // state = { 8 | // user: null, 9 | // login_state: null, 10 | // signup_state: null, 11 | // info: null, 12 | // messages: {}, 13 | // view: 'userpage', 14 | // user_info: null, 15 | // }; 16 | // }); 17 | 18 | // // - Default State 19 | // describe('default state', () => { 20 | // it('should return true when passed an undefined input', () => { 21 | // expect(subject(undefined, { type: undefined })).toEqual(state); 22 | // }); 23 | // }); 24 | 25 | // // unrecongizable inputs 26 | // describe('unrecognizable inputs', () => { 27 | // it('should not make any changes to state when passed unrecognizable input', () => { 28 | // const action = { type: 'sdfkjdsl;fjdsfkl;jdsf' }; 29 | // expect(subject(state, action)).toEqual(state); 30 | // }); 31 | // }); 32 | 33 | // // - LOGIN_STATE 34 | // describe('LOGIN_STATE', () => { 35 | // const action = { 36 | // type: 'LOGIN_STATE', 37 | // payload: 'true', 38 | // }; 39 | 40 | // it('should update login state based on action payload', () => { 41 | // const { login_state } = subject(state, action); 42 | // expect(login_state).toEqual('true'); 43 | // }); 44 | // }); 45 | 46 | // // - LOGIN 47 | // describe('LOGIN', () => { 48 | // const action = { 49 | // type: 'LOGIN', 50 | // payload: { 51 | // user: { _id: 1, username: 'test', language: 'english' }, 52 | // messages: { message: 'hello' }, 53 | // }, 54 | // }; 55 | 56 | // it('should update login_state to true', () => { 57 | // const { login_state } = subject(state, action); 58 | // expect(login_state).toEqual('true'); 59 | // }); 60 | 61 | // it('should update messages to payload messages', () => { 62 | // const { messages } = subject(state, action); 63 | // expect(messages).toEqual(action.payload.messages); 64 | // }); 65 | 66 | // it('should update state user info with payload', () => { 67 | // const { user } = subject(state, action); 68 | // expect(user).toEqual(action.payload.user); 69 | // }); 70 | // }); 71 | 72 | // // - SIGNUP_STATE 73 | // describe('SIGNUP_STATE', () => { 74 | // const action = { 75 | // type: 'SIGNUP_STATE', 76 | // payload: 'true', 77 | // }; 78 | // it('should update signup state based on action payload', () => { 79 | // const { signup_state } = subject(state, action); 80 | // expect(signup_state).toEqual('true'); 81 | // }); 82 | // }); 83 | 84 | // // - SIGNUP 85 | // describe('SIGNUP', () => { 86 | // const action = { 87 | // type: 'SIGNUP', 88 | // }; 89 | // it('should update login_state to true', () => { 90 | // const { login_state } = subject(state, action); 91 | // expect(login_state).toEqual(true); 92 | // }); 93 | // }); 94 | 95 | // // VIEW 96 | // describe('VIEW', () => { 97 | // const action = { 98 | // type: 'VIEW', 99 | // payload: 'newView', 100 | // }; 101 | 102 | // it('should update view based on action payload', () => { 103 | // const { view } = subject(state, action); 104 | // expect(view).toEqual(action.payload); 105 | // }); 106 | // }); 107 | 108 | // // - USER_INFO 109 | // describe('USER_INFO', () => { 110 | // const action = { 111 | // type: 'USER_INFO', 112 | // payload: 'new info', 113 | // }; 114 | // it('should update user info based on action payload', () => { 115 | // const { user_info } = subject(state, action); 116 | // expect(user_info).toEqual(action.payload); 117 | // }); 118 | // }); 119 | 120 | // // - UPDATE_MESSAGES 121 | // describe('UPDATE_MESSAGES', () => { 122 | // const action = { 123 | // type: 'UPDATE_MESSAGES', 124 | // payload: { test: 'new message' }, 125 | // }; 126 | // it('should update messages based on action payload', () => { 127 | // const { messages } = subject(state, action); 128 | // expect(messages).toEqual(action.payload); 129 | // }); 130 | // }); 131 | // }); 132 | -------------------------------------------------------------------------------- /playground/__tests__/routeTest.js: -------------------------------------------------------------------------------- 1 | // const request = require('supertest'); 2 | 3 | // const server = 'http://localhost:3001'; 4 | 5 | // describe('Route Integration', () => { 6 | // describe('/', () => { 7 | // describe('GET', () => { 8 | // xit('should respond with status 200', () => { 9 | // return request(server).get('/').expect(200); 10 | // }); 11 | // }); 12 | // }); 13 | 14 | // describe('Invalid Path', () => { 15 | // xit('should respond to invalid path with status 404', () => { 16 | // return request(server).get('/nonexistant').expect(404); 17 | // }); 18 | // }); 19 | 20 | // describe('/signup', () => { 21 | // describe('POST', () => { 22 | // xit('should respond with status 200', () => { 23 | // return request(server).post('/signup').expect(400); 24 | // }); 25 | // }); 26 | // }); 27 | 28 | // describe('/messages', () => { 29 | // describe('GET', () => { 30 | // xit('should respond with status 200', () => { 31 | // return request(server).get('/messages').expect(200); 32 | // }); 33 | // }); 34 | // }); 35 | 36 | // describe('/login', () => { 37 | // describe('POST', () => { 38 | // xit('should respond with status 200', () => { 39 | // return request(server).post('/login').expect(200); 40 | // }); 41 | // }); 42 | // }); 43 | 44 | // describe('/send', () => { 45 | // describe('POST', () => { 46 | // xit('should respond with status 200', () => { 47 | // return request(server).post('/send').expect(200); 48 | // }); 49 | // }); 50 | // }); 51 | // }); 52 | -------------------------------------------------------------------------------- /playground/__tests__/testToDo.md: -------------------------------------------------------------------------------- 1 | ## Route Testing 2 | - Get /signup 3 | - Get /messages 4 | - Get /signout 5 | - Get /signup 6 | - Get /login 7 | - Post /send 8 | 9 | ## End to End Testing 10 | - Expect log in page to render 11 | - Expect pass in valid credentials and redirect 12 | - Expect sign up with credentials and redirect 13 | - Expect log out to log out 14 | - Expect create new to open form 15 | - Form can input username to send to and message to send 16 | - Expect sent messages to populate with sent messages 17 | - Expect my messages to populate with received messages 18 | 19 | ## React Testing with Enzyme 20 | - 21 | 22 | ### Jest set up 23 | -------------------------------------------------------------------------------- /playground/client/App.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | // import * as ReactDOM from 'react-dom'; 3 | import { FC } from 'react'; 4 | import { Button, Theme, createStyles, makeStyles, Typography, Container, Hidden } from '@material-ui/core'; 5 | import { Element } from 'react-scroll'; 6 | import NavBar from './components/NavBar'; 7 | import FeaturesContainer from './containers/FeaturesContainer'; 8 | import DataDisplay from './DataDisplay'; 9 | import BarDisplay from './BarDisplay'; 10 | import GettingStarted from './components/GettingStarted'; 11 | import TeamContainer from './containers/TeamContainer'; 12 | import Footer from './components/Footer'; 13 | import theme from './theme'; 14 | 15 | const useStyles = makeStyles((theme: Theme) => createStyles({ 16 | container: { 17 | display: 'inline-block', 18 | alignItems: 'center', 19 | // justifyContent: 'flex-end', 20 | flexDirection: 'column', 21 | overflow: 'hidden', 22 | minWidth: '100vw', 23 | maxWidth: '100vw', 24 | }, 25 | topContainer: { 26 | display: 'flex', 27 | alignItems: 'center', 28 | // justifyContent: 'flex-end', 29 | flexDirection: 'column', 30 | minWidth: '100vw', 31 | maxWidth: '100vw', 32 | paddingLeft: '0', 33 | paddingRight: '0', 34 | paddingTop: '0', 35 | overflow: 'hidden', 36 | }, 37 | titleBox: { 38 | display: 'flex', 39 | flexDirection: 'column', 40 | justifyContent: 'center', 41 | alignItems: 'center', 42 | paddingTop: '5vh', 43 | paddingBottom: '5vh', 44 | minWidth: '100vw', 45 | maxWidth: '100vw', 46 | overflow: 'hidden', 47 | background: theme.palette.primary.main 48 | }, 49 | demoContainer: { 50 | minWidth: '70vw', 51 | maxWidth: '70vw' 52 | }, 53 | segment: { 54 | display: 'flex', 55 | flexDirection: 'column', 56 | justifyContent: 'center', 57 | alignItems: 'center', 58 | paddingTop: '5vh', 59 | paddingBottom: '5vh', 60 | }, 61 | bold: { 62 | fontWeight: 900, 63 | } 64 | })) 65 | 66 | const logo = require('./assets/logo.svg').default; 67 | // playground\client\assets\ks-logo-full.svg 68 | const App: FC = () => { 69 | const classes = useStyles(); 70 | return ( 71 | 72 | 73 | 74 | mainLogo 75 | 81 | Kafka Socks 82 | 83 | 89 | Making connecting WebSockets with Kafka as easy as npm install kafka-socks 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | {/* */} 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 |