├── .babelrc ├── .editorconfig ├── .eslintignore ├── .github ├── ISSUE_TEMPLATE │ └── feature_request.md ├── penguin.png └── workflows │ └── main.yml ├── .gitignore ├── .prettierrc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SUMMARY.md ├── clientConfig.ts ├── demo ├── .babelrc ├── .dockerignore ├── .eslintrc.json ├── .gitignore ├── Dockerfile ├── __tests__ │ └── backend │ │ ├── strategy.ts │ │ └── topic.ts ├── client │ ├── App.tsx │ ├── assets │ │ ├── ausar.jpeg │ │ ├── kushal.jpeg │ │ ├── timeo.jpeg │ │ └── ziyad.png │ ├── clientConfig.ts │ ├── components │ │ ├── Error.tsx │ │ ├── GettingStarted.tsx │ │ ├── GlobalNavBar.tsx │ │ ├── LandingBody.tsx │ │ ├── Message.tsx │ │ ├── ParticlesBackdrop.tsx │ │ ├── TeamMember.tsx │ │ └── Topic.tsx │ ├── containers │ │ ├── MainContainer.tsx │ │ ├── MessageErrorContainer.tsx │ │ ├── StrategyContainer.tsx │ │ ├── TeamContainer.tsx │ │ └── TopicsContainer.tsx │ ├── context │ │ ├── BackDropContext.tsx │ │ ├── ErrorContext.tsx │ │ ├── MessageContext.tsx │ │ └── TopicContext.tsx │ ├── index.html │ ├── index.tsx │ └── theme.ts ├── jest.config.js ├── package.json ├── server │ ├── app.ts │ ├── controllers │ │ ├── DLQController.ts │ │ ├── failfastController.ts │ │ ├── ignoreController.ts │ │ ├── kafkaController.ts │ │ ├── strategyController.ts │ │ └── topicsController.ts │ ├── routes │ │ ├── strategy.ts │ │ └── topic.ts │ └── server.ts ├── tsconfig.json ├── tsconfig.spec.json └── webpack.config.js ├── exampleBasic.ts ├── exampleDLQConsumer.ts ├── exampleDLQProducer.ts ├── exampleFailFast.ts ├── exampleIgnoreConsumer.ts ├── exampleIgnoreProducer.ts ├── jest.config.js ├── jest.config.ts ├── kafka-penguin ├── LICENSE ├── README.md ├── babel.config.js ├── dist │ ├── clientConfig.d.ts │ ├── clientConfig.js │ ├── index.d.ts │ └── index.js ├── package.json ├── src │ ├── clientConfig.ts │ ├── deadletterqueue.test.ts │ ├── failfast.test.ts │ ├── ignore.test.ts │ └── index.ts └── tsconfig.json ├── settings.json ├── strategies ├── README.md ├── dead-letter-queue.md ├── failfast.md └── strategies-ignore.md └── tsconfig.json /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env", 4 | "@babel/preset-react", 5 | "@babel/preset-typescript" 6 | ] 7 | } -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | indent_style = space 6 | indent_size = 2 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/penguin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-penguin/d8983545c5169da31318e44792271adee9fcffe0/.github/penguin.png -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | 2 | # Create name 3 | name: CI/CD 4 | 5 | # Trigger the workflow on push or pull request, 6 | # but only for the main branch 7 | on: [push, pull_request] 8 | # One main job: to run testing suite after every potential PR. 9 | # Enter series of terminal CMD's to test application 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | defaults: 14 | run: 15 | working-directory: ./demo 16 | strategy: 17 | matrix: 18 | node-version: [12.x] 19 | 20 | steps: 21 | #Checkout the source code of our github repo 22 | - uses: actions/checkout@v2 23 | - name: Set up Node 24 | uses: actions/setup-node@v1 25 | with: 26 | node-version: ${{ matrix.node-version }} 27 | - name: npm install, build and test 28 | run: | 29 | npm install 30 | npm test 31 | env: 32 | KAFKA_BOOTSTRAP_SERVER: ${{secrets.KAFKA_BOOTSTRAP_SERVER}} 33 | KAFKA_USERNAME: ${{secrets.KAFKA_USERNAME}} 34 | KAFKA_PASSWORD: ${{secrets.KAFKA_PASSWORD}} 35 | #npm run build --if-present 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | package-lock.json 4 | .env 5 | 6 | demo/node_modules 7 | demo/.env 8 | demo/package-lock.json 9 | .DS_Store -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "es5" 4 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | . 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, 4 | email, or any other method with the owners of this repository before making a change. 5 | 6 | Please note we have a code of conduct, please follow it in all your interactions with the project. 7 | 8 | ## Pull Request Process 9 | 10 | 1. Ensure any install or build dependencies are removed before the end of the layer when doing a 11 | build. 12 | 2. Update the README.md with details of changes to the interface, this includes new environment 13 | variables, exposed ports, useful file locations and container parameters. 14 | 3. Increase the version numbers in any examples files and the README.md to the new version that this 15 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 16 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 17 | do not have permission to do that, you may request the second reviewer to merge it for you. 18 | 19 | ## Code of Conduct 20 | 21 | ### Our Pledge 22 | 23 | In the interest of fostering an open and welcoming environment, we as 24 | contributors and maintainers pledge to making participation in our project and 25 | our community a harassment-free experience for everyone, regardless of age, body 26 | size, disability, ethnicity, gender identity and expression, level of experience, 27 | nationality, personal appearance, race, religion, or sexual identity and 28 | orientation. 29 | 30 | ### Our Standards 31 | 32 | Examples of behavior that contributes to creating a positive environment 33 | include: 34 | 35 | * Using welcoming and inclusive language 36 | * Being respectful of differing viewpoints and experiences 37 | * Gracefully accepting constructive criticism 38 | * Focusing on what is best for the community 39 | * Showing empathy towards other community members 40 | 41 | Examples of unacceptable behavior by participants include: 42 | 43 | * The use of sexualized language or imagery and unwelcome sexual attention or 44 | advances 45 | * Trolling, insulting/derogatory comments, and personal or political attacks 46 | * Public or private harassment 47 | * Publishing others' private information, such as a physical or electronic 48 | address, without explicit permission 49 | * Other conduct which could reasonably be considered inappropriate in a 50 | professional setting 51 | 52 | ### Our Responsibilities 53 | 54 | Project maintainers are responsible for clarifying the standards of acceptable 55 | behavior and are expected to take appropriate and fair corrective action in 56 | response to any instances of unacceptable behavior. 57 | 58 | Project maintainers have the right and responsibility to remove, edit, or 59 | reject comments, commits, code, wiki edits, issues, and other contributions 60 | that are not aligned to this Code of Conduct, or to ban temporarily or 61 | permanently any contributor for other behaviors that they deem inappropriate, 62 | threatening, offensive, or harmful. 63 | 64 | ### Scope 65 | 66 | This Code of Conduct applies both within project spaces and in public spaces 67 | when an individual is representing the project or its community. Examples of 68 | representing a project or community include using an official project e-mail 69 | address, posting via an official social media account, or acting as an appointed 70 | representative at an online or offline event. Representation of a project may be 71 | further defined and clarified by project maintainers. 72 | 73 | ### Enforcement 74 | 75 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 76 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 77 | complaints will be reviewed and investigated and will result in a response that 78 | is deemed necessary and appropriate to the circumstances. The project team is 79 | obligated to maintain confidentiality with regard to the reporter of an incident. 80 | Further details of specific enforcement policies may be posted separately. 81 | 82 | Project maintainers who do not follow or enforce the Code of Conduct in good 83 | faith may face temporary or permanent repercussions as determined by other 84 | members of the project's leadership. 85 | 86 | ### Attribution 87 | 88 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 89 | available at [http://contributor-covenant.org/version/1/4][version] 90 | 91 | [homepage]: http://contributor-covenant.org 92 | [version]: http://contributor-covenant.org/version/1/4/ 93 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kafka-Penguin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Home 2 |

3 | 4 |

5 |

Kafka-Penguin

 6 | 7 | 8 |

An easy-to-use, lightweight KafkaJS library for message re-processing.

9 | 10 |

11 | 12 | 13 | license 14 | NPM 15 | last commit 16 | Repo stars 17 |

18 | 19 | ## Table of Contents 20 | 21 | * [About](#About) 22 | * [Getting Started](#Getting-Started) 23 | * [Example](#Example) 24 | * [Strategies](#Strategies) 25 | * [Contributors](#Contributors) 26 | * [License](#License) 27 | 28 | ### About 29 | 30 | Kafka-Penguin is an easy-to-use, lightweight KafkaJS plugin for message re-processing. It provides developers with three strategies for setting up message re-processing: FailFast, Ignore, and Dead Letter Queue.‌ 31 | 32 | The package allows developers to build event-driven applications with dedicated "fail strategies" modeled after best practices in the field. This in turn allows developers to effectively address bugs in development and deploy more fault-tolerant systems in production.‌ 33 | 34 | This package is meant to work in conjunction with with KafkaJS. For more information on KafkaJS, check out [Getting Started with KafkaJS](https://kafka.js.org/docs/getting-started).‌ 35 | 36 | ### Getting Started 37 | 38 | Install Kafka-Penguin as an npm module and save it to your package.json file as a dependency: 39 | 40 | ```text 41 | npm install kafka-penguin 42 | ``` 43 | 44 | Once installed it can now be referenced by simply calling `require('kafka-penguin');`‌ 45 | 46 | ## Example 47 | 48 | All Kafka-Penguin needs is a KafkaJS client to run. Start by passing the client for your preferred strategy and Kafka-Penguin will create bespoke consumers, producers, and admins with built-in functionality to execute the chosen strategy. On the surface, you implement your application exactly as you would with KafkaJS. 49 | 50 | ```text 51 | /* eslint-disable no-console */ 52 | import { FailFast } from 'kafka-penguin'; 53 | 54 | const exampleClient = require('./clientConfig.ts'); 55 | 56 | // Set up the preferred strategy with a configured KafkaJS client 57 | const exampleStrategy = new FailFast(2, exampleClient); 58 | 59 | // Initialize a producer or consumer from the instance of the strategy 60 | const producer = exampleStrategy.producer(); 61 | 62 | const message = { 63 | topic: 'wrong-topic', 64 | messages: [ 65 | { 66 | key: 'hello', 67 | value: 'world', 68 | }, 69 | ], 70 | }; 71 | 72 | // Connect, Subscribe, Send, or Run virtually the same as with KafkaJS 73 | producer.connect() 74 | .then(() => console.log('Connected!')) 75 | // The chosen strategy executes under the hood, like in this send method 76 | .then(() => producer.send(message)) 77 | .catch((e: any) => console.log('error: ', e.message)); 78 | ``` 79 | 80 | ## Strategies 81 | 82 | Dive in deeper to any of the strategies for set up, execution, and implementation. 83 | 84 | ​[FailFast](https://kafkapenguin.gitbook.io/kafka-penguin/strategies/failfast) ​‌ 85 | 86 | ​[Ignore](https://kafkapenguin.gitbook.io/kafka-penguin/strategies/strategies-ignore)​‌ 87 | 88 | ​[Dead Letter Queue](https://kafkapenguin.gitbook.io/kafka-penguin/strategies/dead-letter-queue)​‌ 89 | 90 | ## **Contributors** 91 | 92 | ​[Ausar English](https://www.linkedin.com/in/ausarenglish) [@ausarenglish](https://github.com/ausarenglish)​‌ 93 | 94 | ​[Kushal Talele](https://www.linkedin.com/in/kushal-talele-29040820b/) [@ktrane1](https://github.com/ktrane1)​‌ 95 | 96 | ​[Timeo Williams](https://www.linkedin.com/in/timeowilliams/) [@timeowilliams](https://github.com/timeowilliams)​‌ 97 | 98 | ​[Ziyad El Baz](https://www.linkedin.com/in/ziyadelbaz) [@zelbaz946](https://github.com/zelbaz946)​‌ 99 | 100 | ### License 101 | 102 | This product is licensed under the MIT License - see the [LICENSE.md](https://github.com/oslabs-beta/kafka-penguin/blob/main/LICENSE) file for details.‌ 103 | 104 | This is an open source product. We are not affiliated nor endorsed by either the Apache Software Foundation or KafkaJS.‌ 105 | 106 | This product is accelerated by [OS Labs](https://opensourcelabs.io/). 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Table of contents 2 | 3 | * [Home](README.md) 4 | * [Strategies](strategies/README.md) 5 | * [FailFast](strategies/failfast.md) 6 | * [Ignore](strategies/strategies-ignore.md) 7 | * [Dead Letter Queue](strategies/dead-letter-queue.md) 8 | 9 | -------------------------------------------------------------------------------- /clientConfig.ts: -------------------------------------------------------------------------------- 1 | const { Kafka } = require('kafkajs'); 2 | require('dotenv').config(); 3 | 4 | // Create the client with the broker list 5 | const kafka = new Kafka({ 6 | clientId: 'fail-fast-producer', 7 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 8 | ssl: true, 9 | sasl: { 10 | mechanism: 'plain', 11 | username: process.env.KAFKA_USERNAME, 12 | password: process.env.KAFKA_PASSWORD, 13 | }, 14 | }); 15 | 16 | module.exports = kafka; 17 | -------------------------------------------------------------------------------- /demo/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-env", 4 | "@babel/preset-react", 5 | "@babel/preset-typescript" 6 | ] 7 | } -------------------------------------------------------------------------------- /demo/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /demo/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true, 5 | "node": true 6 | }, 7 | "extends": [ 8 | "plugin:react/recommended", 9 | "airbnb" 10 | ], 11 | "parser": "@typescript-eslint/parser", 12 | "parserOptions": { 13 | "ecmaFeatures": { 14 | "jsx": true, 15 | "tsx": true 16 | }, 17 | "ecmaVersion": 12, 18 | "sourceType": "module" 19 | }, 20 | "plugins": [ 21 | "react", 22 | "@typescript-eslint" 23 | ], 24 | "rules": { 25 | "react/jsx-filename-extension": [2, { "extensions": [".js", ".jsx", ".ts", ".tsx"] }], 26 | "import/extensions": "off", 27 | "no-use-before-define": "off", 28 | "@typescript-eslint/no-use-before-define": ["error"] 29 | }, 30 | "settings": { 31 | "import/resolver": { 32 | "node": { 33 | "extensions": [".js", ".jsx", ".ts", ".tsx"] 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /demo/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12.18.3 2 | 3 | RUN npm i -g webpack 4 | WORKDIR /usr/src/app 5 | COPY . /usr/src/app 6 | RUN npm install 7 | RUN npm run build2 8 | # RUN npm run tsc 9 | EXPOSE 3000 10 | # ENTRYPOINT [ "nodemon", "./server/server.ts" ] 11 | CMD ["npm", "start"] 12 | -------------------------------------------------------------------------------- /demo/__tests__/backend/strategy.ts: -------------------------------------------------------------------------------- 1 | import request from 'supertest'; 2 | import server from '../../server/app'; 3 | 4 | describe('Strategy tests', () => { 5 | const messageValid = { 6 | topic: 'test2', 7 | message: 'Hello world', 8 | retries: 4, 9 | faults: 2, 10 | }; 11 | const messageInvalid = { 12 | topic: 'wrongtopic', 13 | message: 'Hello world', 14 | retries: 4, 15 | }; 16 | beforeAll((done: () => void) => { 17 | done(); 18 | }); 19 | afterAll((done: () => void) => { 20 | done(); 21 | }); 22 | 23 | describe('Failfast tests', () => { 24 | it('Expect valid response from valid message', (done: { (): void; (err: Error, res: request.Response): void; }) => { 25 | request(server) 26 | .post('/strategy/failfast') 27 | .send(messageValid) 28 | .expect('Content-Type', 'application/json; charset=utf-8') 29 | .expect(200) 30 | .expect((res) => { 31 | expect(res.body.length).toEqual(1); 32 | expect(typeof res.body[0] === 'string').toBe(true); 33 | expect(res.body[0]).toBe('kafka-penguin: Message produced successfully'); 34 | done(); 35 | }) 36 | .end(done); 37 | }); 38 | 39 | it('Expect error log in response from invalid message', (done: { (): void; (err: any, res: request.Response): void; }) => { 40 | request(server) 41 | .post('/strategy/failfast') 42 | .send(messageInvalid) 43 | .expect('Content-Type', 'application/json; charset=utf-8') 44 | .expect(200) 45 | .expect((res) => { 46 | expect(res.body.length).toEqual(messageInvalid.retries + 1); 47 | expect(res.body[messageInvalid.retries]).toBe(`kafka-penguin: FailFast stopped producer after ${messageInvalid.retries} times!`); 48 | res.body.forEach((error: any) => { 49 | expect(typeof error === 'string').toBe(true); 50 | }); 51 | done(); 52 | }) 53 | .end(done); 54 | }); 55 | }); 56 | 57 | describe('DLQ tests', () => { 58 | describe('DLQ Consumer: consuming messages from existent topic', () => { 59 | it('Expect valid response without interruption of data flow', (done: { (): void; (err: Error, res: request.Response): void; }) => { 60 | request(server) 61 | .post('/strategy/dlq') 62 | .send(messageValid) 63 | .expect(200) 64 | .expect((res) => { 65 | const messages = res.body.slice(0, res.body.length - 1); 66 | expect(messages.length).toEqual(messageValid.retries - messageValid.faults); 67 | messages.forEach((message: any) => { 68 | //expect(message).toEqual('Hello world'); 69 | expect(typeof message).toBe('string'); 70 | }); 71 | expect(res.body[messageValid.retries - messageValid.faults]).toContain('kafka-penguin'); 72 | expect(res.body[messageValid.retries - messageValid.faults]).toContain('2'); 73 | expect(res.body[messageValid.retries - messageValid.faults]).toContain('test2.deadLetterQueue'); 74 | done(); 75 | }) 76 | .end(done); 77 | }); 78 | }); 79 | describe('DLQ Producer: non-existent and existent topics', () => { 80 | it('Expect kafka-penguin error while posting to non-existent topic', (done: { (): void; (err: any, res: request.Response): void; }) => { 81 | request(server) 82 | .post('/strategy/dlq') 83 | .send(messageInvalid) 84 | .expect(300) 85 | .expect((res) => { 86 | const error = res.body[0]; 87 | expect(res.body.length).toBe(1); 88 | expect(typeof error).toBe('string'); 89 | done(); 90 | }) 91 | .end(done); 92 | }); 93 | }); 94 | }); 95 | }); 96 | -------------------------------------------------------------------------------- /demo/__tests__/backend/topic.ts: -------------------------------------------------------------------------------- 1 | import request from 'supertest'; 2 | import dotenv from 'dotenv'; 3 | import server from '../../server/app'; 4 | 5 | dotenv.config(); 6 | 7 | describe('Static router tests', () => { 8 | const userDetails = { 9 | brokers: process.env.KAFKA_BOOTSTRAP_SERVER, 10 | username: process.env.KAFKA_USERNAME, 11 | password: process.env.KAFKA_PASSWORD, 12 | }; 13 | 14 | it('Returns array of objects containing topic names and partitions', (done) => { 15 | request(server) 16 | .post('/topic/getTopics') 17 | .send(JSON.stringify(userDetails)) 18 | .expect('Content-Type', 'application/json; charset=utf-8') 19 | .expect(200) 20 | .expect((res) => { 21 | res.body.topics.forEach((topic) => { 22 | expect(topic.hasOwnProperty('name')).toBe(true); 23 | expect(topic.hasOwnProperty('partitions')).toBe(true); 24 | expect(typeof topic.name === 'string').toBe(true); 25 | expect(Array.isArray(topic.partitions)).toBe(true); 26 | }); 27 | done(); 28 | }) 29 | .end(done); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /demo/client/App.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | createStyles, makeStyles, Typography, Container, 5 | } from '@material-ui/core'; 6 | import { Element } from 'react-scroll'; 7 | import MainContainer from './containers/MainContainer'; 8 | import { BackdropProvider } from './context/BackDropContext'; 9 | import GlobalNavBar from './components/GlobalNavBar'; 10 | import LandingBody from './components/LandingBody'; 11 | import TeamContainer from './containers/TeamContainer'; 12 | import GettingStarted from './components/GettingStarted'; 13 | import ParticlesBackdrop from './components/ParticlesBackdrop'; 14 | 15 | const useStyles = makeStyles(() => createStyles({ 16 | container: { 17 | display: 'flex', 18 | alignItems: 'center', 19 | justifyContent: 'flex-end', 20 | flexDirection: 'column', 21 | }, 22 | titleBox: { 23 | display: 'flex', 24 | flexDirection: 'column', 25 | justifyContent: 'center', 26 | alignItems: 'center', 27 | paddingTop: '30vh', 28 | paddingBottom: '15vh', 29 | }, 30 | segment: { 31 | display: 'flex', 32 | flexDirection: 'column', 33 | justifyContent: 'center', 34 | alignItems: 'center', 35 | paddingTop: '5vh', 36 | paddingBottom: '5vh', 37 | }, 38 | })); 39 | 40 | const App: FC = () => { 41 | const classes = useStyles(); 42 | return ( 43 | <> 44 | 45 | 46 | 47 | 48 | 54 | kafka-penguin 55 | 56 | mainLogo 57 | 63 | ERROR HANDLING LIBRARY FOR KAFKAJS 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | ); 83 | }; 84 | 85 | export default App; 86 | -------------------------------------------------------------------------------- /demo/client/assets/ausar.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-penguin/d8983545c5169da31318e44792271adee9fcffe0/demo/client/assets/ausar.jpeg -------------------------------------------------------------------------------- /demo/client/assets/kushal.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-penguin/d8983545c5169da31318e44792271adee9fcffe0/demo/client/assets/kushal.jpeg -------------------------------------------------------------------------------- /demo/client/assets/timeo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-penguin/d8983545c5169da31318e44792271adee9fcffe0/demo/client/assets/timeo.jpeg -------------------------------------------------------------------------------- /demo/client/assets/ziyad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/kafka-penguin/d8983545c5169da31318e44792271adee9fcffe0/demo/client/assets/ziyad.png -------------------------------------------------------------------------------- /demo/client/clientConfig.ts: -------------------------------------------------------------------------------- 1 | import { Kafka } from 'kafkajs'; 2 | 3 | require('dotenv').config(); 4 | 5 | // Create the client with the broker list 6 | const kafka = new Kafka({ 7 | clientId: 'fail-fast-producer', 8 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 9 | ssl: true, 10 | sasl: { 11 | mechanism: 'plain', 12 | username: process.env.KAFKA_USERNAME, 13 | password: process.env.KAFKA_PASSWORD, 14 | }, 15 | }); 16 | 17 | module.exports = kafka; 18 | -------------------------------------------------------------------------------- /demo/client/components/Error.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { createStyles, makeStyles, Typography } from '@material-ui/core'; 4 | 5 | const useStyles = makeStyles(() => createStyles({ 6 | root: { 7 | padding: '0em 1em 1em 1em', 8 | height: 'auto', 9 | }, 10 | })); 11 | 12 | type Props = { 13 | errorMessage: string, 14 | } 15 | 16 | const Error: FC = ({ errorMessage }: Props) => { 17 | const classes = useStyles(); 18 | return ( 19 | 20 | {errorMessage} 21 | 22 | ); 23 | }; 24 | 25 | export default Error; 26 | -------------------------------------------------------------------------------- /demo/client/components/GettingStarted.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { 3 | Container, Typography, Divider, makeStyles, createStyles, Button, Theme, 4 | } from '@material-ui/core'; 5 | import SyntaxHighlighter from 'react-syntax-highlighter'; 6 | import { materialLight } from 'react-syntax-highlighter/dist/esm/styles/prism'; 7 | import GetAppRoundedIcon from '@material-ui/icons/GetAppRounded'; 8 | 9 | const GettingStarted: FC = () => { 10 | const useStyles = makeStyles((theme: Theme) => createStyles({ 11 | main: { 12 | display: 'flex', 13 | flexDirection: 'column', 14 | justifyContent: 'space-between', 15 | alignItems: 'center', 16 | alignContent: 'center', 17 | marginTop: '5vh', 18 | backgroundColor: theme.palette.background.default, 19 | paddingTop: '3vh', 20 | paddingBottom: '3vh', 21 | width: 'auto', 22 | }, 23 | button: { 24 | margin: '1rem 1rem 1rem 1rem', 25 | }, 26 | wrapper: { 27 | display: 'flex', 28 | flexDirection: 'column', 29 | justifyItems: 'center', 30 | alignItems: 'center', 31 | }, 32 | buttonsContainer: { 33 | display: 'flex', 34 | justifyContent: 'center', 35 | }, 36 | divider: { 37 | height: '1px', 38 | }, 39 | icon: { 40 | color: theme.palette.secondary.light, 41 | fontSize: '5vh', 42 | padding: '2vh', 43 | }, 44 | })); 45 | const installCode = 'npm install kafka-penguin'; 46 | const importCode = 'import { DeadLetterQueue } from \'kafka-penguin\';'; 47 | const classes = useStyles(); 48 | return ( 49 | 50 | 51 | GETTING STARTED 52 | 53 | 54 | 55 | 56 | Simply install from npm and import into your project 57 | 58 | 59 | 63 | { installCode } 64 | 65 | 69 | { importCode } 70 | 71 | 72 | 80 | 88 | 89 | 90 | 91 | ); 92 | }; 93 | 94 | export default GettingStarted; 95 | -------------------------------------------------------------------------------- /demo/client/components/GlobalNavBar.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | createStyles, makeStyles, withStyles, AppBar, Toolbar, IconButton, Button, Icon, 5 | } from '@material-ui/core'; 6 | import { Link } from 'react-scroll'; 7 | 8 | const useStyles = makeStyles(() => createStyles({ 9 | root: { 10 | width: '100vw', 11 | padding: 0, 12 | }, 13 | landingButtons: { 14 | display: 'flex', 15 | justifyContent: 'flex-end', 16 | }, 17 | logo: { 18 | alignSelf: 'flex-start', 19 | }, 20 | button: { 21 | margin: '1rem 1rem 1rem 1rem', 22 | }, 23 | })); 24 | 25 | const GlobalCss = withStyles({ 26 | '@global': { 27 | 'html, body': { 28 | margin: 0, 29 | padding: 0, 30 | }, 31 | }, 32 | })(() => null); 33 | 34 | const GlobalNavBar: FC = () => { 35 | const classes = useStyles(); 36 | return ( 37 |
38 | 39 | 40 | 41 | 42 | 49 | navLogo 50 | 51 | 52 | 64 | 76 | 88 | 100 | {/* 107 | */} 114 | 115 | 116 |
117 | ); 118 | }; 119 | 120 | export default GlobalNavBar; 121 | -------------------------------------------------------------------------------- /demo/client/components/LandingBody.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { 3 | makeStyles, createStyles, Container, Divider, Theme, Typography, 4 | } from '@material-ui/core'; 5 | import ErrorOutlineRoundedIcon from '@material-ui/icons/ErrorOutlineRounded'; 6 | import FitnessCenterRoundedIcon from '@material-ui/icons/FitnessCenterRounded'; 7 | import AccessibilityNewRoundedIcon from '@material-ui/icons/AccessibilityNewRounded'; 8 | import SyntaxHighlighter from 'react-syntax-highlighter'; 9 | import { materialLight } from 'react-syntax-highlighter/dist/esm/styles/prism'; 10 | 11 | const useStyles = makeStyles((theme: Theme) => createStyles({ 12 | button: { 13 | margin: '1rem 1rem 1rem 1rem', 14 | }, 15 | container: { 16 | display: 'flex', 17 | flexDirection: 'column', 18 | 19 | }, 20 | inner: { 21 | display: 'flex', 22 | alignContent: 'center', 23 | justifyContent: 'space-around', 24 | alignItems: 'center', 25 | marginTop: '3vh', 26 | marginBottom: '5vh', 27 | paddingTop: '5vh', 28 | paddingBottom: '5vh', 29 | background: theme.palette.background.default, 30 | }, 31 | icon: { 32 | fontSize: '7vh', 33 | color: theme.palette.primary.light, 34 | padding: '2vh', 35 | }, 36 | })); 37 | 38 | const lightWeightCode = `import { FailFast } from 'kafka-penguin'; 39 | const failfast = new FailFast(2, kafkaClient);`; 40 | 41 | const easyToUseCode = `const producer = failfast.producer(); 42 | producer.connect() 43 | .then(producer.send(...))`; 44 | 45 | const errorHandlingCode = `import { 46 | DeadLetterQueue, 47 | FailFast, 48 | Ignore 49 | } from 'kafka-penguin';`; 50 | 51 | const LandingBody: FC = () => { 52 | const classes = useStyles(); 53 | return ( 54 | <> 55 | 56 | 57 | FEATURES 58 | 59 | 60 | 61 | 62 | 63 |
64 | LIGHT-WEIGHT 65 | 66 | Minimal Configuration 67 | 68 | 69 | A working KafkaJS client and a callback that returns a boolean. 70 | 71 | 72 | That’s all it takes to implement a strategy. 73 | 74 | 75 | 79 | {lightWeightCode} 80 | 81 | 82 | 83 | 87 | { easyToUseCode } 88 | 89 | 90 | 91 |
92 | EASY-TO-USE 93 | 94 | Plug-and-Play 95 | 96 | 97 | Add one line of code on top of your existing 98 |
99 | implementation and keep using KafkaJS as normal. 100 | 101 | 102 | 103 | 104 | 105 | 106 |
107 | ERROR-HANDLING 108 | 109 | Common, Programmable Strategies 110 | 111 | 112 | Choose from some of the most widely used strategies. 113 | 114 | 115 | Program them to fit your application’s logic. 116 | 117 | 118 | 122 | { errorHandlingCode } 123 | 124 | 125 | 126 | 127 | ); 128 | }; 129 | 130 | export default LandingBody; 131 | -------------------------------------------------------------------------------- /demo/client/components/Message.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | Typography, TextField, makeStyles, createStyles, Slider, 5 | } from '@material-ui/core'; 6 | import { useMesageUpdateContext } from '../context/MessageContext'; 7 | 8 | const useStyles = makeStyles(() => createStyles({ 9 | root: { 10 | display: 'flex', 11 | flexDirection: 'column', 12 | alignItems: 'center', 13 | justifyContent: 'center', 14 | '& .MuiTextField-root': { 15 | // margin: theme.spacing(1), 16 | width: '25vw', 17 | minWidth: '20vw', 18 | marginTop: '0', 19 | }, 20 | }, 21 | textfield: { 22 | paddingBottom: '1rem', 23 | }, 24 | })); 25 | 26 | const Message: FC = () => { 27 | const classes = useStyles(); 28 | const messageUpdate = useMesageUpdateContext(); 29 | return ( 30 |
35 | { 43 | messageUpdate.changeTopic(event.target.value); 44 | }} 45 | /> 46 | { 54 | messageUpdate.changeMessage(event.target.value); 55 | }} 56 | /> 57 | { 60 | messageUpdate.changeRetries(value); 61 | }} 62 | aria-labelledby="discrete-slider" 63 | valueLabelDisplay="auto" 64 | step={1} 65 | marks 66 | min={1} 67 | max={5} 68 | /> 69 | 75 | Set retries / Repeats 76 | 77 | { 80 | messageUpdate.changeFaults(value); 81 | }} 82 | aria-labelledby="discrete-slider" 83 | valueLabelDisplay="auto" 84 | step={1} 85 | marks 86 | min={1} 87 | max={5} 88 | /> 89 | 95 | Faults for DLQ and Ignore 96 | 97 | 98 | ); 99 | }; 100 | 101 | export default Message; 102 | -------------------------------------------------------------------------------- /demo/client/components/ParticlesBackdrop.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import Particles from 'react-tsparticles'; 4 | 5 | const ParticlesBackdrop: FC = () => ( 6 |
15 | 95 |
96 | ); 97 | 98 | export default ParticlesBackdrop; 99 | -------------------------------------------------------------------------------- /demo/client/components/TeamMember.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { 3 | makeStyles, createStyles, Card, CardContent, Typography, 4 | } from '@material-ui/core'; 5 | import GitHubIcon from '@material-ui/icons/GitHub'; 6 | import LinkedInIcon from '@material-ui/icons/LinkedIn'; 7 | 8 | interface Props { 9 | details: { 10 | name: string, 11 | linkedIn: string, 12 | github: string, 13 | photo: string 14 | } 15 | } 16 | 17 | const TeamMember: FC = ({ 18 | details, 19 | }: Props) => { 20 | const useStyles = makeStyles(() => createStyles({ 21 | root: { 22 | display: 'flex', 23 | alignItems: 'center', 24 | alignContent: 'center', 25 | justifyContent: 'center', 26 | padding: '1rem 1rem 1rem 1rem', 27 | margin: '0rem 2rem 2rem 2rem', 28 | minWidth: '15vw', 29 | maxWidth: '15vw', 30 | }, 31 | })); 32 | 33 | const classes = useStyles(); 34 | 35 | return ( 36 | 37 | 38 | 39 | profilePhoto 48 | 49 | 50 | {details.name} 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | ); 63 | }; 64 | 65 | export default TeamMember; 66 | -------------------------------------------------------------------------------- /demo/client/components/Topic.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | Card, CardContent, Typography, createStyles, makeStyles, 5 | } from '@material-ui/core'; 6 | 7 | type Props = { 8 | topicInfo: { 9 | name: string, 10 | partitions: number 11 | }, 12 | id: number, 13 | } 14 | 15 | const Topic: FC = ({ topicInfo, id }: Props) => { 16 | const useStyles = makeStyles(() => createStyles({ 17 | root: { 18 | display: 'flex', 19 | alignItems: 'center', 20 | justifyContent: 'center', 21 | margin: '1rem 1rem 1rem 1rem', 22 | }, 23 | })); 24 | 25 | const classes = useStyles(); 26 | 27 | return ( 28 | 29 | 30 | 31 | {topicInfo.name} 32 | 33 | 34 | {`partitions: ${topicInfo.partitions}`} 35 | 36 | 37 | 38 | ); 39 | }; 40 | 41 | export default Topic; 42 | -------------------------------------------------------------------------------- /demo/client/containers/MainContainer.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | Container, createStyles, makeStyles, Backdrop, CircularProgress, Theme, Typography, Divider, 5 | } from '@material-ui/core'; 6 | import TopicsContainer from './TopicsContainer'; 7 | import StrategyContainer from './StrategyContainer'; 8 | import MessageErrorContainer from './MessageErrorContainer'; 9 | import { TopicsProvider } from '../context/TopicContext'; 10 | import { MessageProvider } from '../context/MessageContext'; 11 | import { ErrorProvider } from '../context/ErrorContext'; 12 | import { useBackdropContext, useBackdropUpdateContext } from '../context/BackDropContext'; 13 | 14 | const MainContainer: FC = () => { 15 | const useStyles = makeStyles((theme: Theme) => createStyles({ 16 | wrapper: { 17 | display: 'flex', 18 | flexDirection: 'column', 19 | justifyContent: 'center', 20 | alignItems: 'center', 21 | alignContent: 'center', 22 | }, 23 | container: { 24 | display: 'flex', 25 | alignItems: 'center', 26 | justifyContent: 'center', 27 | alignContent: 'center', 28 | flexDirection: 'column', 29 | }, 30 | button: { 31 | margin: '1rem 1rem 1rem 1rem', 32 | }, 33 | backdrop: { 34 | zIndex: theme.zIndex.drawer + 1, 35 | color: '#fff', 36 | }, 37 | divider: { 38 | height: '1px', 39 | }, 40 | })); 41 | 42 | const classes = useStyles(); 43 | const backdropContext = useBackdropContext(); 44 | const backdropUpdate = useBackdropUpdateContext(); 45 | 46 | return ( 47 | 48 | 49 | DEMO 50 | 51 | 52 | 53 | Trigger errors to see kafka-penguin in action 54 | 55 | 56 | Errors can be triggered by publishing to a non-existent 57 | topic or prescribing a number of faults. 58 | 59 | 60 | Enter in a topic, message and choose your strategy to test with our sample cluster. 61 | 62 | 63 | Load demo topics 64 |
65 | Publish either to existent topic or non-existent topic 66 |
67 | Select retries 68 |
69 | Execute strategy 70 | 71 | 72 | * DLQ and Ignore require repeats to be greater than faults 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 94 | 95 | 96 | 97 | ); 98 | }; 99 | 100 | export default MainContainer; 101 | -------------------------------------------------------------------------------- /demo/client/containers/MessageErrorContainer.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | Container, Typography, Theme, Paper, createStyles, makeStyles, 5 | } from '@material-ui/core'; 6 | import Error from '../components/Error'; 7 | import Message from '../components/Message'; 8 | import { useErrorContext } from '../context/ErrorContext'; 9 | 10 | const useStyles = makeStyles((theme: Theme) => createStyles({ 11 | paper: { 12 | display: 'flex', 13 | minHeight: '80%', 14 | '& > *': { 15 | margin: theme.spacing(1), 16 | }, 17 | }, 18 | containerVertical: { 19 | display: 'flex', 20 | flexDirection: 'column', 21 | justifyContent: 'flex-start', 22 | paddingTop: '3vh', 23 | // minWidth: '10vw' 24 | }, 25 | containerHorizontal: { 26 | display: 'flex', 27 | justifyContent: 'center', 28 | flexGrow: 1, 29 | // flexWrap: 'wrap' 30 | }, 31 | })); 32 | 33 | const MessageErrorContainer: FC = () => { 34 | const classes = useStyles(); 35 | const updateError = useErrorContext(); 36 | 37 | const errors = updateError.map((error: string) => ( 38 | 39 | )); 40 | return ( 41 | 42 | 43 | 49 | Publish 50 | 51 | 52 | 53 | 54 | 60 | Log 61 | 62 | 66 | 70 | {errors} 71 | 72 | 73 | 74 | 75 | ); 76 | }; 77 | 78 | export default MessageErrorContainer; 79 | -------------------------------------------------------------------------------- /demo/client/containers/StrategyContainer.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | createStyles, makeStyles, Button, Container, 5 | } from '@material-ui/core'; 6 | import { useErrorUpdateContext } from '../context/ErrorContext'; 7 | import { useMessageContext } from '../context/MessageContext'; 8 | import { useBackdropUpdateContext } from '../context/BackDropContext'; 9 | 10 | const useStyles = makeStyles(() => createStyles({ 11 | container: { 12 | display: 'flex', 13 | alignItems: 'center', 14 | justifyContent: 'center', 15 | alignContent: 'space-between', 16 | }, 17 | button: { 18 | margin: '1rem 1rem 1rem 1rem', 19 | }, 20 | })); 21 | 22 | const StrategyContainer: FC = () => { 23 | // After the user clicks on the button FF or DLQ, 24 | // we will render the associated strategy data 25 | // to them in a div below 26 | const message = useMessageContext(); 27 | const handleClicks = useErrorUpdateContext(); 28 | const backdropUpdate = useBackdropUpdateContext(); 29 | const classes = useStyles(); 30 | 31 | return ( 32 | 33 | 44 | 57 | 70 | 71 | ); 72 | }; 73 | export default StrategyContainer; 74 | -------------------------------------------------------------------------------- /demo/client/containers/TeamContainer.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from 'react'; 2 | import { 3 | makeStyles, createStyles, Container, Divider, Typography, 4 | } from '@material-ui/core'; 5 | import TeamMember from '../components/TeamMember'; 6 | 7 | const TeamContainer: FC = () => { 8 | const useStyles = makeStyles(() => createStyles({ 9 | containerHorizontal: { 10 | // height: '80vh', 11 | paddingTop: '10vh', 12 | display: 'flex', 13 | flexDirection: 'column', 14 | alignItems: 'center', 15 | alignContent: 'center', 16 | justifyContent: 'center', 17 | }, 18 | containerVertical: { 19 | display: 'flex', 20 | alignItems: 'center', 21 | justifyContent: 'center', 22 | 23 | }, 24 | })); 25 | const timeo = { 26 | name: 'Timeo Williams', 27 | linkedIn: 'https://www.linkedin.com/in/timeowilliams/', 28 | github: 'https://github.com/timeowilliams', 29 | photo: '/assets/timeo.jpeg', 30 | }; 31 | const ausar = { 32 | name: 'Ausar English', 33 | linkedIn: 'https://www.linkedin.com/in/ausarenglish/', 34 | github: 'https://github.com/ausarenglish', 35 | photo: '/assets/ausar.jpeg', 36 | }; 37 | const ziyad = { 38 | name: 'Ziyad Elbaz', 39 | linkedIn: 'https://www.linkedin.com/in/ziyadelbaz/', 40 | github: 'https://github.com/zelbaz946', 41 | photo: '/assets/ziyad.png', 42 | }; 43 | const kushal = { 44 | name: 'Kushal Talele', 45 | linkedIn: 'https://www.linkedin.com/in/kushaltalele/', 46 | github: 'https://github.com/ktrane1', 47 | photo: '/assets/kushal.jpeg', 48 | }; 49 | 50 | const classes = useStyles(); 51 | 52 | return ( 53 | 54 | 55 | TEAM 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | ); 70 | }; 71 | 72 | export default TeamContainer; 73 | -------------------------------------------------------------------------------- /demo/client/containers/TopicsContainer.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { FC } from 'react'; 3 | import { 4 | createStyles, makeStyles, Container, Button, 5 | } from '@material-ui/core'; 6 | import Topic from '../components/Topic'; 7 | import { useTopicsContext, useTopicsContextUpdate } from '../context/TopicContext'; 8 | import { useBackdropUpdateContext } from '../context/BackDropContext'; 9 | 10 | const useStyles = makeStyles(() => createStyles({ 11 | topicsContainer: { 12 | display: 'flex', 13 | flexWrap: 'wrap', 14 | alignItems: 'center', 15 | justifyContent: 'center', 16 | }, 17 | button: { 18 | margin: '1rem 1rem 1rem 1rem', 19 | }, 20 | })); 21 | 22 | const TopicsContainer: FC = () => { 23 | const backdropUpdate = useBackdropUpdateContext(); 24 | const topicsUpdate = useTopicsContextUpdate(); 25 | const topics = useTopicsContext(); 26 | const topicsMapped = topics.map((topicInfo, i) => ); 27 | 28 | const classes = useStyles(); 29 | 30 | return ( 31 | 32 | 33 | 44 | 54 | 55 | 56 | {topicsMapped} 57 | 58 | 59 | ); 60 | }; 61 | 62 | export default TopicsContainer; 63 | -------------------------------------------------------------------------------- /demo/client/context/BackDropContext.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { 3 | useState, useContext, FC, createContext, 4 | } from 'react'; 5 | 6 | const BackdropContext = createContext(null); 7 | const BackdropUpdateContext = createContext(null); 8 | 9 | const useBackdropContext = () => useContext(BackdropContext); 10 | 11 | const useBackdropUpdateContext = () => useContext(BackdropUpdateContext); 12 | 13 | interface Props { 14 | children: any 15 | } 16 | 17 | const BackdropProvider: FC = ({ children } : Props) => { 18 | const [open, setOpen] = useState(false); 19 | 20 | const handleClose = () => { 21 | setOpen(false); 22 | }; 23 | 24 | const handleToggle = () => { 25 | setOpen(!open); 26 | }; 27 | 28 | return ( 29 | 30 | 38 | {children} 39 | 40 | 41 | ); 42 | }; 43 | 44 | export { BackdropProvider, useBackdropContext, useBackdropUpdateContext }; 45 | -------------------------------------------------------------------------------- /demo/client/context/ErrorContext.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { 3 | useState, useContext, FC, createContext, useEffect, 4 | } from 'react'; 5 | import { useBackdropUpdateContext } from './BackDropContext'; 6 | 7 | const ErrorContext = createContext(null); 8 | const ErrorUpdateContext = createContext(null); 9 | 10 | const useErrorContext = () => useContext(ErrorContext); 11 | 12 | const useErrorUpdateContext = () => useContext(ErrorUpdateContext); 13 | 14 | interface Props { 15 | children: any 16 | } 17 | 18 | const ErrorProvider: FC = ({ children } : Props) => { 19 | const [error, changeError] = useState([]); 20 | 21 | const backdropUpdate = useBackdropUpdateContext(); 22 | // UPDATE BACKDROP STATE 23 | useEffect(() => { 24 | backdropUpdate.handleClose(); 25 | }, [error]); 26 | // FAILFAST POST REQUEST // 27 | const handleFailFast = (input: { 28 | message: string, 29 | topic: string, 30 | retries: number 31 | }) => { 32 | const { message, topic, retries } = input; 33 | if (!topic || !message) return; 34 | 35 | fetch('/strategy/failfast', { 36 | method: 'POST', 37 | headers: { 'Content-Type': 'Application/JSON' }, 38 | body: JSON.stringify({ topic, message, retries }), 39 | }) 40 | .then((data) => data.json()) 41 | .then((errors) => { 42 | changeError(errors); 43 | }); 44 | }; 45 | // DEAD LETTER QUEUE POST REQUEST // 46 | const handleDLQ = (input: { 47 | message: string; 48 | topic: string; 49 | retries: number; 50 | faults: number; 51 | }) => { 52 | const { 53 | message, topic, retries, faults, 54 | } = input; 55 | if (!topic || !message || faults >= retries) return; 56 | 57 | fetch('/strategy/dlq', { 58 | method: 'POST', 59 | headers: { 'Content-Type': 'Application/JSON' }, 60 | body: JSON.stringify({ 61 | topic, message, retries, faults, 62 | }), 63 | }) 64 | .then((res) => res.json()) 65 | .then((messages) => { 66 | changeError(messages); 67 | }) 68 | .catch((e) => console.log(e)); 69 | }; 70 | 71 | const handleIgnore = (input: { 72 | message: string; 73 | topic: string; 74 | retries: number; 75 | faults: number; 76 | }) => { 77 | const { 78 | message, topic, retries, faults, 79 | } = input; 80 | if (!topic || !message || faults >= retries) return; 81 | 82 | fetch('/strategy/ignore', { 83 | method: 'POST', 84 | headers: { 'Content-Type': 'Application/JSON' }, 85 | body: JSON.stringify({ 86 | topic, message, retries, faults, 87 | }), 88 | }) 89 | .then((res) => res.json()) 90 | .then((messages) => { 91 | changeError(messages); 92 | }) 93 | .catch((e) => console.log(e)); 94 | }; 95 | 96 | return ( 97 | 98 | 107 | {children} 108 | 109 | 110 | ); 111 | }; 112 | 113 | export { ErrorProvider, useErrorContext, useErrorUpdateContext }; 114 | -------------------------------------------------------------------------------- /demo/client/context/MessageContext.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { 3 | useState, useContext, FC, createContext, 4 | } from 'react'; 5 | 6 | const MessageContext = createContext(null); 7 | const MessageUpdateContext = createContext(null); 8 | 9 | const useMessageContext = () => useContext(MessageContext); 10 | 11 | const useMesageUpdateContext = () => useContext(MessageUpdateContext); 12 | 13 | interface Props { 14 | children: any 15 | } 16 | 17 | const MessageProvider: FC = ({ children } : Props) => { 18 | const [message, changeMessage] = useState(''); 19 | const [topic, changeTopic] = useState(''); 20 | const [retries, changeRetries] = useState(2); 21 | const [faults, changeFaults] = useState(2); 22 | 23 | return ( 24 | 34 | 44 | {children} 45 | 46 | 47 | ); 48 | }; 49 | 50 | export { MessageProvider, useMessageContext, useMesageUpdateContext }; 51 | -------------------------------------------------------------------------------- /demo/client/context/TopicContext.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { 3 | useState, useContext, FC, createContext, useEffect, 4 | } from 'react'; 5 | import { useBackdropUpdateContext } from './BackDropContext'; 6 | 7 | const TopicsContext = createContext(null); 8 | const TopicsUpdateContext = createContext(null); 9 | 10 | const useTopicsContext = () => useContext(TopicsContext); 11 | 12 | const useTopicsContextUpdate = () => useContext(TopicsUpdateContext); 13 | 14 | interface Props { 15 | children: any 16 | } 17 | 18 | const TopicsProvider: FC = ({ children } : Props) => { 19 | const [topicsArray, changeTopicsArray] = useState([]); 20 | 21 | const backdropUpdate = useBackdropUpdateContext(); 22 | 23 | useEffect(() => { 24 | backdropUpdate.handleClose(); 25 | }, [topicsArray]); 26 | 27 | const getTopics = () => { 28 | const userDetails = localStorage.getItem('userDetails'); 29 | fetch('/topic/getTopics', { 30 | method: 'POST', 31 | headers: { 'Content-Type': 'Application/JSON' }, 32 | body: userDetails, 33 | }) 34 | .then((data) => data.json()) 35 | .then((data) => { 36 | const topicData = data.topics.reduce((acc, cur) => { 37 | acc.push({ 38 | name: cur.name, 39 | partitions: cur.partitions.length, 40 | }); 41 | return acc; 42 | }, []); 43 | changeTopicsArray(topicData); 44 | }); 45 | }; 46 | 47 | const clearTopics = () => { 48 | changeTopicsArray([]); 49 | }; 50 | 51 | return ( 52 | 53 | 59 | {children} 60 | 61 | 62 | ); 63 | }; 64 | 65 | export { TopicsProvider, useTopicsContext, useTopicsContextUpdate }; 66 | -------------------------------------------------------------------------------- /demo/client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | kafka-penguin 12 | 13 | 14 |
15 | 16 | -------------------------------------------------------------------------------- /demo/client/index.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import * as ReactDOM from 'react-dom'; 3 | import { ThemeProvider } from '@material-ui/core/styles'; 4 | import App from './App'; 5 | import theme from './theme'; 6 | 7 | ReactDOM.render( 8 | 9 | {/* */} 10 | 11 | , 12 | document.getElementById('root'), 13 | ); 14 | -------------------------------------------------------------------------------- /demo/client/theme.ts: -------------------------------------------------------------------------------- 1 | import { createMuiTheme } from '@material-ui/core/styles'; 2 | 3 | const theme = createMuiTheme({ 4 | // spacing: factor => `${0.25 * factor}rem`, 5 | palette: { 6 | primary: { 7 | light: '#72cff8', 8 | main: '#4fc3f7', 9 | dark: '#3788ac', 10 | contrastText: '#fff', 11 | }, 12 | secondary: { 13 | light: '#ff7961', 14 | main: '#f44336', 15 | dark: '#ba000d', 16 | contrastText: '#000', 17 | }, 18 | background: { 19 | default: '#f4f4f4', 20 | }, 21 | text: { 22 | primary: '#404040', 23 | secondary: '#696969', 24 | disabled: '#a3a3a3', 25 | hint: '#a3a3a3', 26 | }, 27 | }, 28 | typography: { 29 | fontSize: 14, 30 | fontFamily: [ 31 | 'Montserrat', 32 | ].join(','), 33 | }, 34 | }); 35 | 36 | export default theme; 37 | -------------------------------------------------------------------------------- /demo/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: ['/__tests__'], 3 | transform: { 4 | '^.+\\.tsx?$': 'ts-jest', 5 | }, 6 | testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$', 7 | moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'], 8 | } -------------------------------------------------------------------------------- /demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@kafka-penguin/kafka-penguin", 3 | "version": "0.1.0", 4 | "description": "", 5 | "main": "client/index.tsx", 6 | "engines": { 7 | "node": "12.18.3" 8 | }, 9 | "scripts": { 10 | "build2": "webpack", 11 | "start": "ts-node ./server/server.ts", 12 | "format": "prettier --write src/**/*.ts{,x}", 13 | "lint": "tsc --noEmit && eslint src/**/*.ts{,x}", 14 | "tsc": "tsc -p tsconfig.json", 15 | "build": "tsc", 16 | "dev": "concurrently \"cross-env NODE_ENV=development webpack-dev-server --config webpack.config.js --hot --mode development\" \"nodemon server/server.ts\"", 17 | "test": "jest --detectOpenHandles --testTimeout=20000" 18 | }, 19 | "jest": { 20 | "moduleFileExtensions": [ 21 | "ts", 22 | "tsx", 23 | "js" 24 | ], 25 | "transform": { 26 | "^.+\\.(ts|tsx)$": "ts-jest" 27 | }, 28 | "testMatch": [ 29 | "**/__tests__/*.+(ts|tsx|js)" 30 | ] 31 | }, 32 | "keywords": [], 33 | "author": "", 34 | "license": "ISC", 35 | "dependencies": { 36 | "@babel/plugin-syntax-jsx": "^7.12.13", 37 | "@babel/plugin-transform-runtime": "^7.13.10", 38 | "@material-ui/core": "^4.11.3", 39 | "@material-ui/icons": "^4.11.2", 40 | "awesome-typescript-loader": "^5.2.1", 41 | "babel-jest": "^26.6.3", 42 | "concurrently": "^6.0.0", 43 | "cookie-parser": "^1.4.5", 44 | "cross-env": "^7.0.3", 45 | "dotenv": "^8.2.0", 46 | "express": "^4.17.1", 47 | "highlight.js": "^10.7.2", 48 | "kafka-penguin": "^0.3.2", 49 | "kafkajs": "^1.15.0", 50 | "nodemon": "^2.0.7", 51 | "prop-types": "^15.7.2", 52 | "puppeteer": "^2.1.1", 53 | "react": "^16.14.0", 54 | "react-dom": "^16.14.0", 55 | "react-router-dom": "^5.2.0", 56 | "react-scroll": "^1.8.2", 57 | "react-showdown": "^2.3.0", 58 | "react-syntax-highlighter": "^15.4.3", 59 | "react-tsparticles": "^1.26.3", 60 | "regenerator-runtime": "^0.13.7", 61 | "set-interval": "^2.1.2", 62 | "ts-loader": "^8.0.18", 63 | "ts-node": "^9.1.1", 64 | "typescript": "^3.0.0", 65 | "util": "^0.12.3" 66 | }, 67 | "devDependencies": { 68 | "@babel/core": "^7.13.13", 69 | "@babel/preset-env": "^7.13.12", 70 | "@babel/preset-react": "^7.13.13", 71 | "@babel/preset-typescript": "^7.13.0", 72 | "@types/express": "^4.17.11", 73 | "@types/jest": "^26.0.22", 74 | "@types/mocha": "^8.2.2", 75 | "@types/react": "^17.0.3", 76 | "@types/react-dom": "^17.0.3", 77 | "@types/react-router-dom": "^5.1.7", 78 | "@types/supertest": "^2.0.10", 79 | "@typescript-eslint/eslint-plugin": "^4.22.0", 80 | "@typescript-eslint/parser": "^4.22.0", 81 | "@webpack-cli/serve": "^1.3.0", 82 | "awesome-typescript-loader": "^5.2.1", 83 | "babel-cli": "^6.26.0", 84 | "babel-loader": "^8.2.2", 85 | "babel-preset-env": "^1.7.0", 86 | "enzyme": "^3.11.0", 87 | "enzyme-adapter-react-16": "^1.15.6", 88 | "eslint": "^7.24.0", 89 | "eslint-config-airbnb": "^18.2.1", 90 | "eslint-config-airbnb-typescript": "^12.3.1", 91 | "eslint-plugin-import": "^2.22.1", 92 | "eslint-plugin-jest": "^24.3.2", 93 | "eslint-plugin-jsx-a11y": "^6.4.1", 94 | "eslint-plugin-prettier": "^3.3.1", 95 | "eslint-plugin-react": "^7.23.2", 96 | "eslint-plugin-react-hooks": "^4.2.0", 97 | "express": "^4.17.1", 98 | "html-webpack-plugin": "^5.3.1", 99 | "jest": "^26.6.3", 100 | "mini-css-extract-plugin": "^1.3.9", 101 | "nodemon": "^2.0.7", 102 | "source-map-loader": "^2.0.1", 103 | "style-loader": "^2.0.0", 104 | "superagent": "^6.1.0", 105 | "supertest": "^6.1.3", 106 | "ts-jest": "^26.5.4", 107 | "ts-loader": "^8.0.18", 108 | "typescript": "^4.2.3", 109 | "webpack": "^5.28.0", 110 | "webpack-cli": "^4.5.0", 111 | "webpack-dev-server": "^3.11.2" 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /demo/server/app.ts: -------------------------------------------------------------------------------- 1 | // routers 2 | import express from 'express'; 3 | import path from 'path'; 4 | import dotenv from 'dotenv'; 5 | 6 | import strategyRouter from './routes/strategy'; 7 | import topicRouter from './routes/topic'; 8 | 9 | const app = express(); 10 | 11 | dotenv.config(); app.use(express.json()); 12 | app.use(express.urlencoded({ extended: true })); 13 | 14 | app.use('/assets', express.static(path.resolve(__dirname, '../client/assets/'))); 15 | app.use('/js', express.static(path.resolve(__dirname, '../build/js/'))); 16 | 17 | app.get('/', (req, res) => res.status(200).sendFile(path.join(__dirname, '../build/index.html'))); 18 | 19 | app.use('/topic', topicRouter); 20 | app.use('/strategy', strategyRouter); 21 | app.get('/*', (req, res) => res.status(200).sendFile(path.resolve(__dirname, '../build/index.html'))); 22 | app.get('*', (req, res) => res.status(404).json()); 23 | 24 | app.use((err, req, res, next) => { 25 | const defaultErr = { 26 | log: 'Express error handler caught unknown middleware error', 27 | status: 500, 28 | message: 'An error occurred', 29 | }; 30 | const error = { 31 | ...defaultErr, 32 | ...err, 33 | }; 34 | return res.status(error.status).json(error.message); 35 | }); 36 | 37 | export default app; 38 | -------------------------------------------------------------------------------- /demo/server/controllers/DLQController.ts: -------------------------------------------------------------------------------- 1 | import { RequestHandler } from 'express'; 2 | import { Kafka, logLevel } from 'kafkajs'; 3 | import { DeadLetterQueue } from 'kafka-penguin'; 4 | import dotenv = require('dotenv'); 5 | 6 | dotenv.config(); 7 | 8 | const ERROR_LOG = []; 9 | 10 | const MyLogCreator = (logLevel) => ({ 11 | namespace, level, label, log, 12 | }) => { 13 | // also availabe on log object => timestamp, logger, message and more 14 | const { error, correlationId } = log; 15 | if (correlationId) { 16 | ERROR_LOG.push( 17 | `[${namespace}] Logger: kafka-penguin ${label}: ${error} correlationId: ${correlationId}`, 18 | ); 19 | } 20 | }; 21 | 22 | const DLQKafka = new Kafka({ 23 | clientId: 'makeClient', 24 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 25 | ssl: true, 26 | sasl: { 27 | mechanism: 'plain', 28 | username: process.env.KAFKA_USERNAME, 29 | password: process.env.KAFKA_PASSWORD, 30 | }, 31 | logLevel: logLevel.ERROR, 32 | logCreator: MyLogCreator, 33 | }); 34 | 35 | const dlqProduce: RequestHandler = (req, res, next) => { 36 | const { 37 | topic, message, retries, faults, 38 | } = req.body; 39 | 40 | const messagesArray = []; 41 | // create messages array with specified number of faults 42 | for (let i = 0; i < retries; i++) { 43 | if (i < faults) messagesArray.push({ key: 'test', value: 'fault' }); 44 | else { 45 | messagesArray.push({ 46 | key: 'test', 47 | value: message, 48 | }); 49 | } 50 | } 51 | 52 | const cb = (message) => { 53 | if (message.value.toString() === 'fault') { 54 | return false; 55 | } return true; 56 | }; 57 | 58 | const admin = DLQKafka.admin(); 59 | const DLQClient = new DeadLetterQueue(DLQKafka, topic, cb); 60 | const DLQProducer = DLQClient.producer(); 61 | const DLQConsumer = DLQClient.consumer({ groupId: 'demo' }); 62 | 63 | res.locals.DLQClients = { 64 | consumer: DLQConsumer, 65 | retries, 66 | faults, 67 | }; 68 | // DLQProducer.logger().info('TEST', {KAFKA_PENGUIN: 'TESTING CUSTOM'}) 69 | DLQProducer.connect() 70 | .then(() => { 71 | DLQProducer.send({ 72 | topic, 73 | messages: messagesArray, 74 | }).catch((e) => console.log('this is error in try', e.reference)); 75 | }) 76 | .then(DLQProducer.disconnect()) 77 | .then(admin.connect()) 78 | .then(async () => { 79 | const offsetData = await admin.fetchTopicOffsets(topic); 80 | res.locals.latestOffset = offsetData[0].offset; 81 | }) 82 | .then(admin.disconnect()) 83 | .then(() => next()) 84 | .catch((e: Error) => { 85 | if (e.message === 'This server does not host this topic-partition') { 86 | return res.status(300).json([`This error was executed as part of the kafka-penguin 87 | Dead Letter Queue message reprocessing strategy. Your producer attempted to deliver 88 | a message 6 times but was unsuccessful. As a result, the message was sent to a 89 | Dead Letter Queue. Refer to the original error for further information`]); 90 | } 91 | return next({ 92 | message: `Error implementing Dead Letter Queue strategy, producer side:${e.message}`, 93 | error: e, 94 | }); 95 | }); 96 | }; 97 | 98 | const dlqConsume: RequestHandler = (req, res, next) => { 99 | const { faults, consumer, retries } = res.locals.DLQClients; 100 | const messageLog = []; 101 | consumer.connect() 102 | .then(consumer.subscribe()) 103 | .then(() => { 104 | const latestOffset = Number(res.locals.latestOffset); 105 | consumer.run({ 106 | eachMessage: ({ topic, partitions, message }) => { 107 | const messageOffset = Number(message.offset); 108 | 109 | if (messageOffset >= latestOffset - retries) { 110 | messageLog.push(message.value.toString()); 111 | } 112 | if (messageLog.length === retries - faults) { 113 | messageLog.push(`kafka-penguin: Error with message processing, ${faults} ${faults > 1 ? 'messages' : 'message'} 114 | sent to DLQ topic ${topic}.deadLetterQueue`); 115 | res.locals.messages = messageLog; 116 | consumer.disconnect() 117 | .then(() => res.status(200).json(res.locals.messages)) 118 | .catch((e) => next({ 119 | message: `Error implementing Dead Letter Queue strategy while consuming messages, consumer side: ${e.message}`, 120 | error: e, 121 | })); 122 | } 123 | }, 124 | }); 125 | }) 126 | .catch((e) => next({ 127 | message: `Error implementing Dead Letter Queue strategy, consumer side: ${e.message}`, 128 | error: e, 129 | })); 130 | }; 131 | 132 | export default { 133 | dlqConsume, 134 | dlqProduce, 135 | }; 136 | -------------------------------------------------------------------------------- /demo/server/controllers/failfastController.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { RequestHandler } from 'express'; 3 | import { Kafka, logLevel } from 'kafkajs'; 4 | import { FailFast } from 'kafka-penguin'; 5 | import dotenv = require('dotenv'); 6 | 7 | dotenv.config(); 8 | // cache to store error logs 9 | let ERROR_LOG = []; 10 | 11 | const MyLogCreator = (logLevel) => ({ 12 | namespace, label, log, 13 | }) => { 14 | // also availabe on log object => timestamp, logger, message and more 15 | // console.log(log) 16 | const { error, correlationId } = log; 17 | if (correlationId) { 18 | ERROR_LOG.push( 19 | `[${namespace}] Logger: kafka-penguin ${label}: ${error} correlationId: ${correlationId}`, 20 | ); 21 | } 22 | }; 23 | 24 | // new kafka instance with logCreator added 25 | const failfastKafka = new Kafka({ 26 | clientId: 'makeClient', 27 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 28 | ssl: true, 29 | sasl: { 30 | mechanism: 'plain', 31 | username: process.env.KAFKA_USERNAME, 32 | password: process.env.KAFKA_PASSWORD, 33 | }, 34 | logLevel: logLevel.ERROR, 35 | logCreator: MyLogCreator, 36 | }); 37 | 38 | const failfast: RequestHandler = (req, res, next) => { 39 | const newStrategy = new FailFast(req.body.retries - 1, failfastKafka); 40 | const producer = newStrategy.producer(); 41 | const message = { 42 | topic: req.body.topic, 43 | messages: [ 44 | { 45 | key: 'hello', 46 | value: req.body.message, 47 | }, 48 | ], 49 | }; 50 | producer 51 | .connect() 52 | .then(() => console.log('Connected')) 53 | .then(() => producer.send(message)) 54 | .then(() => { 55 | if (ERROR_LOG.length > 0) { 56 | const plural = ERROR_LOG.length > 1 ? 'times' : 'time'; 57 | ERROR_LOG.push( 58 | `kafka-penguin: FailFast stopped producer after ${ERROR_LOG.length} ${plural}!`, 59 | ); 60 | res.locals.error = [...ERROR_LOG]; 61 | } else { 62 | res.locals.error = ['kafka-penguin: Message produced successfully']; 63 | } 64 | 65 | ERROR_LOG = []; 66 | return next(); 67 | }) 68 | .catch((e : Error) => next({ 69 | message: `Error implementing FailFast strategy: ${e.message}`, 70 | error: e, 71 | })); 72 | }; 73 | 74 | export default { 75 | failfast, 76 | }; 77 | -------------------------------------------------------------------------------- /demo/server/controllers/ignoreController.ts: -------------------------------------------------------------------------------- 1 | import { RequestHandler } from 'express'; 2 | import { Kafka, logLevel } from 'kafkajs'; 3 | import { DeadLetterQueue } from 'kafka-penguin'; 4 | import dotenv = require('dotenv'); 5 | 6 | dotenv.config(); 7 | 8 | const ERROR_LOG = []; 9 | 10 | const MyLogCreator = (logLevel) => ({ 11 | namespace, label, log, 12 | }) => { 13 | // also availabe on log object => timestamp, logger, message and more 14 | const { error, correlationId } = log; 15 | if (correlationId) { 16 | ERROR_LOG.push( 17 | `[${namespace}] Logger: kafka-penguin ${label}: ${error} correlationId: ${correlationId}`, 18 | ); 19 | } 20 | }; 21 | 22 | const ignoreKafka = new Kafka({ 23 | clientId: 'makeClient', 24 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 25 | ssl: true, 26 | sasl: { 27 | mechanism: 'plain', 28 | username: process.env.KAFKA_USERNAME, 29 | password: process.env.KAFKA_PASSWORD, 30 | }, 31 | logLevel: logLevel.ERROR, 32 | logCreator: MyLogCreator, 33 | }); 34 | 35 | const ignoreProduce: RequestHandler = (req, res, next) => { 36 | const { 37 | topic, message, retries, faults, 38 | } = req.body; 39 | 40 | const messagesArray = []; 41 | // create messages array with specified number of faults 42 | for (let i = 0; i < retries; i++) { 43 | if (i < faults) messagesArray.push({ key: 'test', value: 'fault' }); 44 | else { 45 | messagesArray.push({ 46 | key: 'test', 47 | value: message, 48 | }); 49 | } 50 | } 51 | 52 | const cb = (message) => { 53 | if (message.value.toString() === 'fault') { 54 | return false; 55 | } return true; 56 | }; 57 | 58 | const admin = ignoreKafka.admin(); 59 | const ignoreClient = new DeadLetterQueue(ignoreKafka, topic, cb); 60 | const ignoreProducer = ignoreClient.producer(); 61 | const ignoreConsumer = ignoreClient.consumer({ groupId: 'demo' }); 62 | 63 | res.locals.ignoreClients = { 64 | consumer: ignoreConsumer, 65 | retries, 66 | faults, 67 | }; 68 | // ignoreProducer.logger().info('TEST', {KAFKA_PENGUIN: 'TESTING CUSTOM'}) 69 | ignoreProducer.connect() 70 | .then(() => { 71 | ignoreProducer.send({ 72 | topic, 73 | messages: messagesArray, 74 | }).catch((e) => console.log('this is error in try', e.reference)); 75 | }) 76 | .then(ignoreProducer.disconnect()) 77 | .then(admin.connect()) 78 | .then(async () => { 79 | const offsetData = await admin.fetchTopicOffsets(topic); 80 | res.locals.latestOffset = offsetData[0].offset; 81 | }) 82 | .then(admin.disconnect()) 83 | .then(() => next()) 84 | .catch((e: Error) => { 85 | if (e.message === 'This server does not host this topic-partition') { 86 | return res.status(300).json([`This error was executed as part of the kafka-penguin 87 | Ignore message reprocessing strategy.`]); 88 | } 89 | return next({ 90 | message: `Error implementing Ignore strategy, producer side:${e.message}`, 91 | error: e, 92 | }); 93 | }); 94 | }; 95 | 96 | const ignoreConsume: RequestHandler = (req, res, next) => { 97 | const { faults, consumer, retries } = res.locals.ignoreClients; 98 | const messageLog = []; 99 | consumer.connect() 100 | .then(consumer.subscribe()) 101 | .then(() => { 102 | const latestOffset = Number(res.locals.latestOffset); 103 | consumer.run({ 104 | eachMessage: ({ topic, partitions, message }) => { 105 | const messageOffset = Number(message.offset); 106 | 107 | if (messageOffset >= latestOffset - retries) { 108 | messageLog.push(message.value.toString()); 109 | } 110 | if (messageLog.length === retries - faults) { 111 | messageLog.push(`kafka-penguin: Error with message processing, ${faults} ${faults > 1 ? 'messages' : 'message'} 112 | ignored.`); 113 | res.locals.messages = messageLog; 114 | consumer.disconnect() 115 | .then(() => res.status(200).json(res.locals.messages)) 116 | .catch((e) => next({ 117 | message: `Error implementing Ignore strategy while consuming messages, consumer side: ${e.message}`, 118 | error: e, 119 | })); 120 | } 121 | }, 122 | }); 123 | }) 124 | .catch((e) => next({ 125 | message: `Error implementing Ignore strategy, consumer side: ${e.message}`, 126 | error: e, 127 | })); 128 | }; 129 | 130 | export default { 131 | ignoreConsume, 132 | ignoreProduce, 133 | }; 134 | -------------------------------------------------------------------------------- /demo/server/controllers/kafkaController.ts: -------------------------------------------------------------------------------- 1 | import { Kafka } from 'kafkajs'; 2 | import { RequestHandler } from 'express'; 3 | 4 | const makeClient: RequestHandler = (req, res, next) => { 5 | const brokers = !req.body.brokers ? process.env.KAFKA_BOOTSTRAP_SERVER : req.body.brokers; 6 | const kafka = new Kafka({ 7 | clientId: 'makeClient', 8 | brokers: [brokers], 9 | ssl: true, 10 | sasl: { 11 | mechanism: 'plain', 12 | username: !req.body.username ? process.env.KAFKA_USERNAME : req.body.username, 13 | password: !req.body.password ? process.env.KAFKA_PASSWORD : req.body.password, 14 | }, 15 | }); 16 | res.locals.kafka = kafka; 17 | return next(); 18 | }; 19 | 20 | export default { 21 | makeClient, 22 | }; 23 | -------------------------------------------------------------------------------- /demo/server/controllers/strategyController.ts: -------------------------------------------------------------------------------- 1 | import { RequestHandler } from 'express'; 2 | import { Kafka, logLevel } from 'kafkajs'; 3 | // const kafkapenguin = require('kafka-penguin'); 4 | import { FailFast, DeadLetterQueue } from 'kafka-penguin'; 5 | // import { DeadLetterQueue } from '../../../kafka-penguin/src/index' 6 | import dotenv = require('dotenv'); 7 | dotenv.config(); 8 | //cache to store error logs 9 | let ERROR_LOG = []; 10 | let MESSAGE_LOG = []; 11 | 12 | 13 | const MyLogCreator = logLevel => ({ namespace, level, label, log }) => { 14 | //also availabe on log object => timestamp, logger, message and more 15 | const { error, correlationId } = log; 16 | if (correlationId) { 17 | ERROR_LOG.push( 18 | `[${namespace}] Logger: kafka-penguin ${label}: ${error} correlationId: ${correlationId}` 19 | ); 20 | } 21 | }; 22 | 23 | //new kafka instance with logCreator added 24 | const strategyKafka = new Kafka({ 25 | clientId: 'makeClient', 26 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 27 | ssl: true, 28 | sasl: { 29 | mechanism: 'plain', 30 | username: process.env.KAFKA_USERNAME, 31 | password: process.env.KAFKA_PASSWORD, 32 | }, 33 | logLevel: logLevel.ERROR, 34 | logCreator: MyLogCreator, 35 | }); 36 | 37 | const failfast: RequestHandler = (req, res, next) => { 38 | // const strategies = kafkapenguin.failfast; 39 | // const newStrategy = new strategies.FailFast(req.body.retries - 1, strategyKafka); 40 | const newStrategy = new FailFast(req.body.retries - 1, strategyKafka); 41 | const producer = newStrategy.producer(); 42 | const message = { 43 | topic: req.body.topic, 44 | messages: [ 45 | { 46 | key: 'hello', 47 | value: req.body.message, 48 | }, 49 | ], 50 | }; 51 | producer 52 | .connect() 53 | .then(() => console.log('Connected')) 54 | .then(() => producer.send(message)) 55 | .then(() => { 56 | if (ERROR_LOG.length > 0) { 57 | const plural = ERROR_LOG.length > 1 ? 'times' : 'time'; 58 | ERROR_LOG.push( 59 | `kafka-penguin: FailFast stopped producer after ${ERROR_LOG.length} ${plural}!` 60 | ); 61 | res.locals.error = [...ERROR_LOG]; 62 | } else { 63 | res.locals.error = ['kafka-penguin: Message produced successfully']; 64 | } 65 | 66 | ERROR_LOG = []; 67 | return next(); 68 | }) 69 | .catch(e => { 70 | return next({ 71 | message: 'Error implementing FailFast strategy: ' + e.message, 72 | error: e, 73 | }); 74 | }); 75 | }; 76 | 77 | const dlqProduce: RequestHandler = (req, res, next) => { 78 | //create messages array with specified number of faults 79 | const { topic, message, retries, faults } = req.body; 80 | const random = (count, messages, result = new Set()) => { 81 | if (result.size === count) return result; 82 | const num = Math.floor(Math.random() * messages); 83 | result.add(num); 84 | return random(count, messages, result); 85 | }; 86 | 87 | const faultsIndex = random(faults, retries); 88 | 89 | const messagesArray = []; 90 | 91 | for (let i = 0; i < retries; i++) { 92 | if (faultsIndex.has(i)) messagesArray.push({ key: 'test', value: 'fault' }); 93 | else 94 | messagesArray.push({ 95 | key: 'test', 96 | value: message, 97 | }); 98 | } 99 | 100 | const cb = message => { 101 | if (message === 'fault') 102 | return false; 103 | return true; 104 | }; 105 | console.log('topic-----------', topic); 106 | console.log(messagesArray); 107 | const DLQClient = new DeadLetterQueue(strategyKafka, topic, cb); 108 | // produce message to topic 109 | const DLQProducer = DLQClient.producer(); 110 | // const DLQProducer = res.locals.kafka.producer(); 111 | const DLQConsumer = DLQClient.consumer({ groupId: 'demo' }); 112 | 113 | DLQProducer.connect() 114 | .then( 115 | DLQProducer.send({ 116 | topic: topic, 117 | messages: messagesArray, 118 | }) 119 | ) 120 | .then(DLQProducer.disconnect()) 121 | .then(() => { 122 | // res.locals.clientInfo = { 123 | // brokers: process.env.KAFKA_BOOTSTRAP_SERVER, 124 | // username: process.env.KAFKA_USERNAME, 125 | // password: process.env.KAFKA_PASSWORD, 126 | // }; 127 | res.locals.DLQClients = { 128 | producer: DLQProducer, 129 | consumer: DLQConsumer, 130 | retries: retries 131 | } 132 | 133 | return next(); 134 | }) 135 | .catch(e => { 136 | console.log(e); 137 | }); 138 | }; 139 | 140 | const dlqConsume: RequestHandler = (req, res, next) => { 141 | const { producer, consumer, retries } = res.locals.DLQClients 142 | let messageLog = []; 143 | consumer.connect() 144 | .then(consumer.subscribe()) 145 | .then(() => { 146 | // const run = () => { 147 | consumer.run({ 148 | eachMessage: ({topic, partitions, message}) => { 149 | console.log(message.value.toString()) 150 | messageLog.push(message.value.toString()) 151 | console.log(messageLog); 152 | if (messageLog.length === retries) { 153 | res.locals.messages = [...messageLog]; 154 | messageLog = []; 155 | consumer.disconnect() 156 | .then(() => { 157 | return res.status(200).json(res.locals.messages) 158 | }) 159 | 160 | 161 | } 162 | } 163 | }) 164 | // } 165 | }) 166 | .then(async () => { 167 | console.log('message log', await messageLog) 168 | }) 169 | .then(console.log(messageLog)) 170 | .catch((e: Error) => console.log) 171 | 172 | }; 173 | 174 | export default { 175 | failfast, 176 | dlqProduce, 177 | dlqConsume, 178 | }; 179 | -------------------------------------------------------------------------------- /demo/server/controllers/topicsController.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { RequestHandler } from 'express'; 3 | 4 | const getTopics: RequestHandler = (req, res, next) => { 5 | const { kafka } = res.locals; 6 | const admin = kafka.admin(); 7 | admin.connect() 8 | .then(() => console.log('Connected')) 9 | .then(() => admin.fetchTopicMetadata()) 10 | .then((data: { 11 | name: string, 12 | partitions: Array 13 | }) => { res.locals.topicsData = data; }) 14 | .then(() => admin.disconnect()) 15 | .then(() => next()) 16 | .catch((e) => next({ 17 | message: `Error getting topics in topicsController.getTopics ${e.message}`, 18 | error: e, 19 | })); 20 | }; 21 | 22 | export default { 23 | getTopics, 24 | }; 25 | -------------------------------------------------------------------------------- /demo/server/routes/strategy.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import kafkaController from '../controllers/kafkaController'; 3 | import failfastController from '../controllers/failfastController'; 4 | import DLQController from '../controllers/DLQController'; 5 | import ignoreController from '../controllers/ignoreController'; 6 | 7 | const router = express.Router(); 8 | 9 | router.post('/failfast', 10 | failfastController.failfast, 11 | (req, res) => res.status(200).json(res.locals.error)); 12 | 13 | router.post( 14 | '/dlq', 15 | kafkaController.makeClient, 16 | DLQController.dlqProduce, 17 | DLQController.dlqConsume, 18 | (req, res) => res.status(200).json(res.locals.error), 19 | ); 20 | 21 | router.post( 22 | '/ignore', 23 | kafkaController.makeClient, 24 | ignoreController.ignoreProduce, 25 | ignoreController.ignoreConsume, 26 | (req, res) => res.status(200).json(res.locals.error), 27 | ); 28 | 29 | export default router; 30 | -------------------------------------------------------------------------------- /demo/server/routes/topic.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | import topicsController from '../controllers/topicsController'; 3 | import kafkaController from '../controllers/kafkaController'; 4 | 5 | const router = express.Router(); 6 | 7 | router.post('/getTopics', 8 | kafkaController.makeClient, 9 | topicsController.getTopics, 10 | (req, res) => res.status(200).json(res.locals.topicsData)); 11 | 12 | export default router; 13 | -------------------------------------------------------------------------------- /demo/server/server.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import app from './app'; 3 | // const app = require('./app'); 4 | const PORT = process.env.PORT || 3000; 5 | 6 | app.listen(PORT, () => { 7 | console.log(`Server listening on port ${PORT}`); 8 | }); 9 | -------------------------------------------------------------------------------- /demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | // "types": ["node"], 4 | "typeRoots": ["./types", "./node_modules/@types"], 5 | "target": "ES5", 6 | "module": "commonJS", 7 | "jsx": "react", 8 | "sourceMap": true, 9 | "outDir": "./build", 10 | "removeComments": true, 11 | "allowSyntheticDefaultImports": true, 12 | "esModuleInterop": true, 13 | "preserveConstEnums": true 14 | }, 15 | "include": [ 16 | "./client/**/*", 17 | ], 18 | "exclude": [ 19 | "node_modules" 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /demo/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "types": ["jest", "node"], 4 | "typeRoots": ["./types", "./node_modules/@types"], 5 | "target": "ES5", 6 | "module": "commonJS", 7 | "jsx": "react", 8 | "sourceMap": true, 9 | "outDir": "./build", 10 | "removeComments": true, 11 | "allowSyntheticDefaultImports": true, 12 | "esModuleInterop": true, 13 | "preserveConstEnums": true 14 | }, 15 | "include": [ 16 | "./client/**/*", 17 | ], 18 | "exclude": [ 19 | "node_modules" 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /demo/webpack.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | 3 | const path = require('path') 4 | const HtmlWebpackPlugin = require('html-webpack-plugin') 5 | 6 | module.exports = { 7 | mode: "production", 8 | entry: { 9 | app: ['./client/index.tsx'], 10 | vendor: ['react', 'react-dom'] 11 | }, 12 | output: { 13 | path: path.resolve(__dirname, 'build'), 14 | filename: 'js/[name].bundle.js' 15 | }, 16 | devtool: 'source-map', 17 | resolve: { 18 | extensions: ['.ts', '.tsx', '.js', '.jsx', '.json'] 19 | }, 20 | devServer: { 21 | contentBase: './build', 22 | port: 8000, 23 | proxy: { 24 | '/': 'http://localhost:3000' 25 | } 26 | }, 27 | module: { 28 | rules: [ 29 | { 30 | test: /\.(ts|js)x?$/, 31 | exclude: '/node_modules', 32 | loader: 'babel-loader' 33 | } 34 | ] 35 | }, 36 | plugins: [ 37 | new HtmlWebpackPlugin({ 38 | template: path.resolve(__dirname, './client/index.html') 39 | }) 40 | ] 41 | } -------------------------------------------------------------------------------- /exampleBasic.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { FailFast } from 'kafka-penguin'; 3 | 4 | const exampleClient = require('./clientConfig.ts'); 5 | 6 | // Set up the preferred strategy with a configured KafkaJS client 7 | const exampleStrategy = new FailFast(2, exampleClient); 8 | 9 | // Initialize a producer or consumer from the instance of the strategy 10 | const producer = exampleStrategy.producer(); 11 | 12 | const message = { 13 | topic: 'wrong-topic', 14 | messages: [ 15 | { 16 | key: 'hello', 17 | value: 'world', 18 | }, 19 | ], 20 | }; 21 | 22 | // Connect, Subscribe, Send, or Run virtually the same as with KafkaJS 23 | producer.connect() 24 | .then(() => console.log('Connected!')) 25 | // The chosen strategy executes under the hood, like in this send method 26 | .then(() => producer.send(message)) 27 | .catch((e: any) => console.log('error: ', e.message)); 28 | -------------------------------------------------------------------------------- /exampleDLQConsumer.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { DeadLetterQueue } from 'kafka-penguin'; 3 | 4 | const client = require('./clientConfig.ts'); 5 | 6 | const topic = 'test-topic-DLQ'; 7 | 8 | // This allows the consumer to evaluate each message according to a condition 9 | // The callback must return a boolean value 10 | const callback = (message) => { 11 | try { 12 | if (typeof message.value === 'string') { 13 | return true; 14 | } 15 | } catch (e) { 16 | return false; 17 | } 18 | return true; 19 | }; 20 | 21 | // Set up the Dead Letter Queue (DLQ) strategy 22 | // with a configured KafkaJS client, a topic, and the evaluating callback 23 | const exampleDLQConsumer = new DeadLetterQueue(client, topic, callback); 24 | 25 | // Initialize a consumer from the new instance of the Dead Letter Queue strategy 26 | const consumerDLQ = exampleDLQConsumer.consumer({ groupId: 'testID' }); 27 | 28 | // Connecting the consumer creates a DLQ topic in case of bad messages 29 | // If the callback returns false, the strategy moves the message to the topic specific DLQ 30 | // The consumer is able to keep consuming good messages from the topic 31 | consumerDLQ.connect() 32 | .then(consumerDLQ.subscribe()) 33 | .then(() => consumerDLQ.run({ 34 | eachMessage: ({ message }) => { 35 | if (message.value.length < 5) return true; 36 | return false; 37 | }, 38 | })) 39 | .catch((e) => console.log(`Error message from consumer: ${e}`)); 40 | -------------------------------------------------------------------------------- /exampleDLQProducer.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { DeadLetterQueue } from 'kafka-penguin'; 3 | 4 | const producerClientDLQ = require('./clientConfig.ts'); 5 | 6 | // This example simulates an error where the producer sends to a bad topic 7 | const topicGood = 'test-topic-DLQ'; 8 | const topicBad = 'topic-non-existent'; 9 | 10 | // Set up the Dead Letter Queue (DLQ) strategy 11 | // Configure it with a configured KafkaJS client, a topic, and a callback that returns boolean 12 | const exampleDLQProducer = new DeadLetterQueue(producerClientDLQ, topicGood, true); 13 | 14 | // Initialize a producer from the new instance of the Dead Letter Queue strategy 15 | const producerDLQ = exampleDLQProducer.producer(); 16 | 17 | // Connecting the producer creates a DLQ topic in case of bad messages 18 | // If an error occurs, the strategy moves the message to the topic specific DLQ 19 | // The producer is able to keep publishing good messages to the topic 20 | producerDLQ.connect() 21 | .then(() => producerDLQ.send({ 22 | topic: topicGood, 23 | messages: [ 24 | { 25 | key: 'message 1', 26 | value: 'Good Message', 27 | }, 28 | ], 29 | })) 30 | .then(() => producerDLQ.send({ 31 | topic: topicBad, 32 | messages: [ 33 | { 34 | key: 'message 2', 35 | value: 'Bad Message', 36 | }, 37 | ], 38 | })) 39 | .then(() => producerDLQ.send({ 40 | topic: topicGood, 41 | messages: [ 42 | { 43 | key: 'message 3', 44 | value: 'Good Message', 45 | }, 46 | ], 47 | })) 48 | .then(() => producerDLQ.disconnect()) 49 | .catch((e: any) => { 50 | console.log(e); 51 | }); 52 | -------------------------------------------------------------------------------- /exampleFailFast.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { FailFast } from './kafka-penguin/src/index'; 3 | 4 | const FailFastClient = require('../src/clientConfig.ts'); 5 | 6 | // Set up Fail Fast with the number of retried and a configured KafkaJS client 7 | const exampleFailFast = new FailFast(2, FailFastClient); 8 | 9 | // Initialize a producer from the new instance of Fail Fast 10 | const producer = exampleFailFast.producer(); 11 | 12 | // Example error of a producer sending to a non-existent topic 13 | const message = { 14 | topic: 'topic-non-existent', 15 | messages: [ 16 | { 17 | key: 'hello', 18 | value: 'world', 19 | }, 20 | ], 21 | }; 22 | 23 | // Fail Fast will attempt to send the message to the Kafka cluster. 24 | // After the retry count is reached, the producer will automatically disconnect. 25 | // An Error is also thrown. 26 | producer.connect() 27 | .then(() => console.log('Connected!')) 28 | .then(() => producer.send(message)) 29 | .catch((e: any) => console.log('error: ', e.message)); 30 | -------------------------------------------------------------------------------- /exampleIgnoreConsumer.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { Ignore } from './kafka-penguin/src/index'; 3 | 4 | const client = require('./clientConfig.ts'); 5 | 6 | const topic = 'test-topic'; 7 | 8 | // This allows the consumer to evaluate each message according to a condition 9 | // The callback must return a boolean value 10 | const callback = (message) => { 11 | try { 12 | JSON.parse(message.value); 13 | } catch (e) { 14 | return false; 15 | } 16 | return true; 17 | }; 18 | 19 | // Set up the Ignore strategy 20 | // with a configured KafkaJS client, a topic, and the evaluating callback 21 | const exampleIgnoreConsumer = new Ignore(client, topic, callback); 22 | 23 | // Initialize a consumer from the new instance of the Dead Letter Queue strategy 24 | const consumerIgnore = exampleIgnoreConsumer.consumer({ groupId: 'testID' }); 25 | 26 | // Connecting the consumer to consume messages. bad messages 27 | // If the callback evaluates a message as erroneous by returning false, the strategy 28 | // enables the consumer to keep consuming good messages from the topic 29 | consumerIgnore.connect() 30 | .then(consumerIgnore.subscribe()) 31 | .then(() => consumerIgnore.run({ 32 | eachMessage: ({ message }) => { 33 | // if (message.value.length < 5) return true; 34 | // return false; 35 | console.log("message value:", message.value.toString()) 36 | }, 37 | })) 38 | .catch((e) => console.log(`Error message from consumer: ${e}`)); 39 | -------------------------------------------------------------------------------- /exampleIgnoreProducer.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { Ignore } from './kafka-penguin/src/index' 3 | 4 | const producerClientIgnore = require('./clientConfig.ts'); 5 | 6 | // This example simulates an error where the producer sends to a bad topic 7 | const topicGood = 'test-topic'; 8 | const topicBad = 'topic-non-existent'; 9 | 10 | // Set up the Ignore strategy 11 | // Configure it with a configured KafkaJS client, a topic, and a callback that returns boolean 12 | const exampleIgnoreProducer = new Ignore(producerClientIgnore, topicGood, true); 13 | 14 | // Initialize a producer from the new instance of the Ignore strategy 15 | const producerIgnore = exampleIgnoreProducer.producer(); 16 | 17 | // Connecting the producer and send messages. 18 | // If an error occurs with a message, the strategy ignores erroneous message and continues 19 | // publishing good messages to the topic 20 | producerIgnore.connect() 21 | .then(() => producerIgnore.send({ 22 | topic: topicGood, 23 | messages: [ 24 | { 25 | key: 'message 1', 26 | value: JSON.stringify('Good Message'), 27 | }, 28 | ], 29 | })) 30 | .then(() => producerIgnore.send({ 31 | topic: topicBad, 32 | messages: [ 33 | { 34 | key: 'message 2', 35 | value: 'Bad Message', 36 | }, 37 | ], 38 | })) 39 | .then(() => producerIgnore.send({ 40 | topic: topicGood, 41 | messages: [ 42 | { 43 | key: 'message 3', 44 | value: JSON.stringify('Good Message'), 45 | }, 46 | ], 47 | })) 48 | .then(() => producerIgnore.disconnect()) 49 | .catch((e: any) => { 50 | console.log(e); 51 | }); 52 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; 5 | -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable max-len */ 2 | /* 3 | * For a detailed explanation regarding each configuration property and type check, visit: 4 | * https://jestjs.io/docs/en/configuration.html 5 | */ 6 | 7 | export default { 8 | // All imported modules in your tests should be mocked automatically 9 | // automock: false, 10 | 11 | // Stop running tests after `n` failures 12 | // bail: 0, 13 | 14 | // The directory where Jest should store its cached dependency information 15 | // cacheDirectory: "/private/var/folders/r7/gl7yvbmn0_lf04yblmhmxp180000gn/T/jest_dx", 16 | 17 | // Automatically clear mock calls and instances between every test 18 | clearMocks: true, 19 | 20 | // Indicates whether the coverage information should be collected while executing the test 21 | // collectCoverage: false, 22 | 23 | // An array of glob patterns indicating a set of files for which coverage information should be collected 24 | // collectCoverageFrom: undefined, 25 | 26 | // The directory where Jest should output its coverage files 27 | coverageDirectory: 'coverage', 28 | 29 | // An array of regexp pattern strings used to skip coverage collection 30 | // coveragePathIgnorePatterns: [ 31 | // "/node_modules/" 32 | // ], 33 | 34 | // Indicates which provider should be used to instrument code for coverage 35 | // coverageProvider: "babel", 36 | 37 | // A list of reporter names that Jest uses when writing coverage reports 38 | // coverageReporters: [ 39 | // "json", 40 | // "text", 41 | // "lcov", 42 | // "clover" 43 | // ], 44 | 45 | // An object that configures minimum threshold enforcement for coverage results 46 | // coverageThreshold: undefined, 47 | 48 | // A path to a custom dependency extractor 49 | // dependencyExtractor: undefined, 50 | 51 | // Make calling deprecated APIs throw helpful error messages 52 | // errorOnDeprecated: false, 53 | 54 | // Force coverage collection from ignored files using an array of glob patterns 55 | // forceCoverageMatch: [], 56 | 57 | // A path to a module which exports an async function that is triggered once before all test suites 58 | // globalSetup: undefined, 59 | 60 | // A path to a module which exports an async function that is triggered once after all test suites 61 | // globalTeardown: undefined, 62 | 63 | // A set of global variables that need to be available in all test environments 64 | // globals: {}, 65 | 66 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. 67 | // maxWorkers: "50%", 68 | 69 | // An array of directory names to be searched recursively up from the requiring module's location 70 | // moduleDirectories: [ 71 | // "node_modules" 72 | // ], 73 | 74 | // An array of file extensions your modules use 75 | // moduleFileExtensions: [ 76 | // "js", 77 | // "json", 78 | // "jsx", 79 | // "ts", 80 | // "tsx", 81 | // "node" 82 | // ], 83 | 84 | // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module 85 | // moduleNameMapper: {}, 86 | 87 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 88 | // modulePathIgnorePatterns: [], 89 | 90 | // Activates notifications for test results 91 | // notify: false, 92 | 93 | // An enum that specifies notification mode. Requires { notify: true } 94 | // notifyMode: "failure-change", 95 | 96 | // A preset that is used as a base for Jest's configuration 97 | // preset: undefined, 98 | 99 | // Run tests from one or more projects 100 | // projects: undefined, 101 | 102 | // Use this configuration option to add custom reporters to Jest 103 | // reporters: undefined, 104 | 105 | // Automatically reset mock state between every test 106 | // resetMocks: false, 107 | 108 | // Reset the module registry before running each individual test 109 | // resetModules: false, 110 | 111 | // A path to a custom resolver 112 | // resolver: undefined, 113 | 114 | // Automatically restore mock state between every test 115 | // restoreMocks: false, 116 | 117 | // The root directory that Jest should scan for tests and modules within 118 | // rootDir: undefined, 119 | 120 | // A list of paths to directories that Jest should use to search for files in 121 | // roots: [ 122 | // "" 123 | // ], 124 | 125 | // Allows you to use a custom runner instead of Jest's default test runner 126 | // runner: "jest-runner", 127 | 128 | // The paths to modules that run some code to configure or set up the testing environment before each test 129 | // setupFiles: [], 130 | 131 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 132 | // setupFilesAfterEnv: [], 133 | 134 | // The number of seconds after which a test is considered as slow and reported as such in the results. 135 | // slowTestThreshold: 5, 136 | 137 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 138 | // snapshotSerializers: [], 139 | 140 | // The test environment that will be used for testing 141 | testEnvironment: 'node', 142 | 143 | // Options that will be passed to the testEnvironment 144 | // testEnvironmentOptions: {}, 145 | 146 | // Adds a location field to test results 147 | // testLocationInResults: false, 148 | 149 | // The glob patterns Jest uses to detect test files 150 | // testMatch: [ 151 | // "**/__tests__/**/*.[jt]s?(x)", 152 | // "**/?(*.)+(spec|test).[tj]s?(x)" 153 | // ], 154 | 155 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 156 | // testPathIgnorePatterns: [ 157 | // "/node_modules/" 158 | // ], 159 | 160 | // The regexp pattern or array of patterns that Jest uses to detect test files 161 | // testRegex: [], 162 | 163 | // This option allows the use of a custom results processor 164 | // testResultsProcessor: undefined, 165 | 166 | // This option allows use of a custom test runner 167 | // testRunner: "jasmine2", 168 | 169 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 170 | // testURL: "http://localhost", 171 | 172 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 173 | // timers: "real", 174 | 175 | // A map from regular expressions to paths to transformers 176 | // transform: undefined, 177 | 178 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 179 | // transformIgnorePatterns: [ 180 | // "/node_modules/", 181 | // "\\.pnp\\.[^\\/]+$" 182 | // ], 183 | 184 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 185 | // unmockedModulePathPatterns: undefined, 186 | 187 | // Indicates whether each individual test should be reported during the run 188 | // verbose: undefined, 189 | 190 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 191 | // watchPathIgnorePatterns: [], 192 | 193 | // Whether to use watchman for file crawling 194 | // watchman: true, 195 | }; 196 | -------------------------------------------------------------------------------- /kafka-penguin/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 OSLabs Beta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /kafka-penguin/README.md: -------------------------------------------------------------------------------- 1 | # Home 2 | 3 | ![license](https://img.shields.io/github/license/oslabs-beta/kafka-penguin?color=%2357d3af) ![issues](https://img.shields.io/github/issues-raw/oslabs-beta/kafka-penguin?color=yellow) ![last commit](https://img.shields.io/github/last-commit/oslabs-beta/kafka-penguin?color=%2357d3af)​ [![Actions Status](https://github.com/oslabs-beta/kafka-penguin/workflows/CI/CD%20with%20Github%20Actions/badge.svg)](https://github.com/oslabs-beta/kafka-penguin/actions) [​![npm version](https://img.shields.io/npm/v/kafka-penguin?color=%2344cc11&label=stable)​](https://www.npmjs.com/package/kafka-penguin)​‌ 4 | 5 | ### About 6 | 7 | Kafka-Penguin is an easy-to-use, lightweight KafkaJS plugin for message re-processing. It provides developers with three strategies for setting up message re-processing: FailFast, Ignore, and Dead Letter Queue.‌ 8 | 9 | The package allows developers to build event-driven applications with dedicated "fail strategies" modeled after best practices in the field. This in turn allows developers to effectively address bugs in development and deploy more fault-tolerant systems in production.‌ 10 | 11 | This package is meant to work in conjunction with with KafkaJS. For more information on KafkaJS, check out [Getting Started with KafkaJS](https://kafka.js.org/docs/getting-started).‌ 12 | 13 | ### Getting Started 14 | 15 | Install Kafka-Penguin as an npm module and save it to your package.json file as a dependency: 16 | 17 | ```text 18 | npm install kafka-penguin 19 | ``` 20 | 21 | Once installed it can now be referenced by simply calling `require('kafka-penguin');`‌ 22 | 23 | ## Example 24 | 25 | All Kafka-Penguin needs is a KafkaJS client to run. Start by passing the client for your preferred strategy and Kafka-Penguin will create bespoke consumers, producers, and admins with built-in functionality to execute the chosen strategy. On the surface, you implement your application exactly as you would with KafkaJS. 26 | 27 | ```text 28 | /* eslint-disable no-console */ 29 | import { FailFast } from 'kafka-penguin'; 30 | 31 | const exampleClient = require('./clientConfig.ts'); 32 | 33 | // Set up the preferred strategy with a configured KafkaJS client 34 | const exampleStrategy = new FailFast(2, exampleClient); 35 | 36 | // Initialize a producer or consumer from the instance of the strategy 37 | const producer = exampleStrategy.producer(); 38 | 39 | const message = { 40 | topic: 'wrong-topic', 41 | messages: [ 42 | { 43 | key: 'hello', 44 | value: 'world', 45 | }, 46 | ], 47 | }; 48 | 49 | // Connect, Subscribe, Send, or Run virtually the same as with KafkaJS 50 | producer.connect() 51 | .then(() => console.log('Connected!')) 52 | // The chosen strategy executes under the hood, like in this send method 53 | .then(() => producer.send(message)) 54 | .catch((e: any) => console.log('error: ', e.message)); 55 | ``` 56 | 57 | ## Strategies 58 | 59 | Dive in deeper to any of the strategies for set up, execution, and implementation. 60 | 61 | ​[FailFast](https://app.gitbook.com/@kafka-penguin-1/s/kafka-penguin/~/drafts/-MYCUDw3CJmXz95ljr5N/strategies/readme/strategies-readme-fail-fast/@merged) ​‌ 62 | 63 | ​[Ignore](https://app.gitbook.com/@kafka-penguin-1/s/kafka-penguin/~/drafts/-MYCUDw3CJmXz95ljr5N/strategies/readme/strategies-readme-ignore/@merged)​‌ 64 | 65 | ​[Dead Letter Queue](https://app.gitbook.com/@kafka-penguin-1/s/kafka-penguin/~/drafts/-MYCUDw3CJmXz95ljr5N/strategies/readme/strategies-readme-dlq/@merged)​‌ 66 | 67 | ## **Contributors** 68 | 69 | ​[Ausar English](https://www.linkedin.com/in/ausarenglish) [@ausarenglish](https://github.com/ausarenglish)​‌ 70 | 71 | ​[Kushal Talele](https://www.linkedin.com/in/kushal-talele-29040820b/) [@ktrane1](https://github.com/ktrane1)​‌ 72 | 73 | ​[Timeo Williams](https://www.linkedin.com/in/timeowilliams/) [@timeowilliams](https://github.com/timeowilliams)​‌ 74 | 75 | ​[Ziyad El Baz](https://www.linkedin.com/in/ziyadelbaz) [@zelbaz946](https://github.com/zelbaz946)​‌ 76 | 77 | ### License 78 | 79 | This product is licensed under the MIT License - see the [LICENSE.md](https://github.com/oslabs-beta/kafka-penguin/blob/main/LICENSE) file for details.‌ 80 | 81 | This is an open source product. We are not affiliated nor endorsed by either the Apache Software Foundation or KafkaJS.‌ 82 | 83 | This product is accelerated by [OS Labs](https://opensourcelabs.io/). 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /kafka-penguin/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'], 3 | }; 4 | -------------------------------------------------------------------------------- /kafka-penguin/dist/clientConfig.d.ts: -------------------------------------------------------------------------------- 1 | declare const kafka: any; 2 | export default kafka; 3 | -------------------------------------------------------------------------------- /kafka-penguin/dist/clientConfig.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | const { Kafka } = require('kafkajs'); 4 | require('dotenv').config(); 5 | // Create the client with the broker list 6 | const kafka = new Kafka({ 7 | clientId: 'fail-fast-producer', 8 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 9 | ssl: true, 10 | sasl: { 11 | mechanism: 'plain', 12 | username: process.env.KAFKA_USERNAME, 13 | password: process.env.KAFKA_PASSWORD, 14 | }, 15 | }); 16 | exports.default = kafka; 17 | -------------------------------------------------------------------------------- /kafka-penguin/dist/index.d.ts: -------------------------------------------------------------------------------- 1 | import { CompressionTypes } from 'kafkajs'; 2 | interface messageValue { 3 | topic: string; 4 | messages: object[]; 5 | } 6 | export declare class FailFastError extends Error { 7 | message: any; 8 | reference: any; 9 | name: any; 10 | retryCount: number; 11 | strategy: string; 12 | originalError: any; 13 | constructor(e: any); 14 | } 15 | export declare class FailFast { 16 | retry: number; 17 | client: any; 18 | innerProducer: any; 19 | constructor(num: number, kafkaJSClient: any); 20 | producer(): this; 21 | connect(): any; 22 | disconnect(): any; 23 | send(message: messageValue): any; 24 | } 25 | export declare class DeadLetterQueueErrorConsumer extends Error { 26 | message: any; 27 | reference: any; 28 | name: any; 29 | retryCount: number; 30 | strategy: string; 31 | originalError: any; 32 | constructor(e: any); 33 | } 34 | export declare class DeadLetterQueueErrorProducer extends Error { 35 | message: any; 36 | reference: any; 37 | name: any; 38 | retryCount: number; 39 | strategy: string; 40 | originalError: any; 41 | constructor(e: any); 42 | } 43 | export declare class DeadLetterQueue { 44 | client: any; 45 | topic: string; 46 | callback?: (message: any) => boolean; 47 | innerConsumer: any; 48 | admin: any; 49 | innerProducer: any; 50 | constructor(client: any, topic: string, callback?: any); 51 | producer(): any; 52 | consumer(groupId: { 53 | groupId: string; 54 | }): any; 55 | createDLQ(): Promise; 56 | } 57 | export declare class IgnoreErrorProducer extends Error { 58 | message: any; 59 | reference: any; 60 | name: any; 61 | retryCount: number; 62 | strategy: string; 63 | originalError: any; 64 | constructor(e: any); 65 | } 66 | export declare class IgnoreErrorConsumer extends Error { 67 | message: any; 68 | reference: any; 69 | name: any; 70 | retryCount: number; 71 | strategy: string; 72 | originalError: any; 73 | constructor(e: any); 74 | } 75 | interface messageValue { 76 | acks?: Number; 77 | timeout?: Number; 78 | compression?: CompressionTypes; 79 | topic: string; 80 | messages: object[]; 81 | } 82 | export default class Ignore { 83 | client: any; 84 | topic: string; 85 | callback?: (message: any) => boolean; 86 | innerConsumer: any; 87 | admin: any; 88 | innerProducer: any; 89 | constructor(client: any, topic: string, callback?: any); 90 | producer(): any; 91 | consumer(groupId: { 92 | groupId: string; 93 | }): any; 94 | } 95 | export {}; 96 | -------------------------------------------------------------------------------- /kafka-penguin/dist/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | Object.defineProperty(exports, "__esModule", { value: true }); 12 | exports.IgnoreErrorConsumer = exports.IgnoreErrorProducer = exports.DeadLetterQueue = exports.DeadLetterQueueErrorProducer = exports.DeadLetterQueueErrorConsumer = exports.FailFast = exports.FailFastError = void 0; 13 | // Fail Fast Strategy 14 | class FailFastError extends Error { 15 | constructor(e) { 16 | super(e); 17 | Error.captureStackTrace(this, this.constructor); 18 | this.strategy = 'Fail Fast'; 19 | this.reference = `This error was executed as part of the kafka-penguin Fail Fast message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful. As a result, the producer successfully executed a disconnect operation. Refer to the original error for further information`; 20 | this.name = e.name; 21 | this.message = e.message; 22 | this.originalError = e.originalError; 23 | this.retryCount = e.retryCount; 24 | } 25 | } 26 | exports.FailFastError = FailFastError; 27 | class FailFast { 28 | constructor(num, kafkaJSClient) { 29 | this.retry = num; 30 | this.client = kafkaJSClient; 31 | this.innerProducer = null; 32 | } 33 | producer() { 34 | const options = { 35 | retry: { retries: this.retry }, 36 | }; 37 | // Create a producer from client passing in retry options 38 | // Save to FailFast class 39 | this.innerProducer = this.client.producer(options); 40 | // Return curr FailFast instance instead of a producer 41 | return this; 42 | } 43 | connect() { 44 | return this.innerProducer.connect(); 45 | } 46 | disconnect() { 47 | return this.innerProducer.disconnect(); 48 | } 49 | send(message) { 50 | return this.innerProducer.send(message) 51 | .catch((e) => { 52 | this.innerProducer.disconnect(); 53 | const newError = new FailFastError(e); 54 | // eslint-disable-next-line no-console 55 | console.log(newError); 56 | }); 57 | } 58 | } 59 | exports.FailFast = FailFast; 60 | // Dead Letter Queue 61 | class DeadLetterQueueErrorConsumer extends Error { 62 | constructor(e) { 63 | super(e); 64 | Error.captureStackTrace(this, this.constructor); 65 | this.strategy = 'Dead Letter Queue'; 66 | this.reference = `This error was executed as part of the kafka-penguin Dead Letter Queue message reprocessing strategy. Your consumer attempted to receive a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 67 | this.name = `${e.name}(Consumer Side)`; 68 | this.message = e.message; 69 | this.originalError = e.originalError; 70 | this.retryCount = e.retryCount; 71 | } 72 | } 73 | exports.DeadLetterQueueErrorConsumer = DeadLetterQueueErrorConsumer; 74 | class DeadLetterQueueErrorProducer extends Error { 75 | constructor(e) { 76 | super(e); 77 | Error.captureStackTrace(this, this.constructor); 78 | this.strategy = 'Dead Letter Queue'; 79 | this.reference = `This error was executed as part of the kafka-penguin Dead Letter Queue message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 80 | this.name = `${e.name}(Producer Side)`; 81 | this.message = e.message; 82 | this.originalError = e.originalError; 83 | this.retryCount = e.retryCount; 84 | } 85 | } 86 | exports.DeadLetterQueueErrorProducer = DeadLetterQueueErrorProducer; 87 | class DeadLetterQueue { 88 | constructor(client, topic, callback) { 89 | this.topic = topic; 90 | this.client = client; 91 | this.callback = callback; 92 | this.admin = this.client.admin(); 93 | this.innerConsumer = null; 94 | this.innerProducer = this.client.producer(); 95 | } 96 | producer() { 97 | // Reference the DLQ instance for closure in the returned object 98 | const dlqInstance = this; 99 | const { innerProducer } = dlqInstance; 100 | // Return an object with all Producer methods adapted to execute a dead letter queue strategy 101 | console.log('INNER PRODUCER', innerProducer); 102 | return Object.assign(Object.assign({}, innerProducer), { connect() { 103 | return innerProducer.connect() 104 | .then(() => { 105 | dlqInstance.createDLQ(); 106 | }) 107 | .catch((e) => console.log(e)); 108 | }, 109 | send(message) { 110 | return innerProducer.connect() 111 | .then(() => { 112 | innerProducer.send(Object.assign(Object.assign({}, message), { topic: message.topic, messages: message.messages })) 113 | // Upon error, reroute message to DLQ for the strategy topic 114 | .catch((e) => { 115 | innerProducer.send({ 116 | messages: message.messages, 117 | topic: `${dlqInstance.topic}.deadLetterQueue`, 118 | }) 119 | .then(innerProducer.disconnect()) 120 | .catch((e) => console.log(e)); 121 | // Print the error to the console 122 | const newError = new DeadLetterQueueErrorProducer(e); 123 | console.log(newError); 124 | }); 125 | }); 126 | } }); 127 | } 128 | consumer(groupId) { 129 | this.innerConsumer = this.client.consumer(groupId); 130 | const dlqInstance = this; 131 | const { innerConsumer, innerProducer } = dlqInstance; 132 | // Returns an object with all Consumer methods adapter to execute a dead letter queue strategy 133 | return Object.assign(Object.assign({}, innerConsumer), { connect() { 134 | return innerConsumer.connect().then(() => { 135 | dlqInstance.createDLQ(); 136 | }); 137 | }, subscribe(input) { 138 | return innerConsumer.subscribe(Object.assign(Object.assign({}, input), { topic: dlqInstance.topic, fromBeginning: false })); 139 | }, 140 | run(input) { 141 | const { eachMessage } = input; 142 | return innerConsumer.run(Object.assign(Object.assign({}, input), { eachMessage: ({ topic, partitions, message }) => { 143 | try { 144 | // If user doesn't pass in callback, DLQ simply listens and returns errors 145 | if (dlqInstance.callback) { 146 | if (!dlqInstance.callback(message)) 147 | throw Error; 148 | eachMessage({ topic, partitions, message }); 149 | } 150 | } 151 | catch (e) { 152 | const newError = new DeadLetterQueueErrorConsumer(e); 153 | console.error(newError); 154 | innerProducer.connect() 155 | .then(() => console.log('kafka-penguin: Connected to DLQ topic')) 156 | .then(() => { 157 | innerProducer.send({ 158 | topic: `${dlqInstance.topic}.deadLetterQueue`, 159 | messages: [message], 160 | }); 161 | }) 162 | .then(() => console.log('kafka-penguin: Message published to DLQ')) 163 | .then(() => innerProducer.disconnect()) 164 | .then(() => console.log('kafka-penguin: Producer disconnected')) 165 | .catch((e) => console.log('Error with producing to DLQ: ', e)); 166 | } 167 | } })); 168 | } }); 169 | } 170 | // Creates a new DLQ topic with the original topic name 171 | createDLQ() { 172 | return __awaiter(this, void 0, void 0, function* () { 173 | const adminCreateDLQ = yield this.admin.connect() 174 | .then(() => __awaiter(this, void 0, void 0, function* () { 175 | yield this.admin.createTopics({ 176 | topics: [{ 177 | topic: `${this.topic}.deadLetterQueue`, 178 | numPartitions: 1, 179 | replicationFactor: 1, 180 | replicaAssignment: [{ partition: 0, replicas: [0, 1, 2] }], 181 | }], 182 | }); 183 | })) 184 | .then(() => this.admin.disconnect()) 185 | .catch((err) => console.log('Error from createDLQ', err)); 186 | return adminCreateDLQ; 187 | }); 188 | } 189 | } 190 | exports.DeadLetterQueue = DeadLetterQueue; 191 | // Ignore 192 | class IgnoreErrorProducer extends Error { 193 | constructor(e) { 194 | super(e); 195 | Error.captureStackTrace(this, this.constructor); 196 | this.strategy = 'Ignore'; 197 | this.reference = `This error was executed as part of the kafka-penguin Ignore message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful.`; 198 | this.name = `${e.name} (Producer Side)`; 199 | this.message = e.message; 200 | this.originalError = e.originalError; 201 | this.retryCount = e.retryCount; 202 | } 203 | } 204 | exports.IgnoreErrorProducer = IgnoreErrorProducer; 205 | class IgnoreErrorConsumer extends Error { 206 | constructor(e) { 207 | super(e); 208 | Error.captureStackTrace(this, this.constructor); 209 | this.strategy = 'Ignore'; 210 | this.reference = `This error was executed as part of the kafka-penguin Ignore message reprocessing strategy. Your consumer attempted to receive a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 211 | this.name = `${e.name} (Consumer Side)`; 212 | this.message = e.message; 213 | this.originalError = e.originalError; 214 | this.retryCount = e.retryCount; 215 | } 216 | } 217 | exports.IgnoreErrorConsumer = IgnoreErrorConsumer; 218 | class Ignore { 219 | constructor(client, topic, callback) { 220 | this.topic = topic; 221 | this.client = client; 222 | this.callback = callback; 223 | this.admin = this.client.admin(); 224 | this.innerConsumer = null; 225 | this.innerProducer = this.client.producer(); 226 | } 227 | producer() { 228 | // Reference the Ignore instance for closure in the returned object 229 | const ignoreInstance = this; 230 | const { innerProducer } = ignoreInstance; 231 | // Return an object with all Producer methods adapted to execute Ignore strategy 232 | console.log('INNER PRODUCER', innerProducer); 233 | return Object.assign(Object.assign({}, innerProducer), { connect() { 234 | return innerProducer.connect() 235 | .catch((e) => console.log(e)); 236 | }, 237 | send(message) { 238 | return innerProducer.connect() 239 | .then(() => { 240 | innerProducer.send(Object.assign(Object.assign({}, message), { topic: message.topic, messages: message.messages })) 241 | .catch((e) => { 242 | console.log(e); 243 | // Print the error to the console 244 | const newError = new IgnoreErrorProducer(e); 245 | console.log(newError); 246 | }); 247 | }); 248 | } }); 249 | } 250 | consumer(groupId) { 251 | this.innerConsumer = this.client.consumer(groupId); 252 | const ignoreInstance = this; 253 | const { innerConsumer } = ignoreInstance; 254 | // Returns an object with all Consumer methods adapter to execute ignore strategy 255 | return Object.assign(Object.assign({}, innerConsumer), { connect() { 256 | return innerConsumer.connect(); 257 | }, subscribe(input) { 258 | return innerConsumer.subscribe(Object.assign(Object.assign({}, input), { topic: ignoreInstance.topic, fromBeginning: false })); 259 | }, 260 | run(input) { 261 | const { eachMessage } = input; 262 | return innerConsumer.run(Object.assign(Object.assign({}, input), { eachMessage: ({ topic, partitions, message }) => { 263 | try { 264 | // If user doesn't pass in callback, DLQ simply listens and returns errors 265 | if (ignoreInstance.callback) { 266 | if (!ignoreInstance.callback(message)) 267 | throw Error; 268 | eachMessage({ topic, partitions, message }); 269 | } 270 | } 271 | catch (e) { 272 | const newError = new IgnoreErrorConsumer(e); 273 | console.error(newError); 274 | } 275 | } })); 276 | } }); 277 | } 278 | } 279 | exports.default = Ignore; 280 | -------------------------------------------------------------------------------- /kafka-penguin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-penguin", 3 | "version": "0.3.8", 4 | "description": "An easy-to-use, light weight KafkaJS library for message re-processing strategies.", 5 | "main": "./dist/index.js", 6 | "scripts": { 7 | "start": "webpack serve --open", 8 | "build": "tsc", 9 | "dev": "concurrently \"cross-env NODE_ENV=development webpack serve --config webpack.config.js --mode development\" \"nodemon demo/server/server.ts\"", 10 | "test": "jest", 11 | "lint": "eslint . --ext .ts" 12 | }, 13 | "keywords": [ 14 | "Kafka", 15 | "Message Re-processing", 16 | "Micro-services", 17 | "Event-driven", 18 | "Dead Letter Queue" 19 | ], 20 | "author": "", 21 | "license": "MIT", 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/oslabs-beta/kafka-penguin" 25 | }, 26 | "dependencies": { 27 | "@babel/plugin-syntax-jsx": "^7.12.13", 28 | "@babel/plugin-transform-runtime": "^7.13.10", 29 | "dotenv": "^8.2.0", 30 | "kafkajs": "^1.15.0", 31 | "regenerator-runtime": "^0.13.7", 32 | "typescript": "^3.0.0", 33 | "util": "^0.12.3" 34 | }, 35 | "devDependencies": { 36 | "@babel/core": "^7.13.15", 37 | "@babel/preset-env": "^7.13.15", 38 | "@babel/preset-react": "^7.13.13", 39 | "@babel/preset-typescript": "^7.13.0", 40 | "@types/express": "^4.17.11", 41 | "@types/jest": "^26.0.22", 42 | "@types/mocha": "^8.2.2", 43 | "@typescript-eslint/eslint-plugin": "^4.19.0", 44 | "@typescript-eslint/parser": "^4.19.0", 45 | "@webpack-cli/serve": "^1.3.0", 46 | "awesome-typescript-loader": "^5.2.1", 47 | "babel-cli": "^6.26.0", 48 | "babel-jest": "^26.6.3", 49 | "babel-loader": "^8.2.2", 50 | "babel-preset-env": "^1.7.0", 51 | "eslint": "^7.23.0", 52 | "eslint-config-airbnb-typescript": "^12.3.1", 53 | "eslint-plugin-import": "^2.22.1", 54 | "eslint-plugin-jsx-a11y": "^6.4.1", 55 | "eslint-plugin-prettier": "^3.3.1", 56 | "html-webpack-plugin": "^5.3.1", 57 | "jest": "^26.6.3", 58 | "nodemon": "^2.0.7", 59 | "source-map-loader": "^2.0.1", 60 | "style-loader": "^2.0.0", 61 | "superagent": "^6.1.0", 62 | "supertest": "^6.1.3", 63 | "ts-jest": "^26.5.4", 64 | "ts-loader": "^8.0.18", 65 | "typescript": "^3.0.0" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /kafka-penguin/src/clientConfig.ts: -------------------------------------------------------------------------------- 1 | const { Kafka } = require('kafkajs') 2 | require('dotenv').config(); 3 | 4 | // Create the client with the broker list 5 | const kafka = new Kafka({ 6 | clientId: 'fail-fast-producer', 7 | brokers: [process.env.KAFKA_BOOTSTRAP_SERVER], 8 | ssl: true, 9 | sasl: { 10 | mechanism: 'plain', 11 | username: process.env.KAFKA_USERNAME, 12 | password: process.env.KAFKA_PASSWORD, 13 | }, 14 | }) 15 | 16 | export default kafka 17 | -------------------------------------------------------------------------------- /kafka-penguin/src/deadletterqueue.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-shadow */ 2 | /* eslint-disable import/extensions */ 3 | /* eslint-disable import/no-unresolved */ 4 | /* eslint-disable no-unused-vars */ 5 | /* eslint-disable no-console */ 6 | /* eslint-disable no-undef */ 7 | import { DeadLetterQueue, DeadLetterQueueErrorConsumer, DeadLetterQueueErrorProducer } from './index'; 8 | import testClient from './clientConfig'; 9 | 10 | // Dead Letter Queue Tests 11 | describe('Dead Letter Queue Tests', () => { 12 | // Constructor Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 13 | describe('Constructor', () => { 14 | const testInstance = new DeadLetterQueue(testClient, 'test1', () => true); 15 | const mockClient = { 16 | topic: expect.any(String), 17 | innerProducer: expect.any(Object), 18 | callback: expect.any(Function), 19 | innerConsumer: null, 20 | admin: expect.any(Object), 21 | client: expect.any(Object), 22 | }; 23 | 24 | describe('Initial State', () => { 25 | it('Starts off with callback, topic, and client supplied', () => { 26 | expect(testInstance).toBeInstanceOf(DeadLetterQueue); 27 | expect(testInstance).toMatchObject(mockClient); 28 | }); 29 | }); 30 | 31 | describe('Client is live & configured', () => { 32 | it('Client is supplying the class with producers', () => { 33 | const { client } = testInstance; 34 | const producer = jest.fn(() => client.producer()); 35 | producer(); 36 | expect(producer).toReturnWith(expect.objectContaining({ 37 | send: expect.any(Function), 38 | connect: expect.any(Function), 39 | disconnect: expect.any(Function), 40 | })); 41 | }); 42 | it('Client is supplying the class with consumers', () => { 43 | const { client } = testInstance; 44 | const consumer = jest.fn(() => client.consumer({ groupId: 'my-group' })); 45 | consumer(); 46 | expect(consumer).toReturnWith(expect.objectContaining({ 47 | subscribe: expect.any(Function), 48 | run: expect.any(Function), 49 | })); 50 | }); 51 | it('Client is supplying the class with admins', () => { 52 | const { client } = testInstance; 53 | const admin = jest.fn(() => client.admin()); 54 | admin(); 55 | expect(admin).toReturnWith(expect.any(Object)); 56 | }); 57 | }); 58 | }); 59 | 60 | describe('Methods', () => { 61 | let testInstance = new DeadLetterQueue(testClient, 'test1', () => true); 62 | 63 | afterEach(() => { 64 | testInstance = new DeadLetterQueue(testClient, 'test1', () => true); 65 | }); 66 | 67 | // Producer Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | describe('Producer', () => { 70 | const testingProducer = jest.fn(() => testInstance.producer()); 71 | 72 | describe('Returns/SideEffects', () => { 73 | it('returns the DLQ instance', () => { 74 | testingProducer(); 75 | expect(testInstance.producer()).toMatchObject(expect.objectContaining({ 76 | connect: expect.any(Function), 77 | disconnect: expect.any(Function), 78 | send: expect.any(Function), 79 | })); 80 | }); 81 | it('Assigns the producer to a instance of client producer', () => { 82 | testingProducer(); 83 | expect(testInstance.innerProducer).toEqual(expect.objectContaining({ 84 | send: expect.any(Function), 85 | connect: expect.any(Function), 86 | disconnect: expect.any(Function), 87 | })); 88 | }); 89 | }); 90 | }); 91 | 92 | // Consumer Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 93 | 94 | describe('Consumer', () => { 95 | const id = { groupId: 'Jest Tests' }; 96 | const testingConsumer = jest.fn(() => testInstance.consumer(id)); 97 | 98 | describe('Returns/SideEffects', () => { 99 | it('returns the Dead Letter Queue instance', () => { 100 | testingConsumer(); 101 | expect(testInstance.consumer(id)).toMatchObject(expect.objectContaining({ 102 | connect: expect.any(Function), 103 | disconnect: expect.any(Function), 104 | run: expect.any(Function), 105 | subscribe: expect.any(Function), 106 | })); 107 | }); 108 | it('Assigns the producer to a instance of client producer', () => { 109 | testingConsumer(); 110 | expect(testInstance.innerConsumer).toEqual(expect.objectContaining({ 111 | connect: expect.any(Function), 112 | disconnect: expect.any(Function), 113 | run: expect.any(Function), 114 | subscribe: expect.any(Function), 115 | })); 116 | }); 117 | }); 118 | }); 119 | 120 | // Producer Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 121 | 122 | describe('Producer Methods', () => { 123 | const testingProducer = jest.fn(() => testInstance.producer()); 124 | 125 | describe('Connect', () => { 126 | describe('Returns/SideEffects', () => { 127 | it('returns the client producer connect method', () => { 128 | testingProducer(); 129 | expect(testingProducer).toReturn(); 130 | }); 131 | it('connect method resolves', () => { 132 | testingProducer(); 133 | const { innerProducer } = testInstance; 134 | return innerProducer.connect() 135 | .then((input: any) => expect(input).not.toBeNull()) 136 | .finally(() => { innerProducer.disconnect(); }); 137 | }); 138 | }); 139 | }); 140 | 141 | describe('Send', () => { 142 | describe('Returns/SideEffects', () => { 143 | it('throws a DLQ Error with a bad message, then sends it to the DLQ', async () => { 144 | testingProducer(); 145 | const { innerProducer } = testInstance; 146 | const message = { 147 | topic: 'wrong-topic', 148 | messages: [{ 149 | key: 'value', 150 | value: 'key', 151 | }], 152 | }; 153 | 154 | return innerProducer.send(message) 155 | .catch((e?: any) => { 156 | innerProducer.send({ 157 | messages: message.messages, 158 | topic: `${testInstance.topic}.deadLetterQueue`, 159 | }) 160 | .then(innerProducer.disconnect()) 161 | .catch((e: Error) => console.log(e)); 162 | // Print the error to the console 163 | const newError = new DeadLetterQueueErrorProducer(e); 164 | console.log(newError); 165 | }).finally(() => { innerProducer.disconnect(); }); 166 | }); 167 | it('disconnects the producer with an bad message', () => { 168 | testingProducer(); 169 | const { innerProducer } = testInstance; 170 | const message = { 171 | topic: 'wrong-topic', 172 | messages: [{ 173 | key: 'value', 174 | value: 'key', 175 | }], 176 | }; 177 | return innerProducer.send(message).catch((e: any) => { 178 | expect(e).toBeInstanceOf(Error); 179 | }).finally(() => { innerProducer.disconnect(); }); 180 | }); 181 | }); 182 | }); 183 | 184 | describe('Disconnect', () => { 185 | describe('Returns/SideEffects', () => { 186 | it('returns the client producer disconnect method & disconnects successfully', () => { 187 | testingProducer(); 188 | const { innerProducer } = testInstance; 189 | expect(innerProducer.disconnect).toEqual(expect.any(Function)); 190 | return innerProducer.disconnect() 191 | .then((input: any) => expect(input).not.toBeNull()) 192 | .finally(() => { innerProducer.disconnect(); }); 193 | }); 194 | }); 195 | }); 196 | }); 197 | 198 | // Consumer Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 199 | 200 | describe('Consumer Methods', () => { 201 | const group = { groupId: 'JestTests' }; 202 | const testingConsumer = jest.fn(() => testInstance.consumer(group)); 203 | 204 | describe('Connect', () => { 205 | describe('Returns/SideEffects', () => { 206 | it('returns the client consumer connect method', () => { 207 | testingConsumer(); 208 | expect(testingConsumer).toReturn(); 209 | }); 210 | it('connect method resolves', () => { 211 | testingConsumer(); 212 | const { innerConsumer } = testInstance; 213 | return innerConsumer.connect() 214 | .then((input: any) => expect(input).not.toBeNull()) 215 | .finally(() => { innerConsumer.disconnect(); }); 216 | }); 217 | }); 218 | }); 219 | 220 | describe('Send', () => { 221 | const run = jest.fn(async (input) => { 222 | await testInstance.innerConsumer.connect() 223 | .then(() => { 224 | testInstance.innerConsumer.subscribe({ 225 | topic: testInstance.topic, 226 | fromBeginning: false, 227 | }); 228 | }) 229 | .then(() => { 230 | testInstance.innerConsumer.run(input); 231 | }) 232 | .catch((e: any) => { expect(e).toBeInstanceOf(Error); }) 233 | .finally(() => testInstance.innerConsumer.disconnect()); 234 | }); 235 | 236 | describe('Returns/SideEffects', () => { 237 | it('throws a DLQ Error with a bad message, then sends it to the DLQ', async () => { 238 | testingConsumer(); 239 | return run({ 240 | eachMessage: ({ topic, partitions, message }: { 241 | topic: any, partitions: any, message: any 242 | }) => false, 243 | }); 244 | }); 245 | }); 246 | }); 247 | 248 | describe('Disconnect', () => { 249 | describe('Returns/SideEffects', () => { 250 | it('returns the client producer disconnect method & disconnects successfully', () => { 251 | testingConsumer(); 252 | const { innerConsumer } = testInstance; 253 | expect(innerConsumer.disconnect).toEqual(expect.any(Function)); 254 | return innerConsumer.disconnect() 255 | .then((input: any) => expect(input).not.toBeNull()) 256 | .finally(() => { innerConsumer.disconnect(); }); 257 | }); 258 | }); 259 | }); 260 | }); 261 | }); 262 | }); 263 | -------------------------------------------------------------------------------- /kafka-penguin/src/failfast.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/extensions */ 2 | /* eslint-disable import/no-unresolved */ 3 | /* eslint-disable no-console */ 4 | /* eslint-disable no-undef */ 5 | import { FailFast, FailFastError } from './index'; 6 | import testClient from './clientConfig'; 7 | 8 | // Fail Fast Tests 9 | describe('FailFast Tests', () => { 10 | describe('Constructor', () => { 11 | const testInstance = new FailFast(3, testClient); 12 | const mockClient = { 13 | retry: expect.any(Number), 14 | innerProducer: null, 15 | client: expect.any(Object), 16 | }; 17 | 18 | describe('Initial State', () => { 19 | it('Starts off with retries and client supplied', () => { 20 | expect(testInstance).toBeInstanceOf(FailFast); 21 | expect(testInstance).toMatchObject(mockClient); 22 | }); 23 | }); 24 | 25 | describe('Client is live & configured', () => { 26 | it('Client is supplying the class with producers', () => { 27 | const { client } = testInstance; 28 | const producer = jest.fn(() => client.producer()); 29 | producer(); 30 | expect(producer).toReturnWith(expect.objectContaining({ 31 | send: expect.any(Function), 32 | connect: expect.any(Function), 33 | disconnect: expect.any(Function), 34 | })); 35 | }); 36 | it('Client is supplying the class with consumers', () => { 37 | const { client } = testInstance; 38 | const consumer = jest.fn(() => client.consumer({ groupId: 'my-group' })); 39 | consumer(); 40 | expect(consumer).toReturnWith(expect.objectContaining({ 41 | subscribe: expect.any(Function), 42 | run: expect.any(Function), 43 | })); 44 | }); 45 | it('Client is supplying the class with admins', () => { 46 | const { client } = testInstance; 47 | const admin = jest.fn(() => client.admin()); 48 | admin(); 49 | expect(admin).toReturnWith(expect.any(Object)); 50 | }); 51 | }); 52 | }); 53 | 54 | describe('Methods', () => { 55 | let testInstance = new FailFast(3, testClient); 56 | 57 | afterEach(() => { 58 | testInstance = new FailFast(3, testClient); 59 | }); 60 | 61 | describe('Producer', () => { 62 | const testingProducer = jest.fn(() => testInstance.producer()); 63 | 64 | describe('Returns/SideEffects', () => { 65 | it('returns the FailFast instance', () => { 66 | testingProducer(); 67 | expect(testInstance.producer()).toBe(testInstance); 68 | }); 69 | it('Assigns the producer to a instance of client producer', () => { 70 | testingProducer(); 71 | expect(testInstance.innerProducer).toEqual(expect.objectContaining({ 72 | send: expect.any(Function), 73 | connect: expect.any(Function), 74 | disconnect: expect.any(Function), 75 | })); 76 | }); 77 | }); 78 | }); 79 | 80 | describe('Producer Methods', () => { 81 | const testingProducer = jest.fn(() => testInstance.producer()); 82 | 83 | describe('Connect', () => { 84 | describe('Returns/SideEffects', () => { 85 | it('returns the client producer connect method', () => { 86 | testingProducer(); 87 | expect(testingProducer).toReturn(); 88 | }); 89 | it('connect method resolves', () => { 90 | testingProducer(); 91 | const { innerProducer } = testInstance; 92 | return innerProducer.connect() 93 | .then((input: any) => expect(input).not.toBeNull()) 94 | .finally(() => { innerProducer.disconnect(); }); 95 | }); 96 | }); 97 | }); 98 | 99 | describe('Send', () => { 100 | const message = { 101 | topic: 'xyz', 102 | messages: [{ 103 | key: 'xyz', 104 | value: 'xyz', 105 | }], 106 | }; 107 | const send = jest.fn((msg) => { testInstance.send(msg); }); 108 | 109 | describe('Inputs', () => { 110 | it('takes in a message with the message value interface', () => { 111 | testingProducer(); 112 | send(message); 113 | expect(send).toHaveBeenCalledWith(expect.objectContaining(message)); 114 | }); 115 | }); 116 | describe('Returns/SideEffects', () => { 117 | it('throws a FailFast Error with a bad message', async () => { 118 | testingProducer(); 119 | const { innerProducer } = testInstance; 120 | 121 | return innerProducer.send(message) 122 | .catch((e: any) => { 123 | innerProducer.disconnect(); 124 | const newError = new FailFastError(e); 125 | console.log(newError); 126 | expect(newError).toBeInstanceOf(FailFastError); 127 | }).finally(() => { innerProducer.disconnect(); }); 128 | }); 129 | it('disconnects the producer with an bad message', () => { 130 | testingProducer(); 131 | const { innerProducer } = testInstance; 132 | return innerProducer.send(message).catch((e: any) => { 133 | expect(e).toBeInstanceOf(Error); 134 | }).finally(() => { innerProducer.disconnect(); }); 135 | }); 136 | }); 137 | }); 138 | 139 | describe('Disconnect', () => { 140 | describe('Returns/SideEffects', () => { 141 | it('returns the client producer disconnect method & disconnects successfully', () => { 142 | testingProducer(); 143 | const { innerProducer } = testInstance; 144 | expect(innerProducer.disconnect).toEqual(expect.any(Function)); 145 | return innerProducer.disconnect().then((input: any) => expect(input).not.toBeNull()); 146 | }); 147 | }); 148 | }); 149 | }); 150 | }); 151 | }); 152 | -------------------------------------------------------------------------------- /kafka-penguin/src/ignore.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-shadow */ 2 | /* eslint-disable import/extensions */ 3 | /* eslint-disable import/no-unresolved */ 4 | /* eslint-disable no-unused-vars */ 5 | /* eslint-disable no-console */ 6 | /* eslint-disable no-undef */ 7 | import { Ignore, IgnoreErrorConsumer, IgnoreErrorProducer} from './index'; 8 | import testClient from './clientConfig'; 9 | 10 | // Dead Letter Queue Tests 11 | describe('Dead Letter Queue Tests', () => { 12 | // Constructor Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 13 | describe('Constructor', () => { 14 | const testInstance = new Ignore(testClient, 'test1', () => true); 15 | const mockClient = { 16 | topic: expect.any(String), 17 | innerProducer: expect.any(Object), 18 | callback: expect.any(Function), 19 | innerConsumer: null, 20 | admin: expect.any(Object), 21 | client: expect.any(Object), 22 | }; 23 | 24 | describe('Initial State', () => { 25 | it('Starts off with callback, topic, and client supplied', () => { 26 | expect(testInstance).toBeInstanceOf(Ignore); 27 | expect(testInstance).toMatchObject(mockClient); 28 | }); 29 | }); 30 | 31 | describe('Client is live & configured', () => { 32 | it('Client is supplying the class with producers', () => { 33 | const { client } = testInstance; 34 | const producer = jest.fn(() => client.producer()); 35 | producer(); 36 | expect(producer).toReturnWith(expect.objectContaining({ 37 | send: expect.any(Function), 38 | connect: expect.any(Function), 39 | disconnect: expect.any(Function), 40 | })); 41 | }); 42 | it('Client is supplying the class with consumers', () => { 43 | const { client } = testInstance; 44 | const consumer = jest.fn(() => client.consumer({ groupId: 'my-group' })); 45 | consumer(); 46 | expect(consumer).toReturnWith(expect.objectContaining({ 47 | subscribe: expect.any(Function), 48 | run: expect.any(Function), 49 | })); 50 | }); 51 | it('Client is supplying the class with admins', () => { 52 | const { client } = testInstance; 53 | const admin = jest.fn(() => client.admin()); 54 | admin(); 55 | expect(admin).toReturnWith(expect.any(Object)); 56 | }); 57 | }); 58 | }); 59 | 60 | describe('Methods', () => { 61 | let testInstance = new Ignore(testClient, 'test1', () => true); 62 | 63 | afterEach(() => { 64 | testInstance = new Ignore(testClient, 'test1', () => true); 65 | }); 66 | 67 | // Producer Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 68 | 69 | describe('Producer', () => { 70 | const testingProducer = jest.fn(() => testInstance.producer()); 71 | 72 | describe('Returns/SideEffects', () => { 73 | it('returns the DLQ instance', () => { 74 | testingProducer(); 75 | expect(testInstance.producer()).toMatchObject(expect.objectContaining({ 76 | connect: expect.any(Function), 77 | disconnect: expect.any(Function), 78 | send: expect.any(Function), 79 | })); 80 | }); 81 | it('Assigns the producer to a instance of client producer', () => { 82 | testingProducer(); 83 | expect(testInstance.innerProducer).toEqual(expect.objectContaining({ 84 | send: expect.any(Function), 85 | connect: expect.any(Function), 86 | disconnect: expect.any(Function), 87 | })); 88 | }); 89 | }); 90 | }); 91 | 92 | // Consumer Initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 93 | 94 | describe('Consumer', () => { 95 | const id = { groupId: 'Jest Tests' }; 96 | const testingConsumer = jest.fn(() => testInstance.consumer(id)); 97 | 98 | describe('Returns/SideEffects', () => { 99 | it('returns the Dead Letter Queue instance', () => { 100 | testingConsumer(); 101 | expect(testInstance.consumer(id)).toMatchObject(expect.objectContaining({ 102 | connect: expect.any(Function), 103 | disconnect: expect.any(Function), 104 | run: expect.any(Function), 105 | subscribe: expect.any(Function), 106 | })); 107 | }); 108 | it('Assigns the producer to a instance of client producer', () => { 109 | testingConsumer(); 110 | expect(testInstance.innerConsumer).toEqual(expect.objectContaining({ 111 | connect: expect.any(Function), 112 | disconnect: expect.any(Function), 113 | run: expect.any(Function), 114 | subscribe: expect.any(Function), 115 | })); 116 | }); 117 | }); 118 | }); 119 | 120 | // Producer Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 121 | 122 | describe('Producer Methods', () => { 123 | const testingProducer = jest.fn(() => testInstance.producer()); 124 | 125 | describe('Connect', () => { 126 | describe('Returns/SideEffects', () => { 127 | it('returns the client producer connect method', () => { 128 | testingProducer(); 129 | expect(testingProducer).toReturn(); 130 | }); 131 | it('connect method resolves', () => { 132 | testingProducer(); 133 | const { innerProducer } = testInstance; 134 | return innerProducer.connect() 135 | .then((input: any) => expect(input).not.toBeNull()) 136 | .finally(() => { innerProducer.disconnect(); }); 137 | }); 138 | }); 139 | }); 140 | 141 | describe('Send', () => { 142 | describe('Returns/SideEffects', () => { 143 | it('throws a DLQ Error with a bad message, then sends it to the DLQ', async () => { 144 | testingProducer(); 145 | const { innerProducer } = testInstance; 146 | const message = { 147 | topic: 'wrong-topic', 148 | messages: [{ 149 | key: 'value', 150 | value: 'key', 151 | }], 152 | }; 153 | 154 | return innerProducer.send(message) 155 | .catch((e?: any) => { 156 | innerProducer.send({ 157 | messages: message.messages, 158 | topic: `${testInstance.topic}`, 159 | }) 160 | .then(innerProducer.disconnect()) 161 | .catch((e: Error) => console.log(e)); 162 | // Print the error to the console 163 | const newError = new IgnoreErrorProducer(e); 164 | console.log(newError); 165 | }).finally(() => { innerProducer.disconnect(); }); 166 | }); 167 | it('disconnects the producer with an bad message', () => { 168 | testingProducer(); 169 | const { innerProducer } = testInstance; 170 | const message = { 171 | topic: 'wrong-topic', 172 | messages: [{ 173 | key: 'value', 174 | value: 'key', 175 | }], 176 | }; 177 | return innerProducer.send(message).catch((e: any) => { 178 | expect(e).toBeInstanceOf(Error); 179 | }).finally(() => { innerProducer.disconnect(); }); 180 | }); 181 | }); 182 | }); 183 | 184 | describe('Disconnect', () => { 185 | describe('Returns/SideEffects', () => { 186 | it('returns the client producer disconnect method & disconnects successfully', () => { 187 | testingProducer(); 188 | const { innerProducer } = testInstance; 189 | expect(innerProducer.disconnect).toEqual(expect.any(Function)); 190 | return innerProducer.disconnect() 191 | .then((input: any) => expect(input).not.toBeNull()) 192 | .finally(() => { innerProducer.disconnect(); }); 193 | }); 194 | }); 195 | }); 196 | }); 197 | 198 | // Consumer Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 199 | 200 | describe('Consumer Methods', () => { 201 | const group = { groupId: 'JestTests' }; 202 | const testingConsumer = jest.fn(() => testInstance.consumer(group)); 203 | 204 | describe('Connect', () => { 205 | describe('Returns/SideEffects', () => { 206 | it('returns the client consumer connect method', () => { 207 | testingConsumer(); 208 | expect(testingConsumer).toReturn(); 209 | }); 210 | it('connect method resolves', () => { 211 | testingConsumer(); 212 | const { innerConsumer } = testInstance; 213 | return innerConsumer.connect() 214 | .then((input: any) => expect(input).not.toBeNull()) 215 | .finally(() => { innerConsumer.disconnect(); }); 216 | }); 217 | }); 218 | }); 219 | 220 | describe('Send', () => { 221 | const run = jest.fn(async (input) => { 222 | await testInstance.innerConsumer.connect() 223 | .then(() => { 224 | testInstance.innerConsumer.subscribe({ 225 | topic: testInstance.topic, 226 | fromBeginning: false, 227 | }); 228 | }) 229 | .then(() => { 230 | testInstance.innerConsumer.run(input); 231 | }) 232 | .catch((e: any) => { expect(e).toBeInstanceOf(Error); }) 233 | .finally(() => testInstance.innerConsumer.disconnect()); 234 | }); 235 | 236 | describe('Returns/SideEffects', () => { 237 | it('throws a DLQ Error with a bad message, then sends it to the DLQ', async () => { 238 | testingConsumer(); 239 | return run({ 240 | eachMessage: ({ topic, partitions, message }: { 241 | topic: any, partitions: any, message: any 242 | }) => false, 243 | }); 244 | }); 245 | }); 246 | }); 247 | 248 | describe('Disconnect', () => { 249 | describe('Returns/SideEffects', () => { 250 | it('returns the client producer disconnect method & disconnects successfully', () => { 251 | testingConsumer(); 252 | const { innerConsumer } = testInstance; 253 | expect(innerConsumer.disconnect).toEqual(expect.any(Function)); 254 | return innerConsumer.disconnect() 255 | .then((input: any) => expect(input).not.toBeNull()) 256 | .finally(() => { innerConsumer.disconnect(); }); 257 | }); 258 | }); 259 | }); 260 | }); 261 | }); 262 | }); -------------------------------------------------------------------------------- /kafka-penguin/src/index.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-redeclare */ 2 | /* eslint-disable no-shadow */ 3 | /* eslint-disable no-console */ 4 | /* eslint-disable no-unused-vars */ 5 | /* eslint-disable max-classes-per-file */ 6 | import { CompressionTypes } from 'kafkajs'; 7 | 8 | interface messageValue { 9 | topic: string, 10 | messages: object[], 11 | } 12 | // Fail Fast Strategy 13 | 14 | export class FailFastError extends Error { 15 | message: any; 16 | 17 | reference: any; 18 | 19 | name: any; 20 | 21 | retryCount: number; 22 | 23 | strategy: string; 24 | 25 | originalError: any; 26 | 27 | constructor(e: any) { 28 | super(e); 29 | Error.captureStackTrace(this, this.constructor); 30 | this.strategy = 'Fail Fast'; 31 | this.reference = `This error was executed as part of the kafka-penguin Fail Fast message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful. As a result, the producer successfully executed a disconnect operation. Refer to the original error for further information`; 32 | this.name = e.name; 33 | this.message = e.message; 34 | this.originalError = e.originalError; 35 | this.retryCount = e.retryCount; 36 | } 37 | } 38 | 39 | export class FailFast { 40 | retry: number; 41 | 42 | client: any; 43 | 44 | innerProducer: any; 45 | 46 | constructor(num: number, kafkaJSClient: any) { 47 | this.retry = num; 48 | this.client = kafkaJSClient; 49 | this.innerProducer = null; 50 | } 51 | 52 | producer() { 53 | const options = { 54 | retry: 55 | { retries: this.retry }, 56 | }; 57 | // Create a producer from client passing in retry options 58 | // Save to FailFast class 59 | this.innerProducer = this.client.producer(options); 60 | // Return curr FailFast instance instead of a producer 61 | return this; 62 | } 63 | 64 | connect() { 65 | return this.innerProducer.connect(); 66 | } 67 | 68 | disconnect() { 69 | return this.innerProducer.disconnect(); 70 | } 71 | 72 | send(message: messageValue) { 73 | return this.innerProducer.send(message) 74 | .catch((e: any) => { 75 | this.innerProducer.disconnect(); 76 | const newError = new FailFastError(e); 77 | // eslint-disable-next-line no-console 78 | console.log(newError); 79 | }); 80 | } 81 | } 82 | 83 | // Dead Letter Queue 84 | 85 | export class DeadLetterQueueErrorConsumer extends Error { 86 | message: any; 87 | 88 | reference: any; 89 | 90 | name: any; 91 | 92 | retryCount: number; 93 | 94 | strategy: string; 95 | 96 | originalError: any; 97 | 98 | constructor(e: any) { 99 | super(e); 100 | Error.captureStackTrace(this, this.constructor); 101 | this.strategy = 'Dead Letter Queue'; 102 | this.reference = `This error was executed as part of the kafka-penguin Dead Letter Queue message reprocessing strategy. Your consumer attempted to receive a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 103 | this.name = `${e.name}(Consumer Side)`; 104 | this.message = e.message; 105 | this.originalError = e.originalError; 106 | this.retryCount = e.retryCount; 107 | } 108 | } 109 | 110 | export class DeadLetterQueueErrorProducer extends Error { 111 | message: any; 112 | 113 | reference: any; 114 | 115 | name: any; 116 | 117 | retryCount: number; 118 | 119 | strategy: string; 120 | 121 | originalError: any; 122 | 123 | constructor(e: any) { 124 | super(e); 125 | Error.captureStackTrace(this, this.constructor); 126 | this.strategy = 'Dead Letter Queue'; 127 | this.reference = `This error was executed as part of the kafka-penguin Dead Letter Queue message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 128 | this.name = `${e.name}(Producer Side)`; 129 | this.message = e.message; 130 | this.originalError = e.originalError; 131 | this.retryCount = e.retryCount; 132 | } 133 | } 134 | 135 | interface consumerRunInput { 136 | eachMessage: ({ 137 | topic, 138 | partitions, 139 | message, 140 | }: { 141 | topic: string, 142 | partitions: number, 143 | message: any 144 | }) => void, 145 | eachBatchAutoResolve: boolean, 146 | } 147 | 148 | interface consumerSubscribeInput { 149 | groupId?: String, 150 | partitionAssigners?: any, 151 | sessionTimeout?: Number, 152 | rebalanceTimeout?: Number, 153 | heartbeatInterval?: Number, 154 | metadataMaxAge?: Number, 155 | allowAutoTopicCreation?: Boolean, 156 | maxBytesPerPartition?: Number, 157 | minBytes?: Number, 158 | maxBytes?: Number, 159 | maxWaitTimeInMs?: Number, 160 | retry?: Object, 161 | maxInFlightRequests?: Number, 162 | rackId?: String 163 | } 164 | 165 | interface input { 166 | eachMessage: ({ 167 | topic, 168 | partitions, 169 | message, 170 | }: { 171 | topic: string, 172 | partitions: number, 173 | message: any 174 | }) => void 175 | } 176 | export class DeadLetterQueue { 177 | client: any; 178 | 179 | topic: string; 180 | 181 | callback?: (message: any) => boolean; 182 | 183 | innerConsumer: any; 184 | 185 | admin: any; 186 | 187 | innerProducer: any; 188 | 189 | constructor(client: any, topic: string, callback?: any) { 190 | this.topic = topic; 191 | this.client = client; 192 | this.callback = callback; 193 | this.admin = this.client.admin(); 194 | this.innerConsumer = null; 195 | this.innerProducer = this.client.producer(); 196 | } 197 | 198 | producer() { 199 | // Reference the DLQ instance for closure in the returned object 200 | const dlqInstance = this; 201 | const { innerProducer } = dlqInstance; 202 | // Return an object with all Producer methods adapted to execute a dead letter queue strategy 203 | console.log('INNER PRODUCER', innerProducer); 204 | return { 205 | ...innerProducer, 206 | connect() { 207 | return innerProducer.connect() 208 | .then(() => { 209 | dlqInstance.createDLQ(); 210 | }) 211 | .catch((e: Error) => console.log(e)); 212 | }, 213 | send(message: messageValue) { 214 | return innerProducer.connect() 215 | .then(() => { 216 | innerProducer.send({ 217 | ...message, 218 | topic: message.topic, 219 | messages: message.messages, 220 | }) 221 | // Upon error, reroute message to DLQ for the strategy topic 222 | .catch((e?: any) => { 223 | innerProducer.send({ 224 | messages: message.messages, 225 | topic: `${dlqInstance.topic}.deadLetterQueue`, 226 | }) 227 | .then(innerProducer.disconnect()) 228 | .catch((e: Error) => console.log(e)); 229 | // Print the error to the console 230 | const newError = new DeadLetterQueueErrorProducer(e); 231 | console.log(newError); 232 | }); 233 | }); 234 | }, 235 | }; 236 | } 237 | 238 | consumer(groupId: { groupId: string }) { 239 | this.innerConsumer = this.client.consumer(groupId); 240 | const dlqInstance = this; 241 | const { innerConsumer, innerProducer } = dlqInstance; 242 | // Returns an object with all Consumer methods adapter to execute a dead letter queue strategy 243 | 244 | return { 245 | ...innerConsumer, 246 | connect() { 247 | return innerConsumer.connect().then(() => { 248 | dlqInstance.createDLQ(); 249 | }); 250 | }, 251 | subscribe(input?: consumerSubscribeInput) { 252 | return innerConsumer.subscribe({ 253 | ...input, 254 | topic: dlqInstance.topic, 255 | fromBeginning: false, 256 | }); 257 | }, 258 | run(input: consumerRunInput) { 259 | const { eachMessage } = input; 260 | return innerConsumer.run({ 261 | ...input, 262 | eachMessage: ({ topic, partitions, message }: { 263 | topic: string, partitions: number, message: any 264 | }) => { 265 | try { 266 | // If user doesn't pass in callback, DLQ simply listens and returns errors 267 | if (dlqInstance.callback) { 268 | if (!dlqInstance.callback(message)) throw Error; 269 | eachMessage({ topic, partitions, message }); 270 | } 271 | } catch (e) { 272 | const newError = new DeadLetterQueueErrorConsumer(e); 273 | console.error(newError); 274 | innerProducer.connect() 275 | .then(() => console.log('kafka-penguin: Connected to DLQ topic')) 276 | .then(() => { 277 | innerProducer.send({ 278 | topic: `${dlqInstance.topic}.deadLetterQueue`, 279 | messages: [message], 280 | }); 281 | }) 282 | .then(() => console.log('kafka-penguin: Message published to DLQ')) 283 | .then(() => innerProducer.disconnect()) 284 | .then(() => console.log('kafka-penguin: Producer disconnected')) 285 | .catch((e: any) => console.log('Error with producing to DLQ: ', e)); 286 | } 287 | }, 288 | }); 289 | }, 290 | }; 291 | } 292 | 293 | // Creates a new DLQ topic with the original topic name 294 | async createDLQ() { 295 | const adminCreateDLQ = await this.admin.connect() 296 | .then(async () => { 297 | await this.admin.createTopics({ 298 | topics: [{ 299 | topic: `${this.topic}.deadLetterQueue`, 300 | numPartitions: 1, 301 | replicationFactor: 1, 302 | replicaAssignment: [{ partition: 0, replicas: [0, 1, 2] }], 303 | }], 304 | }); 305 | }) 306 | .then(() => this.admin.disconnect()) 307 | .catch((err: any) => console.log('Error from createDLQ', err)); 308 | return adminCreateDLQ; 309 | } 310 | } 311 | 312 | // Ignore 313 | 314 | export class IgnoreErrorProducer extends Error { 315 | message: any; 316 | 317 | reference: any; 318 | 319 | name: any; 320 | 321 | retryCount: number; 322 | 323 | strategy: string; 324 | 325 | originalError: any; 326 | 327 | constructor(e: any) { 328 | super(e); 329 | Error.captureStackTrace(this, this.constructor); 330 | this.strategy = 'Ignore'; 331 | this.reference = `This error was executed as part of the kafka-penguin Ignore message reprocessing strategy. Your producer attempted to deliver a message ${e.retryCount + 1} times but was unsuccessful.`; 332 | this.name = `${e.name} (Producer Side)`; 333 | this.message = e.message; 334 | this.originalError = e.originalError; 335 | this.retryCount = e.retryCount; 336 | } 337 | } 338 | 339 | export class IgnoreErrorConsumer extends Error { 340 | message: any; 341 | 342 | reference: any; 343 | 344 | name: any; 345 | 346 | retryCount: number; 347 | 348 | strategy: string; 349 | 350 | originalError: any; 351 | 352 | constructor(e: any) { 353 | super(e); 354 | Error.captureStackTrace(this, this.constructor); 355 | this.strategy = 'Ignore'; 356 | this.reference = `This error was executed as part of the kafka-penguin Ignore message reprocessing strategy. Your consumer attempted to receive a message ${e.retryCount + 1} times but was unsuccessful. As a result, the message was sent to a Dead Letter Queue. Refer to the original error for further information`; 357 | this.name = `${e.name} (Consumer Side)`; 358 | this.message = e.message; 359 | this.originalError = e.originalError; 360 | this.retryCount = e.retryCount; 361 | } 362 | } 363 | 364 | interface messageValue { 365 | acks?: Number, 366 | timeout?: Number, 367 | compression?: CompressionTypes, 368 | topic: string, 369 | messages: object[], 370 | } 371 | 372 | interface consumerRunInput { 373 | eachMessage: ({ 374 | topic, 375 | partitions, 376 | message, 377 | }: { 378 | topic: string, 379 | partitions: number, 380 | message: any 381 | }) => void, 382 | eachBatchAutoResolve: boolean, 383 | } 384 | 385 | interface consumerSubscribeInput { 386 | groupId?: String, 387 | partitionAssigners?: any, 388 | sessionTimeout?: Number, 389 | rebalanceTimeout?: Number, 390 | heartbeatInterval?: Number, 391 | metadataMaxAge?: Number, 392 | allowAutoTopicCreation?: Boolean, 393 | maxBytesPerPartition?: Number, 394 | minBytes?: Number, 395 | maxBytes?: Number, 396 | maxWaitTimeInMs?: Number, 397 | retry?: Object, 398 | maxInFlightRequests?: Number, 399 | rackId?: String 400 | } 401 | 402 | export class Ignore { 403 | client: any; 404 | 405 | topic: string; 406 | 407 | callback?: (message: any) => boolean; 408 | 409 | innerConsumer: any; 410 | 411 | admin: any; 412 | 413 | innerProducer: any; 414 | 415 | constructor(client: any, topic: string, callback?: any) { 416 | this.topic = topic; 417 | this.client = client; 418 | this.callback = callback; 419 | this.admin = this.client.admin(); 420 | this.innerConsumer = null; 421 | this.innerProducer = this.client.producer(); 422 | } 423 | 424 | producer() { 425 | // Reference the Ignore instance for closure in the returned object 426 | const ignoreInstance = this; 427 | const { innerProducer } = ignoreInstance; 428 | // Return an object with all Producer methods adapted to execute Ignore strategy 429 | console.log('INNER PRODUCER', innerProducer); 430 | return { 431 | ...innerProducer, 432 | connect() { 433 | return innerProducer.connect() 434 | .catch((e: Error) => console.log(e)); 435 | }, 436 | send(message: messageValue) { 437 | return innerProducer.connect() 438 | .then(() => { 439 | innerProducer.send({ 440 | ...message, 441 | topic: message.topic, 442 | messages: message.messages, 443 | }) 444 | .catch((e: Error) => { 445 | console.log(e); 446 | // Print the error to the console 447 | const newError = new IgnoreErrorProducer(e); 448 | console.log(newError); 449 | }); 450 | }); 451 | }, 452 | }; 453 | } 454 | 455 | consumer(groupId: { groupId: string }) { 456 | this.innerConsumer = this.client.consumer(groupId); 457 | const ignoreInstance = this; 458 | const { innerConsumer } = ignoreInstance; 459 | // Returns an object with all Consumer methods adapter to execute ignore strategy 460 | return { 461 | ...innerConsumer, 462 | connect() { 463 | return innerConsumer.connect(); 464 | }, 465 | subscribe(input?: consumerSubscribeInput) { 466 | return innerConsumer.subscribe({ 467 | ...input, 468 | topic: ignoreInstance.topic, 469 | fromBeginning: false, 470 | }); 471 | }, 472 | run(input: consumerRunInput) { 473 | const { eachMessage } = input; 474 | return innerConsumer.run({ 475 | ...input, 476 | eachMessage: ( 477 | { topic, partitions, message }: { 478 | topic: string, partitions: number, message: any 479 | // eslint-disable-next-line comma-dangle 480 | } 481 | ) => { 482 | try { 483 | // If user doesn't pass in callback 484 | if (ignoreInstance.callback) { 485 | if (!ignoreInstance.callback(message)) throw Error; 486 | eachMessage({ topic, partitions, message }); 487 | } 488 | } catch (e) { 489 | const newError = new IgnoreErrorConsumer(e); 490 | console.error("kafka Error:", newError); 491 | } 492 | }, 493 | }) 494 | }, 495 | }; 496 | } 497 | } 498 | -------------------------------------------------------------------------------- /kafka-penguin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "strict": true, 6 | "declaration": true, 7 | "outDir": "./dist" 8 | }, 9 | "exclude": [ 10 | "tests", 11 | "dist", 12 | "node_modules", 13 | "../demo", 14 | "./src/failfast.test.ts", 15 | "./src/deadletterqueue.ts" 16 | ] 17 | } -------------------------------------------------------------------------------- /settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "eslint.validate": [ 3 | "javascript", 4 | "javascriptreact", 5 | "typescript", 6 | "typescriptreact" 7 | ], 8 | "[javascriptreact]": { 9 | "editor.codeActionsOnSave": { 10 | "source.fixAll.eslint": true 11 | } 12 | }, 13 | "editor.formatOnSave": true, 14 | "[typescript]": { 15 | "editor.formatOnSave": false, 16 | "editor.codeActionsOnSave": { 17 | "source.fixAll.eslint": true 18 | } 19 | }, 20 | "[typescriptreact]": { 21 | "editor.formatOnSave": false, 22 | "editor.codeActionsOnSave": { 23 | "source.fixAll.eslint": true 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /strategies/README.md: -------------------------------------------------------------------------------- 1 | # Strategies 2 | 3 | #### FailFast 4 | 5 | Stops processing as soon as an error occurs. 6 | 7 | {% page-ref page="failfast.md" %} 8 | 9 | #### Ignore 10 | 11 | Handle message processing failures by ignoring. Achieves non-blocking for your queue 12 | 13 | {% page-ref page="strategies-ignore.md" %} 14 | 15 | **Dead Letter Queue** 16 | 17 | Handle message processing failures by forwarding problematic messages to a dead-letter queue \(DLQ\). 18 | 19 | {% page-ref page="dead-letter-queue.md" %} 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /strategies/dead-letter-queue.md: -------------------------------------------------------------------------------- 1 | # Dead Letter Queue 2 | 3 | ## About 4 | 5 | This strategy creates another topic that acts as a repository for erroneous messages. It works parallel with your normal topic and is meant to keep flows from producers or consumers unblocked \(while storing problematic messages for later reprocessing\). Potential use cases for this strategy include services with data streaming, non-ACID or transactional message flows, or any system that simply needs to "just keep running". 6 | 7 | ## Syntax 8 | 9 | **DeadLetterQueue\(kafka-client, topic, callback\)** 10 | 11 | `kafka-client` A configured [KafkaJS client](https://kafka.js.org/docs/configuration) provided by the developer. 12 | 13 | `topic` The target topic that producers or consumers will publish or subscribe to in this strategy instance. Kafka-Penguin currently supports one topic per strategy instance. If a dead letter queue for this topic has not been created, the strategy will automatically create it upon producer or consumer connect. 14 | 15 | `callback` A callback that must return a boolean value. The callback will take in one argument: the messages received by the consumer. During execution, the strategy will pass to the callback each message consumed; any message which returns `false` will be rerouted to the topic-specific dead letter queue. This allows the developer to customize the strategy to catch specific conditions when consuming. 16 | 17 | #### **Producer** 18 | 19 | `DeadLetterQueue.producer` Returns a producer initialized from the strategy instance. The producer has "adapted" methods which execute the strategy under the hood. 20 | 21 | `producer.connect` Connects the producer to the Kafka cluster indicated in the configured KafkaJS client. This method will also create a topic-specific dead letter queue if one does not already exist. 22 | 23 | `producer.send(message)` This method takes in one argument, `messages` that are passed in with the same requirements as the counterpart method on KafkaJS, and sends it to the Kafka cluster. However, this `send` is adapted to send to the strategy's dead letter queue upon error. 24 | 25 | #### Consumer 26 | 27 | `DeadLetterQueue.consumer` Returns a consumer initialized from the strategy instance. The consumer has "adapted" methods which execute the strategy under the hood. 28 | 29 | `consumer.connect` Connects the consumer to the Kafka cluster indicated in the configured KafkaJS client. This method will also create a topic-specific dead letter queue if one does not already exist. 30 | 31 | `consumer.run` Starts consuming messages from the Kafka cluster to the consumer client. The `run` method also utilizes `eachMessage`to pass each message received through the callback provided at strategy instantiation. If the callback returns `false`, the `run` method automatically creates a temporary producer, it produces the message to the dead letter queue, and then discards that producer. 32 | 33 | ## Example 34 | 35 | #### Producer 36 | 37 | ```javascript 38 | const producerClientDLQ = require('./clientConfig.ts') 39 | import { DeadLetterQueue } from 'kafka-penguin' 40 | 41 | 42 | // This example simulates an error where the producer sends to a bad topic 43 | const topicGood = 'test-topic-DLQ'; 44 | const topicBad = "topic-non-existent" 45 | 46 | // Set up the Dead Letter Queue (DLQ) strategy with a configured KafkaJS client, a topic, and a callback that evaluates to a boolean 47 | const exampleDLQProducer = new DeadLetterQueue(producerClientDLQ, topicGood, true); 48 | 49 | // Initialize a producer from the new instance of the Dead Letter Queue strategy 50 | const producerDLQ = exampleDLQProducer.producer(); 51 | 52 | // Connecting the producer creates a DLQ topic in case of bad messages 53 | // If an error occurs, the strategy moves the message to the topic specific DLQ 54 | // The producer is able to keep publishing good messages to the topic 55 | producerDLQ.connect() 56 | .then(() => producerDLQ.send({ 57 | topic: topicGood, 58 | messages: [ 59 | { 60 | key: 'message 1', 61 | value: 'Good Message', 62 | }, 63 | ], 64 | })) 65 | .then(() => producerDLQ.send ({ 66 | topic: topicBad, 67 | messages: [ 68 | { 69 | key: 'message 2', 70 | value: 'Bad Message', 71 | } 72 | ] 73 | })) 74 | .then(() => producerDLQ.send ({ 75 | topic: topicGood, 76 | messages: [ 77 | { 78 | key: 'message 3', 79 | value: 'Good Message', 80 | } 81 | ] 82 | })) 83 | .then(() => producerDLQ.disconnect()) 84 | .catch((e: any) => { 85 | console.log(e); 86 | 87 | 88 | ``` 89 | 90 | **Consumer** 91 | 92 | ```javascript 93 | const client = require('./clientConfig.ts'); 94 | import { DeadLetterQueue } from 'kafka-penguin'; 95 | 96 | const topic = 'test-topic-DLQ'; 97 | 98 | // This allows the consumer to evaluate each message according to a condition 99 | // The callback must return a boolean value 100 | const callback = (message) => { 101 | try { 102 | JSON.parse(message.value); 103 | const callback = (message) => { 104 | return true; 105 | }; 106 | 107 | // Set up the Dead Letter Queue (DLQ) strategy with a configured KafkaJS client, a topic, and the evaluating callback 108 | const exampleDLQConsumer = new DeadLetterQueue(client, topic, callback); 109 | 110 | // Initialize a consumer from the new instance of the Dead Letter Queue strategy 111 | const consumerDLQ = exampleDLQConsumer.consumer({ groupId: 'testID' }); 112 | 113 | 114 | 115 | // Connecting the consumer creates a DLQ topic in case of bad messages 116 | // If the callback returns false, the strategy moves the message to the topic specific DLQ 117 | // The consumer is able to keep consuming good messages from the topic 118 | consumerDLQ.connect() 119 | .then(consumerDLQ.subscribe()) 120 | .then(() => consumerDLQ.run({ 121 | eachMessage: ({ topic, partitions, message }) => { 122 | console.log(JSON.parse(message.value)); 123 | }, 124 | })) 125 | .catch((e) => console.log(`Error message from consumer: ${e}`)); 126 | 127 | 128 | 129 | ``` 130 | 131 | -------------------------------------------------------------------------------- /strategies/failfast.md: -------------------------------------------------------------------------------- 1 | # FailFast 2 | 3 | ## About 4 | 5 | This strategy executes a purposeful disconnect after a producer has sent an erroneous message. Potential use cases for this strategy include micro-services where there is a low tolerance for failure, as well as acid and/or transactional workflows. This strategy also works well for development environments in Agile-based workflows. 6 | 7 | ## Syntax 8 | 9 | #### FailFast\(retries, kafka-client\) 10 | 11 | `retries` Number of times the producer attempts to send the message before disconnecting and throwing an error. 12 | 13 | `kafka-client` A configured [KafkaJS client](https://kafka.js.org/docs/configuration) provided by the developer. 14 | 15 | **Producer** 16 | 17 | `FailFast.producer` Returns a producer initialized from the strategy instance. The producer has "adapted" methods that execute the strategy under the hood. 18 | 19 | `producer.connect` Connects the producer to the Kafka cluster indicated in the configured KafkaJS client. 20 | 21 | `producer.send(message)` This method takes in one argument, `message` that is passed in with the same requirements as the counterpart method on KafkaJS, and sends it to the Kafka cluster. However, this `send` will disconnect the producer once it reaches the set number of retries. 22 | 23 | ## Example 24 | 25 | **Producer** 26 | 27 | ```javascript 28 | import { FailFast } from 'kafka-penguin' 29 | const FailFastClient = require('./clientConfig.ts') 30 | 31 | // Set up Fail Fast with the number of retried and a configured KafkaJS client 32 | const exampleFailFast = new FailFast(2, FailFastClient) 33 | 34 | // Initialize a producer from the new instance of Fail Fast 35 | const producer = exampleFailFast.producer(); 36 | 37 | 38 | // Example error of a producer sending to a non-existent topic 39 | const message = { 40 | topic: 'topic-non-existent', 41 | messages: [ 42 | { 43 | key: 'hello' 44 | }] 45 | } 46 | 47 | // FailFast will attempt to send the message to the Kafka cluster. 48 | // After the retry count is reached, the producer will automatically disconnect and an error is thrown. 49 | producer.connect() 50 | .then(() => console.log('Connected!')) 51 | .then(() => producer.send(message)) 52 | ``` 53 | 54 | -------------------------------------------------------------------------------- /strategies/strategies-ignore.md: -------------------------------------------------------------------------------- 1 | # Ignore 2 | 3 | ## About 4 | 5 | This strategy enables you to continuously process messages even if some messages are erroneous, problematic, or fail for some reason. It is meant to keep flows from producers or consumers unblocked even if there are failures. Potential use cases for this strategy include services and systems that rely upon a constant stream of data. 6 | 7 | ## Syntax 8 | 9 | **Ignore\(kafka-client, topic, callback\)** 10 | 11 | `kafka-client` A configured [KafkaJS client](https://kafka.js.org/docs/configuration) provided by the developer. 12 | 13 | `topic` The target topic that producers or consumers will publish or subscribe to in this strategy instance. Kafka-Penguin currently supports one topic per strategy instance. 14 | 15 | `callback` A callback that must return a boolean value. The callback will take in one argument: the messages received by the consumer. During execution, the strategy will pass to the callback each message consumed; Even if a message which returns `false`, the producer or consumer will continue to produce or consume messages uninterrupted. 16 | 17 | #### **Producer** 18 | 19 | `Ignore.producer` Returns a producer initialized from the strategy instance. The producer has "adapted" methods which execute the strategy under the hood. 20 | 21 | `producer.connect` Connects the producer to the Kafka cluster indicated in the configured KafkaJS client. 22 | 23 | `producer.send(message)` This method takes in one argument, `messages` that are passed in with the same requirements as the counterpart method on KafkaJS, and sends it to the Kafka cluster. 24 | 25 | #### Consumer 26 | 27 | `Ignore.consumer` Returns a consumer initialized from the strategy instance. The consumer has "adapted" methods which execute the strategy under the hood. 28 | 29 | `consumer.connect` Connects the consumer to the Kafka cluster indicated in the configured KafkaJS client. 30 | 31 | `consumer.run` Starts consuming messages from the Kafka cluster to the consumer client. The `run` method also utilizes `eachMessage`to pass each message received through the callback provided at strategy instantiation. 32 | 33 | ## Example 34 | 35 | #### Producer: 36 | 37 | ```text 38 | /* eslint-disable no-console */ 39 | import { Ignore } from './kafka-penguin/src/index' 40 | 41 | const producerClientIgnore = require('./clientConfig.ts'); 42 | 43 | // This example simulates an error where the producer sends to a bad topic 44 | const topicGood = 'test-topic'; 45 | const topicBad = 'topic-non-existent'; 46 | 47 | // Set up the Ignore strategy 48 | // Configure it with a configured KafkaJS client, a topic, and a callback that returns boolean 49 | const exampleIgnoreProducer = new Ignore(producerClientIgnore, topicGood, true); 50 | 51 | // Initialize a producer from the new instance of the Ignore strategy 52 | const producerIgnore = exampleIgnoreProducer.producer(); 53 | 54 | // Connecting the producer and send messages. 55 | // If an error occurs with a message, the strategy ignores erroneous message and continues 56 | // publishing good messages to the topic 57 | producerIgnore.connect() 58 | .then(() => producerIgnore.send({ 59 | topic: topicGood, 60 | messages: [ 61 | { 62 | key: 'message 1', 63 | value: JSON.stringify('Good Message'), 64 | }, 65 | ], 66 | })) 67 | .then(() => producerIgnore.send({ 68 | topic: topicGood, 69 | messages: [ 70 | { 71 | key: 'message 2', 72 | value: 'Bad Message', 73 | }, 74 | ], 75 | })) 76 | .then(() => producerIgnore.send({ 77 | topic: topicGood, 78 | messages: [ 79 | { 80 | key: 'message 3', 81 | value: JSON.stringify('Good Message'), 82 | }, 83 | ], 84 | })) 85 | .then(() => producerIgnore.disconnect()) 86 | .catch((e: any) => { 87 | console.log(e); 88 | }); 89 | ``` 90 | 91 | #### Consumer: 92 | 93 | ```text 94 | /* eslint-disable no-console */ 95 | import { Ignore } from './kafka-penguin/src/index'; 96 | 97 | const client = require('./clientConfig.ts'); 98 | const topic = 'test-topic'; 99 | 100 | // This allows the consumer to evaluate each message according to a condition 101 | // The callback must return a boolean value 102 | const callback = (message) => { 103 | try { 104 | JSON.parse(message.value); 105 | } catch (e) { 106 | return false; 107 | } 108 | return true; 109 | }; 110 | 111 | // Set up the Ignore strategy 112 | // with a configured KafkaJS client, a topic, and the evaluating callback 113 | const exampleIgnoreConsumer = new Ignore(client, topic, callback); 114 | 115 | // Initialize a consumer from the new instance of the Dead Letter Queue strategy 116 | const consumerIgnore = exampleIgnoreConsumer.consumer({ groupId: 'testID' }); 117 | 118 | // Connecting the consumer to consume messages. bad messages 119 | // If the callback evaluates a message as erroneous by returning false, the strategy 120 | // enables the consumer to keep consuming good messages from the topic 121 | consumerIgnore.connect() 122 | .then(consumerIgnore.subscribe()) 123 | .then(() => consumerIgnore.run({ 124 | eachMessage: ({ message }) => { 125 | // if (message.value.length < 5) return true; 126 | // return false; 127 | console.log("message value:", message.value.toString()) 128 | }, 129 | })) 130 | .catch((e) => console.log(`Error message from consumer: ${e}`)); 131 | ``` 132 | 133 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES5", 4 | "module": "commonJS", 5 | "jsx": "react", 6 | "sourceMap": true, 7 | "outDir": "./build", 8 | "removeComments": true, 9 | "allowSyntheticDefaultImports": true, 10 | "esModuleInterop": true, 11 | "preserveConstEnums": true 12 | }, 13 | "include": [ 14 | "./demo/client/**/*" 15 | ] 16 | } 17 | --------------------------------------------------------------------------------