├── .dockerignore ├── .env.example ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── .vscode ├── launch.json └── tasks.json ├── Dockerfile ├── LICENSE ├── README.md ├── nest-cli.json ├── package-lock.json ├── package.json ├── src ├── config │ ├── config.export.ts │ ├── config.schema.ts │ └── config.validate.ts ├── main.ts └── modules │ ├── app │ └── app.module.ts │ ├── example │ ├── dto │ │ ├── broadcast.dto.ts │ │ ├── index.ts │ │ ├── limit.dto.ts │ │ ├── metadata.dto.ts │ │ └── queue.dto.ts │ ├── example.controller.spec.ts │ ├── example.controller.ts │ ├── example.module.ts │ └── example.service.ts │ └── kafka │ ├── kafka.module.ts │ └── kafka.service.ts ├── test └── app.e2e.ts ├── tsconfig.build.json └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /dist 3 | *.env -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | KAFKAJS_NO_PARTITIONER_WARNING=1 2 | NODE_ENV=local 3 | HOST=localhost 4 | ADDRESS=http://localhost:3000 5 | SWAGGER_PATH=docs 6 | PORT=3000 7 | KAFKA_ADDRESS=localhost:9092 8 | KAFKA_CLIENT_ID=example-producer 9 | KAFKA_SERVICE_NAME=EXAMPLE_SERVICE -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir: __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | /build 5 | 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | pnpm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | lerna-debug.log* 14 | 15 | # OS 16 | .DS_Store 17 | 18 | # Tests 19 | /coverage 20 | /.nyc_output 21 | 22 | # IDEs and editors 23 | /.idea 24 | .project 25 | .classpath 26 | .c9/ 27 | *.launch 28 | .settings/ 29 | *.sublime-workspace 30 | 31 | # IDE - VSCode 32 | .vscode/* 33 | !.vscode/settings.json 34 | !.vscode/tasks.json 35 | !.vscode/launch.json 36 | !.vscode/extensions.json 37 | 38 | # dotenv environment variable files 39 | .env 40 | .env.development.local 41 | .env.test.local 42 | .env.production.local 43 | .env.local 44 | 45 | # temp directory 46 | .temp 47 | .tmp 48 | 49 | # Runtime data 50 | pids 51 | *.pid 52 | *.seed 53 | *.pid.lock 54 | 55 | # Diagnostic reports (https://nodejs.org/api/report.html) 56 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 57 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Launch Program", 8 | "skipFiles": [ "/**" ], 9 | "program": "${workspaceFolder}/src/main.ts", 10 | "cwd": "${workspaceFolder}", 11 | "console": "integratedTerminal", 12 | "preLaunchTask": "build", 13 | "envFile": "${workspaceFolder}/.env", 14 | "sourceMaps": true 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "npm", 6 | "script": "build", 7 | "group": "build", 8 | "problemMatcher": [], 9 | "label": "build", 10 | "detail": "tsc --build" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM node:21.7.3-slim 3 | WORKDIR /usr/src/app 4 | COPY . . 5 | RUN npm install && npm run build:prod && npm cache clean --force 6 | 7 | FROM node:21.7.3-slim 8 | RUN groupadd -r appuser && useradd -r -g appuser -s /sbin/nologin -d /usr/src/app appuser \ 9 | && mkdir -p /usr/src/app \ 10 | && chown -R appuser:appuser /usr/src/app 11 | 12 | WORKDIR /usr/src/app 13 | COPY --from=0 /usr/src/app/ ./ 14 | USER appuser 15 | EXPOSE 3000 16 | CMD ["npm", "run", "start:prod"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 pendulum 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Nest Logo 3 |

4 | 5 | [circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 6 | [circleci-url]: https://circleci.com/gh/nestjs/nest 7 | 8 |

A progressive Node.js framework for building efficient and scalable server-side applications.

9 |

10 | NPM Version 11 | Package License 12 | NPM Downloads 13 | CircleCI 14 | Coverage 15 | Discord 16 | Backers on Open Collective 17 | Sponsors on Open Collective 18 | Donate us 19 | Support us 20 | Follow us on Twitter 21 |

22 | 24 | 25 | ## Kafka Producer Template 26 | 27 | This repository serves as a template for a Kafka producer, offering flexible and powerful ways to send messages to Kafka topics. It supports three distinct methods of sending messages, making it a versatile solution for various data processing scenarios. 28 | 29 | ## Features 30 | 1. **Broadcast**: 31 | The message is sent to all partitions of a Kafka topic without binding to a specific key. This is used when you need to send a message to all consumers without partitioning by queues. 32 | 33 | 2. **Queueing by Key**: 34 | The message is sent with a specific key, ensuring its delivery to a designated partition of the topic. This ensures sequential processing of messages with the same key and is used to manage tasks that require ordered processing in a queue. 35 | 36 | 3. **Rate-limited Sending**: 37 | The ability to send messages at a specified rate limit. This approach is useful when you need to control the load on Kafka brokers or limit the processing speed for an even distribution of the load. 38 | 39 | ## How to Use 40 | Clone the repository and configure the connection to your Kafka broker. 41 | Choose and configure the desired method for sending messages (broadcast, queueing by key, or rate-limited sending). 42 | Integrate the template into your application for quick and efficient Kafka message handling. 43 | 44 | ## BroadCast 45 | 46 | Each application instance should have its own static consumer group, ensuring they can read from a single partition independently. This design allows for scalability by increasing the number of partitions if needed, though it's not a strict requirement for functionality. 47 | 48 | ```mermaid 49 | flowchart TD 50 | P["Producer"] --> K["Kafka"] 51 | K -->|group Id 1| A["Consumer replica 1"] 52 | K -->|group Id 2| B["Consumer replica 2"] 53 | K -->|group Id 3| C["Consumer replica 3"] 54 | ``` 55 | 56 | ## Queueing by Key 57 | ... 58 | 59 | 60 | ## Project setup 61 | 1. Install dependencies 62 | ```bash 63 | $ npm install 64 | ``` 65 | 2. Create .env file by .env.example 66 | ```env 67 | KAFKAJS_NO_PARTITIONER_WARNING=1 68 | NODE_ENV=local 69 | HOST=localhost 70 | ... 71 | ``` 72 | ## Compile and run the project 73 | 74 | ```bash 75 | # development 76 | $ npm run start 77 | 78 | # watch mode 79 | $ npm run start:dev 80 | 81 | # production mode 82 | $ npm run start:prod 83 | ``` 84 | 85 | ## Run tests 86 | 87 | ```bash 88 | # unit tests 89 | $ npm run test 90 | 91 | # e2e tests 92 | $ npm run test:e2e 93 | 94 | # test coverage 95 | $ npm run test:cov 96 | ``` 97 | 98 | ## Docker build & run 99 | 100 | ```bash 101 | # build docker image 102 | $ docker build -t nestjs-kafka-producer . 103 | 104 | # run image with .env file 105 | $ docker run --env-file .env -d -p 3000:3000 nestjs-kafka-producer 106 | ``` 107 | 108 | ## Resources 109 | 110 | Check out a few resources that may come in handy when working with NestJS: 111 | 112 | - Visit the [NestJS Documentation](https://docs.nestjs.com) to learn more about the framework. 113 | - For questions and support, please visit our [Discord channel](https://discord.gg/G7Qnnhy). 114 | - To dive deeper and get more hands-on experience, check out our official video [courses](https://courses.nestjs.com/). 115 | - Visualize your application graph and interact with the NestJS application in real-time using [NestJS Devtools](https://devtools.nestjs.com). 116 | - Need help with your project (part-time to full-time)? Check out our official [enterprise support](https://enterprise.nestjs.com). 117 | - To stay in the loop and get updates, follow us on [X](https://x.com/nestframework) and [LinkedIn](https://linkedin.com/company/nestjs). 118 | - Looking for a job, or have a job to offer? Check out our official [Jobs board](https://jobs.nestjs.com). 119 | 120 | ## Support 121 | 122 | Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support). 123 | 124 | ## Stay in touch 125 | 126 | - Author - [Kamil Myśliwiec](https://twitter.com/kammysliwiec) 127 | - Website - [https://nestjs.com](https://nestjs.com/) 128 | - Twitter - [@nestframework](https://twitter.com/nestframework) 129 | 130 | ## License 131 | 132 | Nest is [MIT licensed](https://github.com/nestjs/nest/blob/master/LICENSE). 133 | -------------------------------------------------------------------------------- /nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src", 5 | "compilerOptions": { 6 | "deleteOutDir": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nestjs-template", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "build": "nest build", 10 | "build:prod": "npm run build && npm prune --omit=dev", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 12 | "start": "nest start", 13 | "start:dev": "nest start --watch", 14 | "start:debug": "nest start --debug --watch", 15 | "start:prod": "node dist/src/main", 16 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 17 | "test": "jest", 18 | "test:watch": "jest --watch", 19 | "test:cov": "jest --coverage", 20 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 21 | "test:e2e": "jest --config ./test/jest-e2e.json" 22 | }, 23 | "dependencies": { 24 | "-": "^0.0.1", 25 | "@nestjs/common": "^10.0.0", 26 | "@nestjs/core": "^10.0.0", 27 | "@nestjs/microservices": "^10.4.2", 28 | "@nestjs/platform-express": "^10.0.0", 29 | "@nestjs/swagger": "^7.4.2", 30 | "class-transformer": "^0.5.1", 31 | "class-validator": "^0.14.1", 32 | "kafkajs": "^2.2.4", 33 | "reflect-metadata": "^0.2.0", 34 | "rxjs": "^7.8.1", 35 | "save": "^2.9.0" 36 | }, 37 | "devDependencies": { 38 | "@nestjs/cli": "^10.0.0", 39 | "@nestjs/schematics": "^10.0.0", 40 | "@nestjs/testing": "^10.0.0", 41 | "@types/express": "^4.17.17", 42 | "@types/jest": "^29.5.2", 43 | "@types/node": "^20.3.1", 44 | "@types/supertest": "^6.0.0", 45 | "@typescript-eslint/eslint-plugin": "^8.0.0", 46 | "@typescript-eslint/parser": "^8.0.0", 47 | "eslint": "^8.42.0", 48 | "eslint-config-prettier": "^9.0.0", 49 | "eslint-plugin-prettier": "^5.0.0", 50 | "jest": "^29.5.0", 51 | "prettier": "^3.0.0", 52 | "source-map-support": "^0.5.21", 53 | "supertest": "^7.0.0", 54 | "ts-jest": "^29.1.0", 55 | "ts-loader": "^9.4.3", 56 | "ts-node": "^10.9.1", 57 | "tsconfig-paths": "^4.2.0", 58 | "typescript": "^5.1.3" 59 | }, 60 | "jest": { 61 | "moduleFileExtensions": [ 62 | "js", 63 | "json", 64 | "ts" 65 | ], 66 | "roots": [ 67 | "src", 68 | "test" 69 | ], 70 | "testRegex": ".*\\.(spec|e2e)\\.ts$", 71 | "transform": { 72 | "^.+\\.(t|j)s$": "ts-jest" 73 | }, 74 | "collectCoverageFrom": [ 75 | "**/*.(t|j)s" 76 | ], 77 | "coverageDirectory": "../coverage", 78 | "testEnvironment": "node" 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/config/config.export.ts: -------------------------------------------------------------------------------- 1 | import { ConfigKafka, ConfigApp } from "./config.schema"; 2 | import { validateEnv } from "./config.validate"; 3 | 4 | export const CONFIG_KAFKA = validateEnv(ConfigKafka); 5 | 6 | export const CONFIG_APP = validateEnv(ConfigApp); -------------------------------------------------------------------------------- /src/config/config.schema.ts: -------------------------------------------------------------------------------- 1 | import { IsNotEmpty, IsNumber, IsString } from "class-validator"; 2 | 3 | export class ConfigApp { 4 | 5 | @IsString() 6 | @IsNotEmpty() 7 | NODE_ENV: string; 8 | 9 | @IsNumber() 10 | @IsNotEmpty() 11 | PORT: number; 12 | 13 | @IsString() 14 | @IsNotEmpty() 15 | ADDRESS: string; 16 | 17 | @IsString() 18 | @IsNotEmpty() 19 | SWAGGER_PATH: string; 20 | } 21 | 22 | export class ConfigKafka { 23 | @IsString() 24 | @IsNotEmpty() 25 | KAFKA_SERVICE_NAME: string 26 | 27 | @IsString() 28 | @IsNotEmpty() 29 | KAFKA_CLIENT_ID: string 30 | 31 | @IsString() 32 | @IsNotEmpty() 33 | KAFKA_ADDRESS: string; 34 | } -------------------------------------------------------------------------------- /src/config/config.validate.ts: -------------------------------------------------------------------------------- 1 | import { plainToInstance } from "class-transformer"; 2 | import { validateSync } from "class-validator"; 3 | 4 | 5 | export function validateEnv(config: new (...args: any[]) => T): T { 6 | const validatedConfig = plainToInstance(config, process.env, { 7 | enableImplicitConversion: true, 8 | }); 9 | const errors = validateSync(validatedConfig, { skipMissingProperties: false }); 10 | if (errors.length > 0) { 11 | throw new Error(errors.toString()); 12 | } 13 | return validatedConfig; 14 | } -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from '@nestjs/core'; 2 | import { AppModule } from './modules/app/app.module'; 3 | import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; 4 | import { CONFIG_APP } from './config/config.export'; 5 | import { ValidationPipe } from '@nestjs/common'; 6 | 7 | async function bootstrap() { 8 | const app = await NestFactory.create(AppModule); 9 | app.useGlobalPipes( 10 | new ValidationPipe({ 11 | whitelist: true, 12 | forbidNonWhitelisted: true, 13 | }), 14 | ); 15 | const config = new DocumentBuilder() 16 | .setTitle('Kafka Example') 17 | .addServer(CONFIG_APP.ADDRESS) 18 | .build(); 19 | 20 | const document = SwaggerModule.createDocument(app, config); 21 | SwaggerModule.setup(CONFIG_APP.SWAGGER_PATH, app, document); 22 | await app.listen(CONFIG_APP.PORT); 23 | } 24 | bootstrap(); -------------------------------------------------------------------------------- /src/modules/app/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ExampleModule } from '../example/example.module'; 3 | import { KafkaModule } from '../kafka/kafka.module'; 4 | 5 | @Module({ 6 | imports: [ 7 | KafkaModule, 8 | ExampleModule 9 | ], 10 | }) 11 | export class AppModule { } 12 | -------------------------------------------------------------------------------- /src/modules/example/dto/broadcast.dto.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from '@nestjs/swagger'; 2 | import { IsString } from 'class-validator'; 3 | 4 | export class BroadcastDto { 5 | 6 | @ApiProperty({ 7 | type: String, 8 | default: 'Hello World!' 9 | }) 10 | @IsString() 11 | message: string 12 | } -------------------------------------------------------------------------------- /src/modules/example/dto/index.ts: -------------------------------------------------------------------------------- 1 | export * from './metadata.dto'; 2 | export * from './broadcast.dto'; 3 | export * from './queue.dto'; 4 | export * from './limit.dto'; -------------------------------------------------------------------------------- /src/modules/example/dto/limit.dto.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from '@nestjs/swagger'; 2 | import { IsString } from 'class-validator'; 3 | 4 | export class LimitDto { 5 | 6 | @ApiProperty({ 7 | type: String, 8 | default: 'Hello World!' 9 | }) 10 | @IsString() 11 | message: string 12 | } -------------------------------------------------------------------------------- /src/modules/example/dto/metadata.dto.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from '@nestjs/swagger'; 2 | import { Transform } from 'class-transformer'; 3 | import { IsString } from 'class-validator'; 4 | 5 | export class MetadataDto { 6 | 7 | @ApiProperty({ 8 | type: () => [String], 9 | default: ['example.topic'] 10 | }) 11 | @Transform(({ value }) => { 12 | return Array.isArray(value) ? value : [value]; 13 | }) 14 | @IsString({ each: true }) 15 | topics: string[] 16 | } -------------------------------------------------------------------------------- /src/modules/example/dto/queue.dto.ts: -------------------------------------------------------------------------------- 1 | import { ApiProperty } from '@nestjs/swagger'; 2 | import { IsString } from 'class-validator'; 3 | 4 | export class QueueDto { 5 | 6 | @ApiProperty({ 7 | type: String, 8 | default: 'One' 9 | }) 10 | @IsString() 11 | key: string 12 | 13 | @ApiProperty({ 14 | type: String, 15 | default: 'Hello World!' 16 | }) 17 | @IsString() 18 | message: string 19 | } -------------------------------------------------------------------------------- /src/modules/example/example.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { ExampleController } from './example.controller'; 3 | import { ExampleService } from './example.service'; 4 | 5 | describe('AppController', () => { 6 | let appController: ExampleController; 7 | 8 | beforeEach(async () => { 9 | const app: TestingModule = await Test.createTestingModule({ 10 | controllers: [ExampleController], 11 | providers: [ExampleService], 12 | }).compile(); 13 | 14 | appController = app.get(ExampleController); 15 | }); 16 | 17 | describe('root', () => { 18 | it('should return "Hello World!"', () => { 19 | expect(appController.getMetadata({ 20 | topics: [] 21 | })).toBe('Hello World!'); 22 | }); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/modules/example/example.controller.ts: -------------------------------------------------------------------------------- 1 | import { Body, Controller, Get, Post, Query } from '@nestjs/common'; 2 | import { ExampleService } from './example.service'; 3 | import { ApiTags } from '@nestjs/swagger'; 4 | import { 5 | BroadcastDto, 6 | LimitDto, 7 | MetadataDto, 8 | QueueDto 9 | } from './dto'; 10 | 11 | @Controller() 12 | @ApiTags('Activities') 13 | export class ExampleController { 14 | 15 | constructor(private readonly exampleService: ExampleService) {} 16 | 17 | @Get('metadata') 18 | public async getMetadata(@Query() dto: MetadataDto): Promise { 19 | return this.exampleService.getMetadata(dto); 20 | } 21 | 22 | @Post('broadcast') 23 | public async postBroadcast(@Body() dto: BroadcastDto): Promise { 24 | return this.exampleService.postBroadcast(dto); 25 | } 26 | 27 | @Post('queue') 28 | public async postQueue(@Body() dto: QueueDto): Promise { 29 | return this.exampleService.postQueue(dto); 30 | } 31 | 32 | @Post('limit') 33 | public async postLimit(@Body() dto: LimitDto): Promise{ 34 | return this.exampleService.postLimit(dto); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/modules/example/example.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ExampleController } from './example.controller'; 3 | import { ExampleService } from './example.service'; 4 | 5 | @Module({ 6 | controllers: [ExampleController], 7 | providers: [ExampleService], 8 | }) 9 | export class ExampleModule {} 10 | -------------------------------------------------------------------------------- /src/modules/example/example.service.ts: -------------------------------------------------------------------------------- 1 | import { Inject, Injectable } from '@nestjs/common'; 2 | import { ClientKafka } from '@nestjs/microservices'; 3 | import { lastValueFrom } from 'rxjs'; 4 | import { KafkaService } from '../kafka/kafka.service'; 5 | import { CONFIG_KAFKA } from 'src/config/config.export'; 6 | import { Message, RecordMetadata } from 'kafkajs'; 7 | import { 8 | BroadcastDto, 9 | LimitDto, 10 | MetadataDto, 11 | QueueDto 12 | } from './dto'; 13 | 14 | 15 | @Injectable() 16 | export class ExampleService { 17 | 18 | constructor( 19 | @Inject(CONFIG_KAFKA.KAFKA_SERVICE_NAME) 20 | private readonly exampleClient: ClientKafka, 21 | private readonly kafkaService: KafkaService 22 | ){} 23 | 24 | public async getMetadata(dto: MetadataDto): Promise { 25 | const meta = await this.kafkaService 26 | .fetchTopicMetadata(dto.topics); 27 | 28 | return JSON.stringify(meta, null, 2); 29 | } 30 | 31 | public async postBroadcast(dto: BroadcastDto): Promise { 32 | const test = await lastValueFrom( 33 | this.exampleClient.emit('broadcast.topic', { 34 | value: dto.message 35 | }) 36 | ); 37 | return JSON.stringify(test, null, 2); 38 | } 39 | 40 | public async postQueue(dto: QueueDto): Promise { 41 | const test = await lastValueFrom( 42 | this.exampleClient.emit('queue.topic', { 43 | key: dto.key, 44 | value: dto.message 45 | }) 46 | ); 47 | return JSON.stringify(test, null, 2); 48 | } 49 | 50 | public async postLimit(dto: LimitDto): Promise{ 51 | const test = await lastValueFrom( 52 | this.exampleClient.emit('limit.topic', { 53 | value: dto.message, 54 | }) 55 | ); 56 | return JSON.stringify(test, null, 2); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/modules/kafka/kafka.module.ts: -------------------------------------------------------------------------------- 1 | import { Global, Module } from "@nestjs/common"; 2 | import { ClientsModule, Transport } from "@nestjs/microservices"; 3 | import { KafkaService } from "./kafka.service"; 4 | import { CONFIG_KAFKA } from "src/config/config.export"; 5 | 6 | @Global() 7 | @Module({ 8 | imports: [ 9 | ClientsModule.register([ 10 | { 11 | name: CONFIG_KAFKA.KAFKA_SERVICE_NAME, 12 | transport: Transport.KAFKA, 13 | options: { 14 | client: { 15 | clientId: CONFIG_KAFKA.KAFKA_CLIENT_ID, 16 | brokers: [CONFIG_KAFKA.KAFKA_ADDRESS], 17 | }, 18 | producerOnlyMode: true, 19 | }, 20 | }, 21 | ]), 22 | ], 23 | providers: [ 24 | KafkaService 25 | ], 26 | exports: [ 27 | ClientsModule, 28 | KafkaService 29 | ] 30 | }) 31 | export class KafkaModule { } 32 | -------------------------------------------------------------------------------- /src/modules/kafka/kafka.service.ts: -------------------------------------------------------------------------------- 1 | import { Inject, Injectable, Logger } from '@nestjs/common'; 2 | import { ClientKafka } from '@nestjs/microservices'; 3 | import { Admin } from '@nestjs/microservices/external/kafka.interface'; 4 | import { ITopicMetadata } from 'kafkajs'; 5 | import { CONFIG_KAFKA } from 'src/config/config.export'; 6 | 7 | @Injectable() 8 | export class KafkaService { 9 | private kafkaAdmin: Admin; 10 | 11 | constructor( 12 | @Inject(CONFIG_KAFKA.KAFKA_SERVICE_NAME) 13 | private readonly clientKafka: ClientKafka 14 | ) { } 15 | 16 | async onModuleInit() { 17 | await this.clientKafka.connect(); 18 | this.kafkaAdmin = this.clientKafka['client'].admin(); 19 | await this.kafkaAdmin.connect(); 20 | } 21 | 22 | public async fetchTopicMetadata( 23 | topics?: string[] 24 | ): Promise<{ topics: ITopicMetadata[] }> { 25 | try { 26 | return await this.kafkaAdmin.fetchTopicMetadata({ topics }); 27 | } catch(err) { 28 | return err.message; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/app.e2e.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { INestApplication } from '@nestjs/common'; 3 | import * as request from 'supertest'; 4 | import { AppModule } from '../src/modules/app/app.module'; 5 | 6 | 7 | describe('AppController (e2e)', () => { 8 | let app: INestApplication; 9 | 10 | beforeEach(async () => { 11 | const moduleFixture: TestingModule = await Test.createTestingModule({ 12 | imports: [AppModule], 13 | }).compile(); 14 | 15 | app = moduleFixture.createNestApplication(); 16 | await app.init(); 17 | }); 18 | 19 | it('/ (GET)', () => { 20 | return request(app.getHttpServer()) 21 | .get('/') 22 | .expect(200) 23 | .expect('Hello World!'); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "ES2021", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false 20 | }, 21 | "include": ["src", "test"], 22 | } 23 | --------------------------------------------------------------------------------