├── .env.example ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── Makefile ├── README.md ├── docker-compose-issue1.yml ├── docker-compose-step1.yml ├── docker-compose-step2.yml ├── docker-compose.yml ├── docker └── Dockerfile ├── nest-cli.json ├── package-lock.json ├── package.json ├── src ├── common │ ├── common.module.ts │ ├── const.ts │ ├── dto │ │ └── trade-created.dto.ts │ └── utils │ │ └── delay.ts ├── container-role.ts ├── main.ts ├── monitor │ ├── monitor.module.ts │ └── monitor.service.ts └── stacks │ ├── default │ ├── cron │ │ ├── cron.module.ts │ │ └── cron.service.ts │ └── workers │ │ ├── keep-alive.service.ts │ │ ├── trade.service.ts │ │ └── workers.module.ts │ ├── diff-queues │ ├── cron │ │ ├── cron.module.ts │ │ └── cron.service.ts │ ├── default-worker │ │ ├── default.service.ts │ │ ├── keep-alive.service.ts │ │ └── workers.module.ts │ ├── trade-worker │ │ ├── keep-alive.service.ts │ │ ├── trades.service.ts │ │ └── workers.module.ts │ └── workers │ │ ├── default.service.ts │ │ ├── keep-alive.service.ts │ │ ├── trades.service.ts │ │ └── workers.module.ts │ └── issue1 │ └── cron │ ├── cron.module.ts │ └── cron.service.ts ├── tsconfig.build.json └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | CONTAINER_ROLE=api 2 | REDIS_HOST=127.0.0.1 3 | REDIS_PASSWORD= 4 | REDIS_PORT=6379 -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir : __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | 3 | # compiled output 4 | /dist 5 | /node_modules 6 | 7 | # Logs 8 | logs 9 | *.log 10 | npm-debug.log* 11 | pnpm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | 16 | # OS 17 | .DS_Store 18 | 19 | # Tests 20 | /coverage 21 | /.nyc_output 22 | 23 | # IDEs and editors 24 | /.idea 25 | .project 26 | .classpath 27 | .c9/ 28 | *.launch 29 | .settings/ 30 | *.sublime-workspace 31 | 32 | # IDE - VSCode 33 | .vscode/* 34 | !.vscode/settings.json 35 | !.vscode/tasks.json 36 | !.vscode/launch.json 37 | !.vscode/extensions.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | start: 2 | docker-compose up -d 3 | 4 | stop: 5 | docker-compose down 6 | 7 | start-issue1: 8 | docker-compose -f docker-compose-issue1.yml up -d 9 | 10 | stop-issue1: 11 | docker-compose -f docker-compose-issue1.yml down 12 | 13 | start-step1: 14 | docker-compose -f docker-compose-step1.yml up -d 15 | 16 | stop-step1: 17 | docker-compose -f docker-compose-step1.yml down 18 | 19 | start-step2: 20 | docker-compose -f docker-compose-step2.yml up -d 21 | 22 | stop-step2: 23 | docker-compose -f docker-compose-step2.yml down 24 | 25 | watch: 26 | docker logs -f conf42-event-driven-nestjs-demo_worker_1 27 | 28 | watch-cron: 29 | docker logs -f conf42-event-driven-nestjs-demo_cron_1 30 | 31 | monitor: 32 | npm run start:monitor -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scalable event-driven applications with NestJS (demo) 2 | 3 | Join me for a talk on developing [scalable event-driven applications with NestJS](https://www.youtube.com/watch?v=TEZOZQUcplM&list=PLIuxSyKxlQrBnO4ylf_HxSxtRr_j84zzN&index=13). 4 | 5 | An article based on my talk can be found at [Medium - Build Scalable Event-Driven Applications With Nest.js](https://medium.com/better-programming/build-scalable-event-driven-applications-with-nest-js-28676cb093d0). 6 | 7 | If you’ve never tried NestJS - I’ll talk briefly about its advantages and use cases it can solve for you. 8 | 9 | We’ll explore a hands-on example of scalability issues that can happen and the common approaches to solving them. 10 | 11 | Thanks and see you. 12 | 13 | ### Starting the app 14 | 15 | Install dependencies: \ 16 | `npm i` 17 | 18 | Pull Redis container: \ 19 | `docker pull redis:7-alpine` 20 | 21 | Build the application container: \ 22 | `DOCKER_BUILDKIT=1 docker build --pull -t conf42-demo -f docker/Dockerfile .` 23 | 24 | Start the stack: use Makefile commands 25 | 26 | ```makefile 27 | start: 28 | docker-compose up -d 29 | 30 | stop: 31 | docker-compose down 32 | 33 | start-issue1: 34 | docker-compose -f docker-compose-issue1.yml up -d 35 | 36 | stop-issue1: 37 | docker-compose -f docker-compose-issue1.yml down 38 | 39 | start-step1: 40 | docker-compose -f docker-compose-step1.yml up -d 41 | 42 | stop-step1: 43 | docker-compose -f docker-compose-step1.yml down 44 | 45 | start-step2: 46 | docker-compose -f docker-compose-step2.yml up -d 47 | 48 | stop-step2: 49 | docker-compose -f docker-compose-step2.yml down 50 | 51 | watch: 52 | docker logs -f conf42-event-driven-nestjs-demo_worker_1 53 | 54 | watch-cron: 55 | docker logs -f conf42-event-driven-nestjs-demo_cron_1 56 | 57 | monitor: 58 | npm run start:monitor 59 | ``` 60 | -------------------------------------------------------------------------------- /docker-compose-issue1.yml: -------------------------------------------------------------------------------- 1 | # issue1 demo stack: traffic increases, but workers are still unbalanced 2 | version: '3.8' 3 | 4 | services: 5 | redis: 6 | image: redis:7-alpine 7 | ports: 8 | - mode: host 9 | published: 6379 10 | target: 6379 11 | deploy: 12 | mode: global 13 | restart_policy: 14 | condition: on-failure 15 | logging: 16 | driver: json-file 17 | options: 18 | max-size: "100k" 19 | max-file: "1" 20 | 21 | cron: 22 | image: conf42-demo:latest 23 | deploy: 24 | mode: global 25 | restart_policy: 26 | condition: on-failure 27 | environment: 28 | - CONTAINER_ROLE=cron_issue1 29 | - REDIS_HOST=redis 30 | volumes: 31 | - .:/app 32 | logging: 33 | driver: json-file 34 | options: 35 | max-size: "100k" 36 | max-file: "1" 37 | 38 | worker: 39 | image: conf42-demo:latest 40 | deploy: 41 | mode: replicated 42 | replicas: 1 43 | restart_policy: 44 | condition: on-failure 45 | environment: 46 | - CONTAINER_ROLE=worker 47 | - REDIS_HOST=redis 48 | volumes: 49 | - .:/app 50 | logging: 51 | driver: json-file 52 | options: 53 | max-size: "100k" 54 | max-file: "1" -------------------------------------------------------------------------------- /docker-compose-step1.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | redis: 5 | image: redis:7-alpine 6 | ports: 7 | - mode: host 8 | published: 6379 9 | target: 6379 10 | deploy: 11 | mode: global 12 | restart_policy: 13 | condition: on-failure 14 | logging: 15 | driver: json-file 16 | options: 17 | max-size: "100k" 18 | max-file: "1" 19 | 20 | cron: 21 | image: conf42-demo:latest 22 | deploy: 23 | mode: global 24 | restart_policy: 25 | condition: on-failure 26 | environment: 27 | - CONTAINER_ROLE=cron_1 28 | - REDIS_HOST=redis 29 | volumes: 30 | - .:/app 31 | logging: 32 | driver: json-file 33 | options: 34 | max-size: "100k" 35 | max-file: "1" 36 | 37 | worker: 38 | image: conf42-demo:latest 39 | deploy: 40 | mode: replicated 41 | replicas: 1 42 | restart_policy: 43 | condition: on-failure 44 | environment: 45 | - CONTAINER_ROLE=worker_1 46 | - REDIS_HOST=redis 47 | volumes: 48 | - .:/app 49 | logging: 50 | driver: json-file 51 | options: 52 | max-size: "100k" 53 | max-file: "1" -------------------------------------------------------------------------------- /docker-compose-step2.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | redis: 5 | image: redis:7-alpine 6 | ports: 7 | - mode: host 8 | published: 6379 9 | target: 6379 10 | deploy: 11 | mode: global 12 | restart_policy: 13 | condition: on-failure 14 | logging: 15 | driver: json-file 16 | options: 17 | max-size: "100k" 18 | max-file: "1" 19 | 20 | cron: 21 | image: conf42-demo:latest 22 | deploy: 23 | mode: global 24 | restart_policy: 25 | condition: on-failure 26 | environment: 27 | - CONTAINER_ROLE=cron_1 28 | - REDIS_HOST=redis 29 | volumes: 30 | - .:/app 31 | logging: 32 | driver: json-file 33 | options: 34 | max-size: "100k" 35 | max-file: "1" 36 | 37 | worker-trade: 38 | image: conf42-demo:latest 39 | deploy: 40 | mode: replicated 41 | replicas: 1 42 | restart_policy: 43 | condition: on-failure 44 | environment: 45 | - CONTAINER_ROLE=worker_trade 46 | - REDIS_HOST=redis 47 | volumes: 48 | - .:/app 49 | logging: 50 | driver: json-file 51 | options: 52 | max-size: "100k" 53 | max-file: "1" 54 | 55 | worker-default: 56 | image: conf42-demo:latest 57 | deploy: 58 | mode: replicated 59 | replicas: 3 60 | restart_policy: 61 | condition: on-failure 62 | environment: 63 | - CONTAINER_ROLE=worker_default 64 | - REDIS_HOST=redis 65 | volumes: 66 | - .:/app 67 | logging: 68 | driver: json-file 69 | options: 70 | max-size: "100k" 71 | max-file: "1" -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # default demo stack: steady workflow 2 | version: '3.8' 3 | 4 | services: 5 | redis: 6 | image: redis:7-alpine 7 | ports: 8 | - mode: host 9 | published: 6379 10 | target: 6379 11 | deploy: 12 | mode: global 13 | restart_policy: 14 | condition: on-failure 15 | logging: 16 | driver: json-file 17 | options: 18 | max-size: "100k" 19 | max-file: "1" 20 | 21 | cron: 22 | image: conf42-demo:latest 23 | deploy: 24 | mode: global 25 | restart_policy: 26 | condition: on-failure 27 | environment: 28 | - CONTAINER_ROLE=cron 29 | - REDIS_HOST=redis 30 | volumes: 31 | - .:/app 32 | logging: 33 | driver: json-file 34 | options: 35 | max-size: "100k" 36 | max-file: "1" 37 | 38 | worker: 39 | image: conf42-demo:latest 40 | deploy: 41 | mode: replicated 42 | replicas: 1 43 | restart_policy: 44 | condition: on-failure 45 | environment: 46 | - CONTAINER_ROLE=worker 47 | - REDIS_HOST=redis 48 | volumes: 49 | - .:/app 50 | logging: 51 | driver: json-file 52 | options: 53 | max-size: "100k" 54 | max-file: "1" -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:16-alpine as builder 2 | WORKDIR /app 3 | COPY .. . 4 | RUN npm i 5 | CMD ["npm", "run", "start:dev"] 6 | -------------------------------------------------------------------------------- /nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src" 5 | } 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "conf42-event-driven-nestjs-demo", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "prebuild": "rimraf dist", 10 | "build": "nest build", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 12 | "start": "nest start", 13 | "start:dev": "nest start --watch", 14 | "start:debug": "nest start --debug --watch", 15 | "start:prod": "node dist/main", 16 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 17 | "start:monitor": "CONTAINER_ROLE=monitor nest start --watch", 18 | "start:worker": "CONTAINER_ROLE=worker nest start --watch", 19 | "start:cron": "CONTAINER_ROLE=cron nest start --watch" 20 | }, 21 | "dependencies": { 22 | "@nestjs/bull": "^0.6.1", 23 | "@nestjs/common": "^9.0.0", 24 | "@nestjs/config": "^2.2.0", 25 | "@nestjs/core": "^9.0.0", 26 | "@nestjs/schedule": "^2.1.0", 27 | "cache-manager": "^5.1.1", 28 | "cache-manager-ioredis": "^2.1.0", 29 | "ioredis": "^5.2.3", 30 | "reflect-metadata": "^0.1.13", 31 | "rimraf": "^3.0.2", 32 | "rxjs": "^7.2.0" 33 | }, 34 | "devDependencies": { 35 | "@nestjs/cli": "^9.0.0", 36 | "@nestjs/schematics": "^9.0.0", 37 | "@nestjs/testing": "^9.0.0", 38 | "@types/express": "^4.17.13", 39 | "@types/jest": "28.1.8", 40 | "@types/node": "^16.18.0", 41 | "@types/supertest": "^2.0.11", 42 | "@typescript-eslint/eslint-plugin": "^5.0.0", 43 | "@typescript-eslint/parser": "^5.0.0", 44 | "eslint": "^8.0.1", 45 | "eslint-config-prettier": "^8.3.0", 46 | "eslint-plugin-prettier": "^4.0.0", 47 | "jest": "28.1.3", 48 | "prettier": "^2.3.2", 49 | "source-map-support": "^0.5.20", 50 | "supertest": "^6.1.3", 51 | "ts-jest": "28.0.8", 52 | "ts-loader": "^9.2.3", 53 | "ts-node": "^10.0.0", 54 | "tsconfig-paths": "4.1.0", 55 | "typescript": "^4.7.4" 56 | }, 57 | "jest": { 58 | "moduleFileExtensions": [ 59 | "js", 60 | "json", 61 | "ts" 62 | ], 63 | "rootDir": "src", 64 | "testRegex": ".*\\.spec\\.ts$", 65 | "transform": { 66 | "^.+\\.(t|j)s$": "ts-jest" 67 | }, 68 | "collectCoverageFrom": [ 69 | "**/*.(t|j)s" 70 | ], 71 | "coverageDirectory": "../coverage", 72 | "testEnvironment": "node" 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/common/common.module.ts: -------------------------------------------------------------------------------- 1 | import { BullModule } from '@nestjs/bull'; 2 | import { Module } from '@nestjs/common'; 3 | import { ConfigModule, ConfigService } from '@nestjs/config'; 4 | import { QUEUE_DEFAULT, QUEUE_TRADES } from './const'; 5 | 6 | @Module({ 7 | imports: [ 8 | ConfigModule.forRoot({ 9 | isGlobal: true, 10 | }), 11 | BullModule.forRootAsync({ 12 | imports: [ConfigModule], 13 | useFactory: async (configService: ConfigService) => ({ 14 | redis: { 15 | host: configService.get('REDIS_HOST') || '127.0.0.1', 16 | port: +configService.get('REDIS_PORT') || 6379, 17 | password: configService.get('REDIS_PASSWORD') || undefined, 18 | }, 19 | }), 20 | inject: [ConfigService], 21 | }), 22 | BullModule.registerQueue({ 23 | name: QUEUE_DEFAULT, 24 | }), 25 | BullModule.registerQueue({ 26 | name: QUEUE_TRADES, 27 | }), 28 | ], 29 | exports: [BullModule], 30 | }) 31 | export class CommonModule {} 32 | -------------------------------------------------------------------------------- /src/common/const.ts: -------------------------------------------------------------------------------- 1 | export const QUEUE_DEFAULT = 'default'; 2 | export const QUEUE_TRADES = 'trades'; 3 | 4 | export const JOB_ANALYTICS = 'analytics'; 5 | export const JOB_NOTIFICATION = 'notification'; 6 | export const JOB_STORE = 'store'; 7 | export const JOB_TRADE_CONFIRM = 'trade-confirm'; 8 | -------------------------------------------------------------------------------- /src/common/dto/trade-created.dto.ts: -------------------------------------------------------------------------------- 1 | export class TradeCreatedDto { 2 | uuid: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/common/utils/delay.ts: -------------------------------------------------------------------------------- 1 | export const delay = function (time) { 2 | return new Promise((resolve) => setTimeout(resolve, time)); 3 | }; 4 | -------------------------------------------------------------------------------- /src/container-role.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from '@nestjs/core'; 2 | import { CronModule } from './stacks/default/cron/cron.module'; 3 | import { CronModuleStep1 } from './stacks/diff-queues/cron/cron.module'; 4 | import { DefaultWorkersModuleStep2 } from './stacks/diff-queues/default-worker/workers.module'; 5 | import { TradeWorkersModuleStep2 } from './stacks/diff-queues/trade-worker/workers.module'; 6 | import { WorkersModuleStep1 } from './stacks/diff-queues/workers/workers.module'; 7 | import { MonitorModule } from './monitor/monitor.module'; 8 | import { WorkersModule } from './stacks/default/workers/workers.module'; 9 | import { CronModuleIssue1 } from './stacks/issue1/cron/cron.module'; 10 | 11 | export const rolesMapBootstrap = { 12 | monitor: async () => { 13 | return await NestFactory.createApplicationContext(MonitorModule); 14 | }, 15 | 16 | // default stack 17 | cron: async () => { 18 | return await NestFactory.createApplicationContext(CronModule); 19 | }, 20 | worker: async () => { 21 | return await NestFactory.createApplicationContext(WorkersModule); 22 | }, 23 | 24 | // issue 1: increased traffic 25 | cron_issue1: async () => { 26 | return await NestFactory.createApplicationContext(CronModuleIssue1); 27 | }, 28 | 29 | // step 1 upgrade 30 | cron_1: async () => { 31 | return await NestFactory.createApplicationContext(CronModuleStep1); 32 | }, 33 | worker_1: async () => { 34 | return await NestFactory.createApplicationContext(WorkersModuleStep1); 35 | }, 36 | 37 | // step 2 upgrade 38 | worker_default: async () => { 39 | return await NestFactory.createApplicationContext( 40 | DefaultWorkersModuleStep2, 41 | ); 42 | }, 43 | worker_trade: async () => { 44 | return await NestFactory.createApplicationContext(TradeWorkersModuleStep2); 45 | }, 46 | }; 47 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from '@nestjs/common'; 2 | import { rolesMapBootstrap } from './container-role'; 3 | 4 | async function bootstrap() { 5 | const role = process.env.CONTAINER_ROLE || 'worker'; 6 | 7 | await rolesMapBootstrap[role](); 8 | 9 | const logger = new Logger(); 10 | logger.log(`Container role: ${role}`); 11 | } 12 | bootstrap(); 13 | -------------------------------------------------------------------------------- /src/monitor/monitor.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../common/common.module'; 4 | import { MonitorService } from './monitor.service'; 5 | 6 | @Module({ 7 | imports: [CommonModule, ScheduleModule.forRoot()], 8 | providers: [MonitorService], 9 | }) 10 | export class MonitorModule {} 11 | -------------------------------------------------------------------------------- /src/monitor/monitor.service.ts: -------------------------------------------------------------------------------- 1 | import { InjectQueue } from '@nestjs/bull'; 2 | import { Injectable } from '@nestjs/common'; 3 | import { ConfigService } from '@nestjs/config'; 4 | import { Cron, CronExpression } from '@nestjs/schedule'; 5 | import { Queue } from 'bull'; 6 | import { QUEUE_DEFAULT, QUEUE_TRADES } from '../common/const'; 7 | import Redis from 'ioredis'; 8 | 9 | @Injectable() 10 | export class MonitorService { 11 | private readonly redis: Redis; 12 | 13 | constructor( 14 | @InjectQueue(QUEUE_DEFAULT) private queue: Queue, 15 | @InjectQueue(QUEUE_TRADES) private queueTrades: Queue, 16 | private readonly config: ConfigService, 17 | ) { 18 | this.redis = new Redis({ 19 | host: config.get('REDIS_HOST') || '127.0.0.1', 20 | port: this.config.get('REDIS_PORT') || 6379, 21 | password: config.get('REDIS_PASSWORD') || undefined, 22 | db: 2, 23 | }); 24 | } 25 | 26 | @Cron(CronExpression.EVERY_SECOND) 27 | async report() { 28 | const workers_count_default = 29 | (await this.getWorkersCount('workers:timestamps'))?.length || 30 | (await this.getWorkersCount('workers-default:timestamps'))?.length; 31 | 32 | const workers_count_trade = 33 | (await this.getWorkersCount('workers:timestamps'))?.length || 34 | (await this.getWorkersCount('workers-trade:timestamps'))?.length; 35 | 36 | const data = [ 37 | { 38 | queue: QUEUE_DEFAULT, 39 | jobs_waiting: await this.queue.getWaitingCount(), 40 | jobs_completed: await this.queue.getCompletedCount(), 41 | workers_count: workers_count_default, 42 | }, 43 | { 44 | queue: QUEUE_TRADES, 45 | jobs_waiting: await this.queueTrades.getWaitingCount(), 46 | jobs_completed: await this.queueTrades.getCompletedCount(), 47 | workers_count: workers_count_trade, 48 | }, 49 | ]; 50 | 51 | console.clear(); 52 | console.table(data.filter((d) => d.jobs_completed > 0)); 53 | } 54 | 55 | private async getWorkersCount(name: string) { 56 | return this.redis.zrangebyscore(name, Date.now() - 2000, Date.now()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/stacks/default/cron/cron.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { CronService } from './cron.service'; 5 | 6 | @Module({ 7 | imports: [CommonModule, ScheduleModule.forRoot()], 8 | providers: [CronService], 9 | }) 10 | export class CronModule {} 11 | -------------------------------------------------------------------------------- /src/stacks/default/cron/cron.service.ts: -------------------------------------------------------------------------------- 1 | import { InjectQueue } from '@nestjs/bull'; 2 | import { Injectable, Logger } from '@nestjs/common'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { Queue } from 'bull'; 5 | import { randomUUID } from 'crypto'; 6 | import { 7 | JOB_ANALYTICS, 8 | JOB_NOTIFICATION, 9 | JOB_STORE, 10 | JOB_TRADE_CONFIRM, 11 | QUEUE_DEFAULT, 12 | } from '../../../common/const'; 13 | 14 | @Injectable() 15 | export class CronService { 16 | protected readonly logger = new Logger(this.constructor.name); 17 | private readonly tradesPercycle: number; 18 | 19 | constructor(@InjectQueue(QUEUE_DEFAULT) private queue: Queue) { 20 | this.tradesPercycle = 1; 21 | } 22 | 23 | @Cron(CronExpression.EVERY_SECOND) 24 | async generate() { 25 | for (let i = 0; i < this.tradesPercycle; i++) { 26 | const uuid = randomUUID(); 27 | this.queue.add(JOB_ANALYTICS, { uuid }); 28 | this.queue.add(JOB_NOTIFICATION, { uuid }); 29 | this.queue.add(JOB_STORE, { uuid }); 30 | this.queue.add(JOB_TRADE_CONFIRM, { uuid }); 31 | this.logger.log(`Trade ${uuid} created`); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/stacks/default/workers/keep-alive.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { randomUUID } from 'crypto'; 5 | import Redis from 'ioredis'; 6 | 7 | @Injectable() 8 | export class KeepAliveService { 9 | private readonly uuid = randomUUID(); 10 | private readonly redis: Redis; 11 | 12 | constructor(private readonly config: ConfigService) { 13 | this.redis = new Redis({ 14 | host: config.get('REDIS_HOST') || '127.0.0.1', 15 | port: this.config.get('REDIS_PORT') || 6379, 16 | password: config.get('REDIS_PASSWORD') || undefined, 17 | db: 2, 18 | }); 19 | } 20 | 21 | @Cron(CronExpression.EVERY_SECOND) 22 | keepAlive() { 23 | this.redis.zadd('workers:timestamps', Date.now(), this.uuid); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/stacks/default/workers/trade.service.ts: -------------------------------------------------------------------------------- 1 | import { Process, Processor } from '@nestjs/bull'; 2 | import { Logger } from '@nestjs/common'; 3 | import { Job } from 'bull'; 4 | import { QUEUE_DEFAULT } from '../../../common/const'; 5 | import { TradeCreatedDto } from '../../../common/dto/trade-created.dto'; 6 | import { delay } from '../../../common/utils/delay'; 7 | 8 | @Processor(QUEUE_DEFAULT) 9 | export class TradeService { 10 | protected readonly logger = new Logger(this.constructor.name); 11 | 12 | @Process({ name: '*', concurrency: 1 }) 13 | async process(job: Job) { 14 | await delay(150); 15 | this.logger.log(`${job.name} - ${job.data.uuid}`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/stacks/default/workers/workers.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { KeepAliveService } from './keep-alive.service'; 5 | import { TradeService } from './trade.service'; 6 | 7 | @Module({ 8 | imports: [CommonModule, ScheduleModule.forRoot()], 9 | providers: [TradeService, KeepAliveService], 10 | }) 11 | export class WorkersModule {} 12 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/cron/cron.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { CronService } from './cron.service'; 5 | 6 | @Module({ 7 | imports: [CommonModule, ScheduleModule.forRoot()], 8 | providers: [CronService], 9 | }) 10 | export class CronModuleStep1 {} 11 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/cron/cron.service.ts: -------------------------------------------------------------------------------- 1 | import { InjectQueue } from '@nestjs/bull'; 2 | import { Injectable, Logger } from '@nestjs/common'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { Queue } from 'bull'; 5 | import { randomUUID } from 'crypto'; 6 | import { 7 | JOB_ANALYTICS, 8 | JOB_NOTIFICATION, 9 | JOB_STORE, 10 | JOB_TRADE_CONFIRM, 11 | QUEUE_DEFAULT, 12 | QUEUE_TRADES, 13 | } from '../../../common/const'; 14 | 15 | @Injectable() 16 | export class CronService { 17 | protected readonly logger = new Logger(this.constructor.name); 18 | private readonly tradesPercycle: number; 19 | 20 | constructor( 21 | @InjectQueue(QUEUE_DEFAULT) private queue: Queue, 22 | @InjectQueue(QUEUE_TRADES) private queueTrades: Queue, 23 | ) { 24 | this.tradesPercycle = 5; 25 | } 26 | 27 | @Cron(CronExpression.EVERY_SECOND) 28 | async generate() { 29 | for (let i = 0; i < this.tradesPercycle; i++) { 30 | const uuid = randomUUID(); 31 | this.queue.add(JOB_ANALYTICS, { uuid }); 32 | this.queue.add(JOB_NOTIFICATION, { uuid }); 33 | this.queue.add(JOB_STORE, { uuid }); 34 | // this.queue.add(JOB_TRADE_CONFIRM, { uuid }); 35 | this.queueTrades.add(JOB_TRADE_CONFIRM, { uuid }); 36 | this.logger.log(`Trade ${uuid} created`); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/default-worker/default.service.ts: -------------------------------------------------------------------------------- 1 | import { Process, Processor } from '@nestjs/bull'; 2 | import { Logger } from '@nestjs/common'; 3 | import { Job } from 'bull'; 4 | import { QUEUE_DEFAULT } from '../../../common/const'; 5 | import { TradeCreatedDto } from '../../../common/dto/trade-created.dto'; 6 | import { delay } from '../../../common/utils/delay'; 7 | 8 | @Processor(QUEUE_DEFAULT) 9 | export class DefaultService { 10 | protected readonly logger = new Logger(this.constructor.name); 11 | 12 | @Process({ name: '*', concurrency: 1 }) 13 | async process(job: Job) { 14 | await delay(150); 15 | this.logger.log(`${job.name} - ${job.data.uuid}`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/default-worker/keep-alive.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { randomUUID } from 'crypto'; 5 | import Redis from 'ioredis'; 6 | 7 | @Injectable() 8 | export class KeepAliveService { 9 | private readonly uuid = randomUUID(); 10 | private readonly redis: Redis; 11 | 12 | constructor(private readonly config: ConfigService) { 13 | this.redis = new Redis({ 14 | host: config.get('REDIS_HOST') || '127.0.0.1', 15 | port: this.config.get('REDIS_PORT') || 6379, 16 | password: config.get('REDIS_PASSWORD') || undefined, 17 | db: 2, 18 | }); 19 | } 20 | 21 | @Cron(CronExpression.EVERY_SECOND) 22 | keepAlive() { 23 | this.redis.zadd('workers-default:timestamps', Date.now(), this.uuid); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/default-worker/workers.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { DefaultService } from './default.service'; 5 | import { KeepAliveService } from './keep-alive.service'; 6 | 7 | @Module({ 8 | imports: [CommonModule, ScheduleModule.forRoot()], 9 | providers: [DefaultService, KeepAliveService], 10 | }) 11 | export class DefaultWorkersModuleStep2 {} 12 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/trade-worker/keep-alive.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { randomUUID } from 'crypto'; 5 | import Redis from 'ioredis'; 6 | 7 | @Injectable() 8 | export class KeepAliveService { 9 | private readonly uuid = randomUUID(); 10 | private readonly redis: Redis; 11 | 12 | constructor(private readonly config: ConfigService) { 13 | this.redis = new Redis({ 14 | host: config.get('REDIS_HOST') || '127.0.0.1', 15 | port: this.config.get('REDIS_PORT') || 6379, 16 | password: config.get('REDIS_PASSWORD') || undefined, 17 | db: 2, 18 | }); 19 | } 20 | 21 | @Cron(CronExpression.EVERY_SECOND) 22 | keepAlive() { 23 | this.redis.zadd('workers-trade:timestamps', Date.now(), this.uuid); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/trade-worker/trades.service.ts: -------------------------------------------------------------------------------- 1 | import { Process, Processor } from '@nestjs/bull'; 2 | import { Logger } from '@nestjs/common'; 3 | import { Job } from 'bull'; 4 | import { QUEUE_TRADES } from '../../../common/const'; 5 | import { TradeCreatedDto } from '../../../common/dto/trade-created.dto'; 6 | import { delay } from '../../../common/utils/delay'; 7 | 8 | @Processor(QUEUE_TRADES) 9 | export class TradesService { 10 | protected readonly logger = new Logger(this.constructor.name); 11 | 12 | @Process({ name: '*', concurrency: 1 }) 13 | async process(job: Job) { 14 | await delay(150); 15 | this.logger.verbose(`${job.name} - ${job.data.uuid}`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/trade-worker/workers.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { KeepAliveService } from './keep-alive.service'; 5 | import { TradesService } from './trades.service'; 6 | 7 | @Module({ 8 | imports: [CommonModule, ScheduleModule.forRoot()], 9 | providers: [KeepAliveService, TradesService], 10 | }) 11 | export class TradeWorkersModuleStep2 {} 12 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/workers/default.service.ts: -------------------------------------------------------------------------------- 1 | import { Process, Processor } from '@nestjs/bull'; 2 | import { Logger } from '@nestjs/common'; 3 | import { Job } from 'bull'; 4 | import { QUEUE_DEFAULT } from '../../../common/const'; 5 | import { TradeCreatedDto } from '../../../common/dto/trade-created.dto'; 6 | import { delay } from '../../../common/utils/delay'; 7 | 8 | @Processor(QUEUE_DEFAULT) 9 | export class DefaultService { 10 | protected readonly logger = new Logger(this.constructor.name); 11 | 12 | @Process({ name: '*', concurrency: 1 }) 13 | async process(job: Job) { 14 | await delay(150); 15 | this.logger.log(`${job.name} - ${job.data.uuid}`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/workers/keep-alive.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { randomUUID } from 'crypto'; 5 | import Redis from 'ioredis'; 6 | 7 | @Injectable() 8 | export class KeepAliveService { 9 | private readonly uuid = randomUUID(); 10 | private readonly redis: Redis; 11 | 12 | constructor(private readonly config: ConfigService) { 13 | this.redis = new Redis({ 14 | host: config.get('REDIS_HOST') || '127.0.0.1', 15 | port: this.config.get('REDIS_PORT') || 6379, 16 | password: config.get('REDIS_PASSWORD') || undefined, 17 | db: 2, 18 | }); 19 | } 20 | 21 | @Cron(CronExpression.EVERY_SECOND) 22 | keepAlive() { 23 | this.redis.zadd('workers:timestamps', Date.now(), this.uuid); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/workers/trades.service.ts: -------------------------------------------------------------------------------- 1 | import { Process, Processor } from '@nestjs/bull'; 2 | import { Logger } from '@nestjs/common'; 3 | import { Job } from 'bull'; 4 | import { QUEUE_TRADES } from '../../../common/const'; 5 | import { TradeCreatedDto } from '../../../common/dto/trade-created.dto'; 6 | import { delay } from '../../../common/utils/delay'; 7 | 8 | @Processor(QUEUE_TRADES) 9 | export class TradesService { 10 | protected readonly logger = new Logger(this.constructor.name); 11 | 12 | @Process({ name: '*', concurrency: 1 }) 13 | async process(job: Job) { 14 | await delay(150); 15 | this.logger.verbose(`${job.name} - ${job.data.uuid}`); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/stacks/diff-queues/workers/workers.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { DefaultService } from './default.service'; 5 | import { KeepAliveService } from './keep-alive.service'; 6 | import { TradesService } from './trades.service'; 7 | 8 | @Module({ 9 | imports: [CommonModule, ScheduleModule.forRoot()], 10 | providers: [DefaultService, KeepAliveService, TradesService], 11 | }) 12 | export class WorkersModuleStep1 {} 13 | -------------------------------------------------------------------------------- /src/stacks/issue1/cron/cron.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ScheduleModule } from '@nestjs/schedule'; 3 | import { CommonModule } from '../../../common/common.module'; 4 | import { CronService } from './cron.service'; 5 | 6 | @Module({ 7 | imports: [CommonModule, ScheduleModule.forRoot()], 8 | providers: [CronService], 9 | }) 10 | export class CronModuleIssue1 {} 11 | -------------------------------------------------------------------------------- /src/stacks/issue1/cron/cron.service.ts: -------------------------------------------------------------------------------- 1 | import { InjectQueue } from '@nestjs/bull'; 2 | import { Injectable, Logger } from '@nestjs/common'; 3 | import { Cron, CronExpression } from '@nestjs/schedule'; 4 | import { Queue } from 'bull'; 5 | import { randomUUID } from 'crypto'; 6 | import { 7 | JOB_ANALYTICS, 8 | JOB_NOTIFICATION, 9 | JOB_STORE, 10 | JOB_TRADE_CONFIRM, 11 | QUEUE_DEFAULT, 12 | } from '../../../common/const'; 13 | 14 | @Injectable() 15 | export class CronService { 16 | protected readonly logger = new Logger(this.constructor.name); 17 | private readonly tradesPercycle: number; 18 | 19 | constructor(@InjectQueue(QUEUE_DEFAULT) private queue: Queue) { 20 | this.tradesPercycle = 3; 21 | } 22 | 23 | @Cron(CronExpression.EVERY_SECOND) 24 | async generate() { 25 | for (let i = 0; i < this.tradesPercycle; i++) { 26 | const uuid = randomUUID(); 27 | this.queue.add(JOB_ANALYTICS, { uuid }); 28 | this.queue.add(JOB_NOTIFICATION, { uuid }); 29 | this.queue.add(JOB_STORE, { uuid }); 30 | this.queue.add(JOB_TRADE_CONFIRM, { uuid }); 31 | this.logger.log(`Trade ${uuid} created`); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "es2017", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false 20 | } 21 | } 22 | --------------------------------------------------------------------------------