├── .env.example ├── .gitignore ├── .prettierignore ├── .prettierrc ├── LICENSE ├── README.md ├── docs └── api-spec.yml ├── jest.config.js ├── package-lock.json ├── package.json ├── secrets.yml.example ├── serverless.offline.yml ├── serverless.yml ├── src ├── constants │ └── environment.constants.ts ├── controllers │ └── message │ │ ├── index.ts │ │ └── message.controller.ts ├── enums │ └── environment.enum.ts ├── errors │ ├── base.error.ts │ ├── non-retriable.error.ts │ └── partial-failure.error.ts ├── http-app.ts ├── interfaces │ ├── dequeued-message.interface.ts │ ├── message.interface.ts │ ├── queue-service.interface.ts │ ├── queued-message.interface.ts │ └── response.interface.ts ├── middlewares │ ├── api-spec.middleware.ts │ ├── error.middleware.ts │ └── validator.middleware.ts ├── routers │ └── message.router.ts ├── serverless-consumer.ts ├── serverless-producer.ts ├── services │ ├── event │ │ ├── event.service.ts │ │ └── index.ts │ ├── log │ │ ├── index.ts │ │ └── log.service.ts │ └── queue │ │ ├── index.ts │ │ ├── local-queue.service.ts │ │ └── sqs-queue.service.ts └── validators │ └── message.validator.ts ├── test ├── .env.test ├── controllers │ └── message.controller.spec.ts ├── env-setup.ts ├── middlewares │ ├── api-spec.middleware.spec.ts │ ├── error.middleware.spec.ts │ └── validator.middleware.spec.ts ├── mock-factories │ ├── aws-sdk.mock.factory.ts │ ├── express.mock-factory.ts │ └── message.mock-factory.ts ├── services │ ├── event-service.spec.ts │ ├── log.service.spec.ts │ └── sqs-queue.service.spec.ts └── validators │ └── message.validator.spec.ts └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | # Use environment local to use the mocked queue service 2 | ENVIRONMENT=local 3 | 4 | # Your aws credentials (only needed if you do not want to use the mocked queue service) 5 | AWS_ACCESS_KEY_ID= 6 | AWS_SECRET_ACCESS_KEY= 7 | AWS_REGION= 8 | 9 | # SQS main message queue url (only needed if you do not want to use the mocked queue service) 10 | MESSAGE_QUEUE_URL= 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Output 2 | /dist 3 | 4 | # Libraries 5 | /node_modules 6 | 7 | # Logs 8 | logs 9 | *.log 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | lerna-debug.log* 14 | 15 | # OS 16 | .DS_Store 17 | 18 | # Tests 19 | /coverage 20 | /.nyc_output 21 | /lcov-report 22 | *.info 23 | 24 | # IDEs and editors 25 | /.idea 26 | .project 27 | .classpath 28 | .c9/ 29 | *.launch 30 | .settings/ 31 | *.sublime-workspace 32 | .vscode/ 33 | 34 | # IDE - VSCode 35 | .vscode/* 36 | !.vscode/settings.json 37 | !.vscode/tasks.json 38 | !.vscode/launch.json 39 | !.vscode/extensions.json 40 | 41 | # Serverless 42 | /.build 43 | /.serverless 44 | /_warmup 45 | /_optimize 46 | .webpack/* 47 | 48 | # Local files 49 | /.env 50 | secrets.yml 51 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /package.json 2 | /package-lock.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Limehome 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # serverless-express-typescript-sqs 2 | 3 | This project provides a full starter kit for a [serverless](https://www.serverless.com/) producer consumer queue setup implemented with [AWS Lambda](https://aws.amazon.com/lambda) and [SQS](https://aws.amazon.com/sqs/). 4 | The goal is that you can use this code base to add your domain logic on top of it. 5 | 6 | - [serverless-express-typescript-sqs](#serverless-express-typescript-sqs) 7 | - [Abstract overview](#abstract-overview) 8 | - [Producer function](#producer-function) 9 | - [Consumer function](#consumer-function) 10 | - [Purpose](#purpose) 11 | - [Content](#content) 12 | - [Project structure](#project-structure) 13 | - [Development guides](#development-guides) 14 | - [Create a new service or controller](#create-a-new-service-or-controller) 15 | - [Add a new http endpoint](#add-a-new-http-endpoint) 16 | - [Define message processing logic](#define-message-processing-logic) 17 | - [Change Dead Letter Queue behavior](#change-dead-letter-queue-behavior) 18 | - [Change message processing retry interval time](#change-message-processing-retry-interval-time) 19 | - [Add a new serverless function](#add-a-new-serverless-function) 20 | - [Change handling of unexpected errors during endpoint access](#change-handling-of-unexpected-errors-during-endpoint-access) 21 | - [Change validation error response format](#change-validation-error-response-format) 22 | - [Define environment variables (local)](#define-environment-variables-local) 23 | - [Define secrets (AWS Lambda)](#define-secrets-aws-lambda) 24 | - [Run locally](#run-locally) 25 | - [Install and start project](#install-and-start-project) 26 | - [Produce message](#produce-message) 27 | - [Consume messages](#consume-messages) 28 | - [Run tests](#run-tests) 29 | - [Format](#format) 30 | - [Deploy on AWS](#deploy-on-aws) 31 | - [Known issues](#known-issues) 32 | - [Contribution](#contribution) 33 | 34 | ## Abstract overview 35 | The project consists out of two serverless functions: 36 | 37 | ### Producer function 38 | 1. Provides **http endpoints** that can be called to produce messages 39 | 2. Producer endpoints will **enqueue** messages into a queue 40 | 41 | ### Consumer function 42 | 1. Will be **triggered** when a message was passed to the queue 43 | 2. Payload of the functions are enqueued messages 44 | 3. Each message will be processed (e.g. by sending to another http api) 45 | 4. Proccessing can lead to two kind of errors: 46 | - **Retriable errors** 47 | - **Non retriable errors** 48 | 5. If processing of a message leads to a **retriable error** the message will **stay** in the queue 49 | 6. If processing of a message leads to a **non retriable error** the message will be **deleted** from the queue 50 | 7. If processing of a message was **successful** the message will be also **deleted** from the queue 51 | 8. After a message caused **n** times a **retriable error** it will be moved to a so called **Dead Letter Queue** 52 | - Message will stay in that queue for a specific time (Maximum 7 days) and will not be processed anymore 53 | 54 | ## Purpose 55 | There are scenarios in which the processing of a message in the consumer could fail unexpectedly. For the example an api could become temporarly unavailable. This queue architecture makes sure that the messages will be kept and processing retried multiple times. 56 | 57 | ## Content 58 | The project was created with TypeScript and Node.js. It contains the following content: 59 | 60 | - Serverless setup for AWS Lambda with [Serverless](https://www.npmjs.com/package/serverless) framework 61 | - SQS interaction with [AWS SDK](https://www.npmjs.com/package/aws-sdk) 62 | - Queue error handling setup with [AWS SDK](https://www.npmjs.com/package/aws-sdk) 63 | - Http server setup with [Express](https://www.npmjs.com/package/express) 64 | - Request validation with [Express validator](https://www.npmjs.com/package/express-validator) 65 | - Swagger setup with [Swagger UI Express](https://www.npmjs.com/package/swagger-ui-express) 66 | - Unit test setup with [Jest](https://www.npmjs.com/package/jest) 67 | - Formatting setup with [Prettier](https://www.npmjs.com/package/prettier) 68 | 69 | ## Project structure 70 | 71 | **Configuration files** 72 | | Path | Description | 73 | | ------------------------ | :---------------------------------------------------------- | 74 | | `.env.example` | Example environment variable configuration | 75 | | `.prettierignore` | Files to exclude from formatting | 76 | | `.prettierrc` | Prettier formatting config | 77 | | `jest.config.js` | Jest config file for unit tests | 78 | | `secrets.yml.example` | Example secret configuration to run functions on AWS Lambda | 79 | | `serverless.offline.yml` | Serverless configuration to run functions locally | 80 | | `serverless.yml` | Servlerless configuration to run functions on AWS Lambda | 81 | | `tsconfig.json` | TypeScript configuration file | 82 | 83 | **`/docs` folder** 84 | - `/docs` contains everything that has to do with documentation about the project 85 | - `/docs/api-spec.yml` contains the swagger specification for the http app 86 | 87 | **`/test` folder** 88 | - `/test` contains all unit tests executed with Jest 89 | - Has also similar folder structure to `src` 90 | - `env.test` contains environment variable setup for tests 91 | 92 | **`/src` folder** 93 | - `/src` contains the main source code of the project. 94 | - Folder structure similar to typical Express projects 95 | 96 | ## Development guides 97 | The following section provides you a short guide on how to change or add functionalities. 98 | You can also read the comments in the source code of the files to get more information about the specific implementation. 99 | 100 | ### Create a new service or controller 101 | 1. Create a new folder with the service or controller name (e.g `/src/services/log`) 102 | 2. Create a file for the class (e.g `/src/services/log/log.service.ts`) 103 | - You can use dependency injection to access other services 104 | 3. Create a file that contains an exported instance of that class (e.g `/src/services/index.ts`). 105 | - Here you then can import exported instances of other services or controllers to inject them 106 | 4. Create a unit test file (e.g `/test/services/log.service.spec.ts`) 107 | 108 | ### Add a new http endpoint 109 | 1. Create a new router or add a route to an existing router in `/src/routers` 110 | 2. Create a new validator for that route in `/src/validators` and use it as middleware for that route 111 | - Do not forget to always add the validation check middleware (`/src/validator.middleware.ts`) right after you used a new validation middleware 112 | 3. Add a controller as shown in the section before that will be called from the new route 113 | 4. Add a service as shown in the section before that will called from the new controller 114 | 5. Register the router in `/src/http-app.ts` 115 | 6. Update the swagger documentation in `/docs/api-spec.yml` 116 | 117 | ### Define message processing logic 118 | 1. Go to file `/src/services/event/event.service.ts` 119 | 2. In the function `processMessage : (message: DequeuedMessage) => Promise` of class `EventService` you can define how to process the message 120 | 3. Or you can create also a seperate service for message processing and use it with dependency injection 121 | 4. Check out the file for more specific information 122 | 123 | ### Change Dead Letter Queue behavior 124 | 1. Go to file `serverless.yml` into section `resources.Resources` 125 | 2. `resources.Resources.MessagesQueue.Properties.RedrivePolicy.maxReceiveCount` represents the number of times a message that caused a retriable error should be retried 126 | 3. `resources.Resources.DeadLetterMessagesQueue.Properties.MessageRetentionPeriod` represents the maximum time in seconds a message should be stored in the dead letter queue 127 | 128 | ### Change message processing retry interval time 129 | 1. Go to file `serverless.yml` into section `resources.Resources.MessagesQueue.Properties` 130 | 2. After a minimum time of `VisibilityTimeout - consumer execution time` (in seconds) the consumer will pick up a failed message that should be retried again 131 | 132 | ### Add a new serverless function 133 | 1. Add a new file with the function in `src` (e.g `/src/example-function.ts`) 134 | 2. To call a service you can import an exported instance 135 | 3. Add the serverless function to `functions` section in `serverless.yml` 136 | 137 | ### Change handling of unexpected errors during endpoint access 138 | 1. Modify the function in `/src/middlewares/error.middleware.ts` 139 | 140 | ### Change validation error response format 141 | 1. Modify the function in `/src/middlewares/validator.middleware.ts` 142 | 143 | ### Define environment variables (local) 144 | 1. Add the new environment variable to `.env` file (and also to `.env.example` as reference) 145 | 2. Add the variable to `/test/.env.test` 146 | 3. Use the variable in the `provider.environment` section in `serverless.offline.yml` 147 | 4. Use the variable in `/src/constants/environment.constants.ts` 148 | 149 | ### Define secrets (AWS Lambda) 150 | 1. Follow the steps which are explained in the local section 151 | 2. Add the variable to `secrets.yml` file for the different stages (and also to `secrets.yml.example` as reference) 152 | 3. Use the variable in the `provider.environment` section in `serverless.yml` 153 | 154 | ## Run locally 155 | Right now there is no configuration setup to run this project in the exact same way as on AWS. 156 | You can check out [LocalStack](https://github.com/localstack/localstack) to simulate AWS services on your local computer. 157 | When running the project locally without LocalStack there is no automatic triggering of serverless functions. So you have to call the functions manually to test them. If you run the project with `ENVIRONMENT=local` every interaction with SQS will be realized with a mocked queue service (`/services/queue/local-queue.service.ts`). You can change that behavior in `/services/queue/index.ts` to always use the real service (`/services/queue/sqs-queue.service.ts`). 158 | 159 | ### Install and start project 160 | 1. Clone the repository 161 | 2. Move into cloned directory 162 | 3. Run `npm i` to install all packages 163 | 4. Create a `.env` file based on `.env.example` 164 | 5. Run `npm run start:serverless` to start the serverless offline application 165 | 166 | ### Produce message 167 | 1. Open `http://localhost:3000/local/apispec/` in browser 168 | 2. Use the endpoint `/produce-message` to enqueue a message 169 | 170 | ### Consume messages 171 | 1. Install the [AWS CLI](https://aws.amazon.com/cli/) 172 | 2. Create a payload file that can contains mocked queue messages 173 | ```json 174 | { 175 | "Records": [ 176 | { 177 | "messageId": "message-id1", 178 | "receiptHandle": "message-handle1", 179 | "body": "{\"payload\":\"message-payload1\"}" 180 | } 181 | ] 182 | } 183 | 184 | ``` 185 | 3. Run the following command to send the mocked messages to the consumer: 186 | ``` 187 | aws lambda invoke \ 188 | --output json {absolute output file path} \ 189 | --endpoint-url http://localhost:3100 \ 190 | --function-name serverless-express-typescript-sqs-local-serverless-consumer \s 191 | --payload file://{relative payload file path} \ 192 | --cli-binary-format raw-in-base64-out 193 | ``` 194 | 195 | ### Run tests 196 | 1. Run `npm run test` to run all tests 197 | 2. Run `npm run test:watch` to tests in interactive mode 198 | 199 | ### Format 200 | 1. Run `npm run format` to format all files with Prettier 201 | 2. Run `npm run format:check` if all files are formatted correctly 202 | 203 | ## Deploy on AWS 204 | 1. Read about [serverless deploying](https://www.serverless.com/framework/docs/providers/aws/guide/deploying/) 205 | 2. Create a `secrects.yml` file based on `secrects.yml.example` 206 | 3. Run `npm run serverless -- deploy --stage={stage to deploy for}` to deploy on AWS 207 | 208 | ## Known issues 209 | - TypeScript compiling on changes is quite slow ([Check out this GitHub issue](https://github.com/prisma-labs/serverless-plugin-typescript/issues/220)) 210 | 211 | ## Contribution 212 | Feel free to open an issue if you found any error or to create a pull request if want to add additional content. 213 | -------------------------------------------------------------------------------- /docs/api-spec.yml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.0 2 | info: 3 | title: serverless-express-typescript-sqs 4 | version: 1.0.0 5 | 6 | servers: 7 | - url: http://localhost:3000/local 8 | description: Local serverless API 9 | 10 | paths: 11 | /produce-message: 12 | post: 13 | description: > 14 | Adds a message with a specified payload to a queue. 15 | operationId: produceMessage 16 | requestBody: 17 | required: true 18 | content: 19 | application/json: 20 | schema: 21 | $ref: '#/components/schemas/Message' 22 | responses: 23 | '200': 24 | description: Message has been added to the queue. 25 | content: 26 | application/json: 27 | schema: 28 | type: object 29 | properties: 30 | message: 31 | type: string 32 | payload: 33 | $ref: '#/components/schemas/QueuedMessage' 34 | default: 35 | $ref: '#/components/responses/DefaultErrorResponse' 36 | 37 | components: 38 | schemas: 39 | Message: 40 | type: object 41 | required: 42 | - payload 43 | properties: 44 | payload: 45 | description: > 46 | Defines the payload of the message. Should be between 2 and 1000 characters long. 47 | type: string 48 | QueuedMessage: 49 | properties: 50 | id: 51 | description: > 52 | Specified the id of the message in the sqs queue 53 | type: string 54 | payload: 55 | $ref: '#/components/schemas/Message' 56 | ErrorResponse: 57 | type: object 58 | properties: 59 | message: 60 | type: string 61 | errors: 62 | type: array 63 | items: 64 | type: string 65 | responses: 66 | DefaultErrorResponse: 67 | description: Body is invalid. 68 | content: 69 | application/json: 70 | schema: 71 | $ref: '#/components/schemas/ErrorResponse' 72 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | globals: { 4 | 'ts-jest': { 5 | tsconfig: 'tsconfig.json', 6 | }, 7 | }, 8 | transform: { 9 | '^.+\\.tsx?$': 'ts-jest', 10 | }, 11 | testEnvironment: 'node', 12 | testMatch: [ 13 | '/test/**/*.{test,spec}.ts', 14 | '/test/*.{test,spec}.ts', 15 | ], 16 | moduleFileExtensions: ['ts', 'js', 'json'], 17 | setupFiles: ['/test/env-setup.ts'], 18 | resetModules: false, 19 | collectCoverage: true, 20 | coverageDirectory: './', 21 | collectCoverageFrom: ['**/*.{ts,dts}', '!**/node_modules/**', '!**/test/**'], 22 | coverageReporters: ['lcov', 'text'], 23 | coveragePathIgnorePatterns: ['/dist/'], 24 | }; 25 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-express-typescript-sqs", 3 | "version": "1.0.0", 4 | "description": "A starter kit for a serverless producer consumer queue setup implemented with Node.js", 5 | "author": "Limehome GmbH", 6 | "license": "MIT", 7 | "scripts": { 8 | "start:serverless": "serverless offline start --config serverless.offline.yml --httpPort 3000 --lambdaPort 3100 --stage local --watch", 9 | "test": "jest --verbose --config=jest.config.js --runInBand", 10 | "test:watch": "npm run test -- --watch", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"*.{yaml,yml,json,js}\"", 12 | "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"*.{yaml,yml,json,js}\"", 13 | "serverless": "serverless" 14 | }, 15 | "dependencies": { 16 | "@types/aws-lambda": "^8.10.68", 17 | "@types/aws-serverless-express": "^3.3.3", 18 | "@types/body-parser": "^1.19.0", 19 | "@types/cors": "^2.8.9", 20 | "@types/express": "^4.17.9", 21 | "@types/js-yaml": "^3.12.5", 22 | "@types/supertest": "^2.0.10", 23 | "@types/swagger-ui-express": "^4.1.2", 24 | "aws-lambda": "^1.0.6", 25 | "aws-sdk": "^2.820.0", 26 | "aws-serverless-express": "^3.4.0", 27 | "cors": "^2.8.5", 28 | "express": "^4.17.1", 29 | "express-async-errors": "^3.1.1", 30 | "express-validator": "^6.9.0", 31 | "install": "^0.13.0", 32 | "jest": "^26.6.3", 33 | "npm": "^6.14.10", 34 | "supertest": "^6.0.1", 35 | "swagger-ui-express": "^4.3.0", 36 | "ts-jest": "^26.4.4" 37 | }, 38 | "devDependencies": { 39 | "dotenv": "^8.2.0", 40 | "prettier": "^2.2.1", 41 | "serverless": "^2.16.1", 42 | "serverless-dotenv-plugin": "^3.1.0", 43 | "serverless-offline": "^8.5.0", 44 | "serverless-plugin-typescript": "^1.1.9", 45 | "typescript": "^4.1.3" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /secrets.yml.example: -------------------------------------------------------------------------------- 1 | develop: 2 | ENVIRONMENT: 'develop' 3 | 4 | staging: 5 | ENVIRONMENT: 'staging' 6 | 7 | production: 8 | ENVIRONMENT: 'prod' 9 | -------------------------------------------------------------------------------- /serverless.offline.yml: -------------------------------------------------------------------------------- 1 | service: ${file(./serverless.yml):service} 2 | 3 | useDotenv: true 4 | 5 | plugins: 6 | - serverless-plugin-typescript 7 | - serverless-offline 8 | 9 | package: ${file(./serverless.yml):package} 10 | 11 | provider: 12 | name: ${file(./serverless.yml):provider.name} 13 | runtime: ${file(./serverless.yml):provider.runtime} 14 | region: ${file(./serverless.yml):provider.region} 15 | stage: local 16 | environment: 17 | ENVIRONMENT: ${env:ENVIRONMENT} 18 | 19 | functions: ${file(./serverless.yml):functions} 20 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: serverless-express-typescript-sqs 2 | frameworkVersion: '2' 3 | 4 | plugins: 5 | - serverless-plugin-typescript 6 | 7 | package: 8 | individually: true 9 | exclude: 10 | - coverage/** 11 | - test/** 12 | - secrets.yml 13 | include: 14 | - docs/api-spec.yml 15 | 16 | custom: 17 | secrets: ${file(./secrets.yml):${self:provider.stage}} 18 | 19 | provider: 20 | name: aws 21 | runtime: nodejs12.x 22 | region: eu-central-1 23 | stage: ${opt:stage, 'develop'} 24 | timeout: 20 25 | iamRoleStatements: 26 | - Effect: Allow 27 | Action: 28 | - sqs:SendMessage 29 | - sqs:DeleteMessageBatch 30 | Resource: 31 | - Fn::GetAtt: [MessagesQueue, Arn] 32 | environment: 33 | ENVIRONMENT: ${self:custom.secrets.ENVIRONMENT} 34 | MESSAGE_QUEUE_URL: { Ref: MessagesQueue } 35 | 36 | 37 | functions: 38 | serverless-producer: 39 | handler: src/serverless-producer.handler 40 | memorySize: 512 41 | timeout: 5 42 | events: 43 | - http: 44 | path: /{proxy+} 45 | method: ANY 46 | serverless-consumer: 47 | handler: src/serverless-consumer.handler 48 | events: 49 | - sqs: 50 | arn: 51 | Fn::GetAtt: 52 | - MessagesQueue 53 | - Arn 54 | resources: 55 | Resources: 56 | MessagesQueue: 57 | Type: AWS::SQS::Queue 58 | Properties: 59 | RedrivePolicy: 60 | deadLetterTargetArn: !GetAtt DeadLetterMessagesQueue.Arn 61 | maxReceiveCount: 3 62 | # Two minutes 63 | VisibilityTimeout: 120 64 | DeadLetterMessagesQueue: 65 | Type: AWS::SQS::Queue 66 | Properties: 67 | # Seven days 68 | MessageRetentionPeriod: 604800 69 | -------------------------------------------------------------------------------- /src/constants/environment.constants.ts: -------------------------------------------------------------------------------- 1 | import { Environment } from './../enums/environment.enum'; 2 | 3 | const env = process.env; 4 | 5 | export const ENVIRONMENT: Environment = env.ENVIRONMENT as Environment; 6 | 7 | export const MESSAGE_QUEUE_URL = env.MESSAGE_QUEUE_URL as string; 8 | 9 | export const AWS_REGION = env.AWS_REGION as string; 10 | -------------------------------------------------------------------------------- /src/controllers/message/index.ts: -------------------------------------------------------------------------------- 1 | import { queueService } from './../../services/queue'; 2 | import { MessageController } from './message.controller'; 3 | 4 | export const messageController = new MessageController(queueService); 5 | -------------------------------------------------------------------------------- /src/controllers/message/message.controller.ts: -------------------------------------------------------------------------------- 1 | import { QueueService } from './../../interfaces/queue-service.interface'; 2 | import { Response } from './../../interfaces/response.interface'; 3 | import { Message } from '../../interfaces/message.interface'; 4 | import * as Express from 'express'; 5 | import { QueuedMessage } from 'src/interfaces/queued-message.interface'; 6 | 7 | /** 8 | * Controller to produce new messages for the sqs queue 9 | */ 10 | export class MessageController { 11 | constructor(private readonly queueService: QueueService) {} 12 | 13 | public async produceMessage(req: Express.Request, res: Express.Response) { 14 | const message: Message = req.body; 15 | const queuedMessage = await this.queueService.enqueueMessage(message); 16 | res.status(200).send({ 17 | message: 'Produced message', 18 | payload: queuedMessage, 19 | } as Response); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/enums/environment.enum.ts: -------------------------------------------------------------------------------- 1 | export enum Environment { 2 | LOCAL = 'local', 3 | TEST = 'test', 4 | DEVELOP = 'develop', 5 | STAGING = 'staging', 6 | PROD = 'prod', 7 | } 8 | -------------------------------------------------------------------------------- /src/errors/base.error.ts: -------------------------------------------------------------------------------- 1 | export class BaseError extends Error { 2 | constructor(error: unknown = {}) { 3 | super(JSON.stringify(error)); 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/errors/non-retriable.error.ts: -------------------------------------------------------------------------------- 1 | import { BaseError } from './base.error'; 2 | 3 | /** 4 | * Throw this error if message should not stay in the queue when error occurs 5 | */ 6 | export class NonRetriableError extends BaseError {} 7 | -------------------------------------------------------------------------------- /src/errors/partial-failure.error.ts: -------------------------------------------------------------------------------- 1 | import { BaseError } from './base.error'; 2 | 3 | export class PartialFailureError extends BaseError {} 4 | -------------------------------------------------------------------------------- /src/http-app.ts: -------------------------------------------------------------------------------- 1 | require('express-async-errors'); 2 | import { messagesRouter } from './routers/message.router'; 3 | import { useApiSpec } from './middlewares/api-spec.middleware'; 4 | import * as Express from 'express'; 5 | import * as cors from 'cors'; 6 | import { json, urlencoded } from 'body-parser'; 7 | import { handleErrors } from './middlewares/error.middleware'; 8 | 9 | /** 10 | * Creates an Express.js http app which contains endpoints 11 | * to produce messages for the sqs queue 12 | */ 13 | export const createHttpApp = () => { 14 | const app = Express(); 15 | app.use(cors()); 16 | app.use(json()); 17 | app.use(urlencoded({ extended: true })); 18 | 19 | // setup swagger 20 | useApiSpec(app); 21 | 22 | // setup routers 23 | app.use('/', messagesRouter); 24 | 25 | // setup handler for unexpected errors 26 | app.use(handleErrors); 27 | 28 | return app; 29 | }; 30 | -------------------------------------------------------------------------------- /src/interfaces/dequeued-message.interface.ts: -------------------------------------------------------------------------------- 1 | import { QueuedMessage } from './queued-message.interface'; 2 | export interface DequeuedMessage extends QueuedMessage { 3 | receiptHandle: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/interfaces/message.interface.ts: -------------------------------------------------------------------------------- 1 | export interface Message { 2 | payload: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/interfaces/queue-service.interface.ts: -------------------------------------------------------------------------------- 1 | import { QueuedMessage } from 'src/interfaces/queued-message.interface'; 2 | import { Message } from './message.interface'; 3 | import { DequeuedMessage } from './dequeued-message.interface'; 4 | 5 | export interface QueueService { 6 | enqueueMessage(message: Message): Promise; 7 | deleteMessages(messageHeaders: DequeuedMessage[]): Promise; 8 | } 9 | -------------------------------------------------------------------------------- /src/interfaces/queued-message.interface.ts: -------------------------------------------------------------------------------- 1 | import { Message } from './message.interface'; 2 | export interface QueuedMessage extends Message { 3 | id: string; 4 | } 5 | -------------------------------------------------------------------------------- /src/interfaces/response.interface.ts: -------------------------------------------------------------------------------- 1 | export interface Response { 2 | message: string; 3 | payload: T; 4 | errors?: string[]; 5 | } 6 | -------------------------------------------------------------------------------- /src/middlewares/api-spec.middleware.ts: -------------------------------------------------------------------------------- 1 | import * as SwaggerUI from 'swagger-ui-express'; 2 | import { safeLoad } from 'js-yaml'; 3 | import { Express } from 'express'; 4 | import { readFileSync } from 'fs'; 5 | import { resolve } from 'path'; 6 | import { ServerlessApplicationRepository } from 'aws-sdk'; 7 | 8 | let swaggerSpec: object; 9 | 10 | /** 11 | * Load and parse yaml file that contains the swagger specification 12 | */ 13 | const loadSpec = () => 14 | safeLoad( 15 | readFileSync(resolve(__dirname, '../../docs/api-spec.yml'), 'utf-8'), 16 | ) as object; 17 | 18 | /** 19 | * Serves a swagger doc under /apispec 20 | */ 21 | export const useApiSpec = (app: Express) => { 22 | if (!swaggerSpec) { 23 | swaggerSpec = loadSpec(); 24 | } 25 | 26 | const swaggerUIMiddleware = SwaggerUI.setup(swaggerSpec); 27 | 28 | // Make a mock request to the swagger ui middleware to initialize it. 29 | // Otherwise some js files will not be loaded with serverless 30 | // Workaround issue: https://github.com/scottie1984/swagger-ui-express/issues/178 31 | swaggerUIMiddleware({} as any, { send: () => {} } as any, () => {}); 32 | 33 | app.use( 34 | `/apispec`, 35 | SwaggerUI.serveWithOptions({ 36 | redirect: false, // Disabled as it does not work with API gateway 37 | }), 38 | ); 39 | app.get(`/apispec/`, swaggerUIMiddleware); 40 | }; 41 | -------------------------------------------------------------------------------- /src/middlewares/error.middleware.ts: -------------------------------------------------------------------------------- 1 | import { Response } from './../interfaces/response.interface'; 2 | import * as Express from 'express'; 3 | 4 | /** 5 | * Handles unexpected errors that have not been handled by any controller 6 | */ 7 | export const handleErrors = ( 8 | error: unknown, 9 | req: Express.Request, 10 | res: Express.Response, 11 | next: Express.NextFunction, 12 | ) => { 13 | // Transform or check any unexpected errors 14 | return res.status(500).send({ 15 | message: 'An unexpected error occured.', 16 | } as Response); 17 | }; 18 | -------------------------------------------------------------------------------- /src/middlewares/validator.middleware.ts: -------------------------------------------------------------------------------- 1 | import * as Express from 'express'; 2 | import { NextFunction } from 'express'; 3 | import { validationResult } from 'express-validator'; 4 | import { Response } from '../interfaces/response.interface'; 5 | 6 | /** 7 | * Checks if there are validation errors. 8 | * Send a 400 response if errors do exist 9 | */ 10 | export const checkValidation = ( 11 | req: Express.Request, 12 | res: Express.Response, 13 | next: NextFunction, 14 | ) => { 15 | // Check if there are any validation errors 16 | const validationErrors = validationResult(req); 17 | 18 | // Go to next middleware if there are no errors 19 | if (validationErrors.isEmpty()) { 20 | next(); 21 | return; 22 | } 23 | 24 | // Map errors to your format 25 | let errors = validationErrors 26 | .array() 27 | .map((error) => `${error.location} -> ${error.param}: ${error.msg}`); 28 | 29 | res.status(400).send({ 30 | message: 'There are validation errors', 31 | errors, 32 | } as Response); 33 | }; 34 | -------------------------------------------------------------------------------- /src/routers/message.router.ts: -------------------------------------------------------------------------------- 1 | import { checkValidation } from './../middlewares/validator.middleware'; 2 | import { validateMessage } from './../validators/message.validator'; 3 | import * as Express from 'express'; 4 | import { messageController } from '../controllers/message'; 5 | 6 | export const messagesRouter = Express.Router(); 7 | 8 | messagesRouter.post( 9 | `/produce-message`, 10 | 11 | // Check the incoming message for errors 12 | validateMessage, 13 | 14 | // Handle the errors 15 | checkValidation, 16 | 17 | (req: Express.Request, res: Express.Response) => 18 | messageController.produceMessage(req, res), 19 | ); 20 | -------------------------------------------------------------------------------- /src/serverless-consumer.ts: -------------------------------------------------------------------------------- 1 | import { eventService } from './services/event/index'; 2 | import { SQSEvent } from 'aws-lambda/trigger/sqs'; 3 | 4 | /** 5 | * Entry point for consumer that will be triggered from sqs events 6 | */ 7 | export const handler = (sqsEvent: SQSEvent) => eventService.handle(sqsEvent); 8 | -------------------------------------------------------------------------------- /src/serverless-producer.ts: -------------------------------------------------------------------------------- 1 | import { createServer, proxy } from 'aws-serverless-express'; 2 | import { Context, APIGatewayProxyEvent } from 'aws-lambda'; 3 | import { createHttpApp } from './http-app'; 4 | 5 | const httpApp = createHttpApp(); 6 | const httpServer = createServer(httpApp); 7 | 8 | /** 9 | * Entry point for the http producer for serverless 10 | */ 11 | export const handler = (event: APIGatewayProxyEvent, context: Context) => 12 | proxy(httpServer, event, context, 'PROMISE').promise; 13 | -------------------------------------------------------------------------------- /src/services/event/event.service.ts: -------------------------------------------------------------------------------- 1 | import { Message } from '../../interfaces/message.interface'; 2 | import { QueueService } from './../../interfaces/queue-service.interface'; 3 | import { PartialFailureError } from './../../errors/partial-failure.error'; 4 | import { NonRetriableError } from './../../errors/non-retriable.error'; 5 | import { DequeuedMessage } from './../../interfaces/dequeued-message.interface'; 6 | import { SQSEvent } from 'aws-lambda/trigger/sqs'; 7 | import { LogService } from '../log/log.service'; 8 | 9 | export class EventService { 10 | constructor( 11 | private readonly queueService: QueueService, 12 | private readonly logService: LogService, 13 | ) {} 14 | 15 | /** 16 | * Handles an sqs event by processing every message of it 17 | */ 18 | async handle(event: SQSEvent) { 19 | // Get parsed messages from the event 20 | const dequeuedMessages = this.mapEventToDequeuedMessages(event); 21 | const messagesToDelete: DequeuedMessage[] = []; 22 | 23 | // Process all messages 24 | // To delete every message from the event from the queue that has been successful processed, 25 | // the handle function must not throw an error. 26 | // To delete successful messages and keep unsuccessful messages in the queue, 27 | // the handle function has to throw an error. 28 | // Successfull messages have to be deleted manually in this case. 29 | const promises = dequeuedMessages.map(async (message) => { 30 | try { 31 | await this.processMessage(message); 32 | messagesToDelete.push(message); 33 | } catch (error) { 34 | if (error instanceof NonRetriableError) { 35 | messagesToDelete.push(message); 36 | this.logService.error( 37 | EventService.name, 38 | 'Processing message', 39 | message, 40 | 'caused a non retriable error. Error:', 41 | error, 42 | ); 43 | } else { 44 | this.logService.error( 45 | EventService.name, 46 | 'Processing message', 47 | message, 48 | 'caused a retriable error. Error:', 49 | error, 50 | ); 51 | } 52 | } 53 | }); 54 | // await until all messages have been processed 55 | await Promise.all(promises); 56 | 57 | // Delete successful messages manually if other processings failed 58 | const numRetriableMessages = 59 | dequeuedMessages.length - messagesToDelete.length; 60 | if (numRetriableMessages > 0) { 61 | await this.queueService.deleteMessages(messagesToDelete); 62 | 63 | const errorMessage = `Failing due to ${numRetriableMessages} unsuccessful and retriable errors.`; 64 | 65 | throw new PartialFailureError(errorMessage); 66 | } 67 | } 68 | 69 | async processMessage(message: DequeuedMessage) { 70 | // Here you can process the message. 71 | // For example you could send this message to another http api. 72 | // If processing of the message fails 73 | // and you want that this process should be not retried for this specific case, throw a NonRetriableError 74 | // otherwise throw any other error to leave the message in the sqs queue. 75 | this.logService.info( 76 | EventService.name, 77 | 'Processed message', 78 | message, 79 | 'successfully.', 80 | ); 81 | } 82 | 83 | private mapEventToDequeuedMessages(event: SQSEvent): DequeuedMessage[] { 84 | return event.Records.map((record) => { 85 | const message: Message = JSON.parse(record.body); 86 | return { 87 | id: record.messageId, 88 | receiptHandle: record.receiptHandle, 89 | ...message, 90 | }; 91 | }); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/services/event/index.ts: -------------------------------------------------------------------------------- 1 | import { EventService } from './event.service'; 2 | import { queueService } from '../queue'; 3 | import { logService } from '../log'; 4 | 5 | export const eventService = new EventService(queueService, logService); 6 | -------------------------------------------------------------------------------- /src/services/log/index.ts: -------------------------------------------------------------------------------- 1 | import { LogService } from './log.service'; 2 | 3 | export const logService = new LogService(); 4 | -------------------------------------------------------------------------------- /src/services/log/log.service.ts: -------------------------------------------------------------------------------- 1 | import { inspect } from 'util'; 2 | 3 | /** 4 | * Service to log messages. Usesa the standart console object 5 | */ 6 | export class LogService { 7 | /** 8 | * Creates a info log 9 | * @param tag associate the log to a specific service or file 10 | * @param messages does log objects up to property depth 6 11 | */ 12 | info(tag: string, ...messages: any[]) { 13 | console.info(this.buildLogMessage(this.tintGreen, tag, ...messages)); 14 | } 15 | 16 | /** 17 | * Creates a error log 18 | * @param tag associate the log to a specific service or file 19 | * @param messages does log objects up to property depth 6 20 | */ 21 | error(tag: string, ...messages: any[]) { 22 | console.error(this.buildLogMessage(this.tintRed, tag, ...messages)); 23 | } 24 | 25 | private buildLogMessage( 26 | tint: (v: string) => string, 27 | tag: string, 28 | ...messages: any[] 29 | ) { 30 | return `${tint(tag)}: ${messages 31 | .map((m) => (m instanceof Object ? inspect(m, true, 6, true) : m)) 32 | .join(' ')}`; 33 | } 34 | 35 | private tintGreen(value: string) { 36 | return `\x1b[32m[${value}]\x1b[0m`; 37 | } 38 | 39 | private tintRed(value: string) { 40 | return `\x1b[31m[${value}]\x1b[0m`; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/services/queue/index.ts: -------------------------------------------------------------------------------- 1 | import { QueueService } from './../../interfaces/queue-service.interface'; 2 | import { 3 | AWS_REGION, 4 | ENVIRONMENT, 5 | } from './../../constants/environment.constants'; 6 | import { SqsQueueService } from './sqs-queue.service'; 7 | import * as SQS from 'aws-sdk/clients/sqs'; 8 | import { Environment } from '../../enums/environment.enum'; 9 | import { LocalQueueService } from './local-queue.service'; 10 | import { logService } from '../log'; 11 | 12 | const sqsClient = new SQS({ 13 | region: AWS_REGION, 14 | }); 15 | 16 | export const queueService: QueueService = 17 | ENVIRONMENT === Environment.LOCAL 18 | ? new LocalQueueService(logService) 19 | : new SqsQueueService(sqsClient); 20 | -------------------------------------------------------------------------------- /src/services/queue/local-queue.service.ts: -------------------------------------------------------------------------------- 1 | import { QueueService } from './../../interfaces/queue-service.interface'; 2 | import { QueuedMessage } from 'src/interfaces/queued-message.interface'; 3 | import { Message } from 'src/interfaces/message.interface'; 4 | import { DequeuedMessage } from 'src/interfaces/dequeued-message.interface'; 5 | import { LogService } from '../log/log.service'; 6 | 7 | /** 8 | * This class provides a mock implementation of QueueService. 9 | * Instead of performing actions with a real sqs queue, the action just gets logged. 10 | */ 11 | export class LocalQueueService implements QueueService { 12 | constructor(private readonly logService: LogService) {} 13 | 14 | /** 15 | * Logs a that message got enqueded 16 | */ 17 | async enqueueMessage(message: Message): Promise { 18 | this.logService.info( 19 | LocalQueueService.name, 20 | 'Enqueued message to fake sqs queue. Message:', 21 | message, 22 | ); 23 | 24 | return { 25 | id: 'local-id', 26 | ...message, 27 | }; 28 | } 29 | 30 | /** 31 | * Logs a that messages got deleted 32 | */ 33 | async deleteMessages(messageHeaders: DequeuedMessage[]): Promise { 34 | this.logService.info( 35 | LocalQueueService.name, 36 | 'Deleted messages from fake sqs queue. Deleted messages:', 37 | messageHeaders, 38 | ); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/services/queue/sqs-queue.service.ts: -------------------------------------------------------------------------------- 1 | import { DequeuedMessage } from './../../interfaces/dequeued-message.interface'; 2 | import { QueuedMessage } from './../../interfaces/queued-message.interface'; 3 | import { MESSAGE_QUEUE_URL } from './../../constants/environment.constants'; 4 | import { Message } from './../../interfaces/message.interface'; 5 | import { SQS } from 'aws-sdk'; 6 | import { QueueService } from '../../interfaces/queue-service.interface'; 7 | 8 | /** 9 | * Service to handle different sqs queue actions 10 | */ 11 | export class SqsQueueService implements QueueService { 12 | constructor(private readonly sqsClient: SQS) {} 13 | 14 | /** 15 | * Enqueue a message to the sqs queue 16 | */ 17 | async enqueueMessage(message: Message): Promise { 18 | const result = await this.sqsClient 19 | .sendMessage({ 20 | QueueUrl: MESSAGE_QUEUE_URL, 21 | MessageBody: JSON.stringify(message), 22 | }) 23 | .promise(); 24 | 25 | return { 26 | id: result.MessageId || '', 27 | ...message, 28 | }; 29 | } 30 | 31 | /** 32 | * Deletes up to ten messages from a sqs queue 33 | */ 34 | async deleteMessages( 35 | deleteMessageRequests: DequeuedMessage[], 36 | ): Promise { 37 | if (deleteMessageRequests.length <= 0) { 38 | return; 39 | } 40 | 41 | const result = await this.sqsClient 42 | .deleteMessageBatch({ 43 | QueueUrl: MESSAGE_QUEUE_URL, 44 | Entries: deleteMessageRequests.map((m) => ({ 45 | Id: m.id, 46 | ReceiptHandle: m.receiptHandle, 47 | })), 48 | }) 49 | .promise(); 50 | 51 | if (result.Failed.length > 0) { 52 | throw new Error('Unable to delete messages from queue.'); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/validators/message.validator.ts: -------------------------------------------------------------------------------- 1 | import { body } from 'express-validator'; 2 | 3 | // Sets the validation constrains for a message 4 | // Add more items to validate multiple properties 5 | export const validateMessage = [ 6 | body('payload').isString().isLength({ min: 2, max: 1000 }), 7 | ]; 8 | -------------------------------------------------------------------------------- /test/.env.test: -------------------------------------------------------------------------------- 1 | ENVIRONMENT=test 2 | MESSAGE_QUEUE_URL=http://localhost:3100 -------------------------------------------------------------------------------- /test/controllers/message.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { createMessageMock } from './../mock-factories/message.mock-factory'; 2 | import { MessageController } from '../../src/controllers/message/message.controller'; 3 | import { QueuedMessage } from '../../src/interfaces/queued-message.interface'; 4 | import { 5 | createRequestMock, 6 | createResponseMock, 7 | } from '../mock-factories/express.mock-factory'; 8 | import { MaybeMockedDeep, mocked } from 'ts-jest/dist/utils/testing'; 9 | import { SqsQueueService } from '../../src/services/queue/sqs-queue.service'; 10 | 11 | jest.mock('../../src/services/queue/sqs-queue.service'); 12 | 13 | describe('message controller', () => { 14 | let mockedQueueService: MaybeMockedDeep; 15 | let messageController: MessageController; 16 | 17 | const TEST_ID = 'test-id'; 18 | 19 | beforeAll(() => { 20 | mockedQueueService = mocked(SqsQueueService, true); 21 | 22 | mockedQueueService.prototype.enqueueMessage.mockImplementation( 23 | async (message): Promise => ({ 24 | ...message, 25 | id: TEST_ID, 26 | }), 27 | ); 28 | 29 | messageController = new MessageController(new SqsQueueService(null as any)); 30 | }); 31 | 32 | describe('produceMessage', () => { 33 | it('should call queue service with correct arguments', async () => { 34 | const message = createMessageMock(); 35 | const req = createRequestMock({ body: message }); 36 | const res = createResponseMock(); 37 | 38 | await messageController.produceMessage(req, res); 39 | 40 | expect(mockedQueueService.prototype.enqueueMessage).toHaveBeenCalledWith( 41 | message, 42 | ); 43 | }); 44 | 45 | it('should send queued message with status 200', async () => { 46 | const message = createMessageMock(); 47 | const req = createRequestMock({ body: message }); 48 | const res = createResponseMock(); 49 | 50 | await messageController.produceMessage(req, res); 51 | 52 | expect(res.status).toHaveBeenCalledWith(200); 53 | expect(res.send.mock.calls[0][0].payload).toMatchObject({ 54 | ...message, 55 | id: TEST_ID, 56 | } as QueuedMessage); 57 | }); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /test/env-setup.ts: -------------------------------------------------------------------------------- 1 | import { config } from 'dotenv'; 2 | config({ path: './tests/.env.test' }); 3 | -------------------------------------------------------------------------------- /test/middlewares/api-spec.middleware.spec.ts: -------------------------------------------------------------------------------- 1 | import * as Express from 'express'; 2 | import * as request from 'supertest'; 3 | import { createHttpApp } from '../../src/http-app'; 4 | import * as ApiSpecMiddleware from '../../src/middlewares/api-spec.middleware'; 5 | 6 | describe('api spec middleware', () => { 7 | let app: Express.Express; 8 | beforeEach(() => { 9 | app = createHttpApp(); 10 | }); 11 | 12 | it('should send status 200 when accessing the apispec', async () => { 13 | const res = await request(app).get('/apispec/'); 14 | expect(res.status).toBe(200); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /test/middlewares/error.middleware.spec.ts: -------------------------------------------------------------------------------- 1 | import { handleErrors } from './../../src/middlewares/error.middleware'; 2 | import { 3 | createRequestMock, 4 | createResponseMock, 5 | createNextMock, 6 | } from '../mock-factories/express.mock-factory'; 7 | 8 | describe('error middleware', () => { 9 | it('should send status 500', async () => { 10 | const req = createRequestMock(); 11 | const res = createResponseMock(); 12 | const next = createNextMock(); 13 | 14 | const error = 'test-error'; 15 | 16 | handleErrors(error, req, res, next); 17 | 18 | expect(next).toHaveBeenCalledTimes(0); 19 | expect(res.status).toHaveBeenCalledWith(500); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /test/middlewares/validator.middleware.spec.ts: -------------------------------------------------------------------------------- 1 | import { 2 | createRequestMock, 3 | createResponseMock, 4 | createNextMock, 5 | } from '../mock-factories/express.mock-factory'; 6 | import { body } from 'express-validator'; 7 | import { checkValidation } from '../../src/middlewares/validator.middleware'; 8 | 9 | describe('validator middleware', () => { 10 | it('should send status 400 if there are validation errors and map errors correctly', async () => { 11 | const req = createRequestMock(); 12 | const res = createResponseMock(); 13 | const next = createNextMock(); 14 | 15 | req.body = { 16 | payload: 1, 17 | }; 18 | 19 | await body('payload').isString().run(req); 20 | 21 | checkValidation(req, res, next); 22 | 23 | expect(next).toHaveBeenCalledTimes(0); 24 | expect(res.status).toHaveBeenCalledWith(400); 25 | expect(res.send.mock.calls[0][0].errors).toMatchObject([ 26 | 'body -> payload: Invalid value', 27 | ]); 28 | }); 29 | 30 | it('should call next if there are no validation erros', async () => { 31 | const req = createRequestMock(); 32 | const res = createResponseMock(); 33 | const next = createNextMock(); 34 | 35 | req.body = { 36 | payload: 'valid', 37 | }; 38 | 39 | await body('payload').isString().run(req); 40 | 41 | checkValidation(req, res, next); 42 | 43 | expect(next).toHaveBeenCalledTimes(1); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /test/mock-factories/aws-sdk.mock.factory.ts: -------------------------------------------------------------------------------- 1 | import { AWSError, Request } from 'aws-sdk'; 2 | 3 | export const createAwsSdkRequestMock = ( 4 | response: T, 5 | ): Request => { 6 | return { 7 | promise: async () => response, 8 | } as any; 9 | }; 10 | -------------------------------------------------------------------------------- /test/mock-factories/express.mock-factory.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from 'express'; 2 | 3 | export const createRequestMock = ( 4 | request: Partial = {}, 5 | ): jest.Mocked => { 6 | return { 7 | url: 'http://localhost', 8 | method: 'get', 9 | header: jest.fn(), 10 | query: {}, 11 | params: {}, 12 | ...request, 13 | } as any; 14 | }; 15 | 16 | export const createResponseMock = ( 17 | response: Partial = {}, 18 | ): jest.Mocked => { 19 | return { 20 | status: jest.fn().mockReturnThis(), 21 | json: jest.fn().mockReturnThis(), 22 | redirect: jest.fn().mockReturnThis(), 23 | setHeader: jest.fn().mockReturnThis(), 24 | send: jest.fn().mockReturnThis(), 25 | ...response, 26 | } as any; 27 | }; 28 | 29 | export const createNextMock = () => jest.fn(() => {}); 30 | -------------------------------------------------------------------------------- /test/mock-factories/message.mock-factory.ts: -------------------------------------------------------------------------------- 1 | import { Message } from '../../src/interfaces/message.interface'; 2 | 3 | const useModifier = (value: T) => ( 4 | modifyFunc: (v: T) => void = () => {}, 5 | ) => { 6 | const copiedValue: T = JSON.parse(JSON.stringify(value)); 7 | modifyFunc(copiedValue); 8 | return copiedValue; 9 | }; 10 | 11 | export const createMessageMock = useModifier({ 12 | payload: 'this is a valid payload', 13 | }); 14 | -------------------------------------------------------------------------------- /test/services/event-service.spec.ts: -------------------------------------------------------------------------------- 1 | import { DequeuedMessage } from './../../.build/src/interfaces/dequeued-message.interface.d'; 2 | import { createMessageMock } from './../mock-factories/message.mock-factory'; 3 | import { SqsQueueService } from '../../src/services/queue/sqs-queue.service'; 4 | import { MaybeMockedDeep } from 'ts-jest/dist/utils/testing'; 5 | import { mocked } from 'ts-jest/utils'; 6 | import { NonRetriableError } from '../../src/errors/non-retriable.error'; 7 | import { PartialFailureError } from '../../src/errors/partial-failure.error'; 8 | import { SQSRecord } from 'aws-lambda'; 9 | import { EventService } from '../../src/services/event/event.service'; 10 | import { LogService } from '../../src/services/log/log.service'; 11 | 12 | jest.mock('../../src/services/queue/sqs-queue.service'); 13 | jest.mock('../../src/services/log/log.service'); 14 | 15 | describe('sqs message handler', () => { 16 | const fakeRecord = { 17 | messageId: 'someId', 18 | receiptHandle: 'someHandle', 19 | body: JSON.stringify(createMessageMock()), 20 | } as SQSRecord; 21 | 22 | let mockedSqsQueueService: MaybeMockedDeep; 23 | 24 | let eventService: EventService; 25 | let processMessageSpy: jest.SpyInstance; 26 | 27 | beforeEach(() => { 28 | mockedSqsQueueService = mocked(SqsQueueService, true); 29 | eventService = new EventService( 30 | new SqsQueueService(null as any), 31 | new LogService(), 32 | ); 33 | 34 | processMessageSpy = jest.spyOn(eventService, 'processMessage'); 35 | 36 | mockedSqsQueueService.prototype.deleteMessages.mockReset(); 37 | }); 38 | 39 | it('should finish successfully if all messages have been delivered', async () => { 40 | await eventService.handle({ Records: [fakeRecord] }); 41 | expect(processMessageSpy).toHaveBeenCalledTimes(1); 42 | expect( 43 | mockedSqsQueueService.prototype.deleteMessages, 44 | ).toHaveBeenCalledTimes(0); 45 | }); 46 | it('should skip non retriable errors', async () => { 47 | processMessageSpy.mockImplementationOnce(async () => {}); 48 | processMessageSpy.mockImplementationOnce(async () => { 49 | throw new NonRetriableError('NoRetryNeeded'); 50 | }); 51 | await eventService.handle({ Records: [fakeRecord, fakeRecord] }); 52 | expect(processMessageSpy).toHaveBeenCalledTimes(2); 53 | expect( 54 | mockedSqsQueueService.prototype.deleteMessages, 55 | ).toHaveBeenCalledTimes(0); 56 | }); 57 | it('should delete successful and nonretriable messages and throw error on partial failure', async () => { 58 | const nonRetryMessageId = 'nonRetriableMessage'; 59 | const retryMessageId = 'retryMessage'; 60 | 61 | processMessageSpy.mockImplementationOnce(async () => {}); 62 | processMessageSpy.mockImplementationOnce(async () => { 63 | throw new NonRetriableError('DoNotRetryMe'); 64 | }); 65 | processMessageSpy.mockImplementationOnce(async () => { 66 | throw new Error('RetryMe'); 67 | }); 68 | expect.assertions(3); 69 | try { 70 | await eventService.handle({ 71 | Records: [ 72 | fakeRecord, 73 | { ...fakeRecord, messageId: nonRetryMessageId }, 74 | { ...fakeRecord, messageId: retryMessageId }, 75 | ], 76 | }); 77 | } catch (e) { 78 | expect(e).toBeInstanceOf(PartialFailureError); 79 | } 80 | 81 | expect(processMessageSpy).toHaveBeenCalledTimes(3); 82 | 83 | expect(mockedSqsQueueService.prototype.deleteMessages).toHaveBeenCalledWith( 84 | [ 85 | { 86 | id: fakeRecord.messageId, 87 | receiptHandle: fakeRecord.receiptHandle, 88 | ...createMessageMock(), 89 | }, 90 | { 91 | id: nonRetryMessageId, 92 | receiptHandle: fakeRecord.receiptHandle, 93 | ...createMessageMock(), 94 | }, 95 | ] as DequeuedMessage[], 96 | ); 97 | }); 98 | }); 99 | -------------------------------------------------------------------------------- /test/services/log.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { LogService } from '../../src/services/log/log.service'; 2 | 3 | describe('log service', () => { 4 | let infoSpy: jest.SpyInstance; 5 | let errorSpy: jest.SpyInstance; 6 | beforeAll(() => { 7 | infoSpy = jest.spyOn(console, 'info'); 8 | errorSpy = jest.spyOn(console, 'error'); 9 | 10 | infoSpy.mockImplementation(() => jest.fn); 11 | errorSpy.mockImplementation(() => jest.fn); 12 | }); 13 | 14 | it('should log info', async () => { 15 | const logService = new LogService(); 16 | 17 | logService.info('prefix', 'test'); 18 | 19 | expect(infoSpy).toHaveBeenCalledTimes(1); 20 | }); 21 | it('should log error', async () => { 22 | const logService = new LogService(); 23 | 24 | logService.error('prefix', 'test'); 25 | 26 | expect(errorSpy).toHaveBeenCalledTimes(1); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /test/services/sqs-queue.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { createAwsSdkRequestMock } from './../mock-factories/aws-sdk.mock.factory'; 2 | import { DequeuedMessage } from './../../.build/src/interfaces/dequeued-message.interface.d'; 3 | import { QueuedMessage } from './../../src/interfaces/queued-message.interface'; 4 | import { SQS } from 'aws-sdk'; 5 | import { SqsQueueService } from '../../src/services/queue/sqs-queue.service'; 6 | import { createMessageMock } from '../mock-factories/message.mock-factory'; 7 | import { MESSAGE_QUEUE_URL } from '../../src/constants/environment.constants'; 8 | 9 | describe('sqs queue service', () => { 10 | let sqsQueueService: SqsQueueService; 11 | let mockedSqsClient: jest.Mocked; 12 | beforeAll(() => { 13 | mockedSqsClient = { 14 | sendMessage: jest.fn(() => {}), 15 | deleteMessage: jest.fn(() => {}), 16 | deleteMessageBatch: jest.fn(() => {}), 17 | } as any; 18 | sqsQueueService = new SqsQueueService(mockedSqsClient); 19 | }); 20 | 21 | describe('enqueueMessage', () => { 22 | it('should enqueue a message', async () => { 23 | const messageId = 'test'; 24 | const message = createMessageMock(); 25 | 26 | mockedSqsClient.sendMessage.mockReturnValue( 27 | createAwsSdkRequestMock({ 28 | MessageId: messageId, 29 | }), 30 | ); 31 | const result = await sqsQueueService.enqueueMessage(message); 32 | expect(result).toMatchObject({ 33 | ...message, 34 | id: messageId, 35 | } as QueuedMessage); 36 | expect(mockedSqsClient.sendMessage).toHaveBeenCalledWith({ 37 | QueueUrl: MESSAGE_QUEUE_URL, 38 | MessageBody: JSON.stringify(message), 39 | } as SQS.SendMessageRequest); 40 | }); 41 | }); 42 | 43 | describe('deleteMessages', () => { 44 | it('should delete messages', async () => { 45 | const messageHeaders: DequeuedMessage[] = [ 46 | { 47 | id: 'testId', 48 | receiptHandle: 'testHandle', 49 | ...createMessageMock(), 50 | }, 51 | ]; 52 | 53 | mockedSqsClient.deleteMessageBatch.mockReturnValue( 54 | createAwsSdkRequestMock({ 55 | Successful: [], 56 | Failed: [], 57 | }), 58 | ); 59 | await sqsQueueService.deleteMessages(messageHeaders); 60 | expect(mockedSqsClient.deleteMessageBatch).toHaveBeenCalledWith({ 61 | QueueUrl: MESSAGE_QUEUE_URL, 62 | Entries: messageHeaders.map((m) => ({ 63 | Id: m.id, 64 | ReceiptHandle: m.receiptHandle, 65 | })), 66 | } as SQS.DeleteMessageBatchRequest); 67 | mockedSqsClient.deleteMessageBatch.mockReset(); 68 | }); 69 | 70 | it('should throw error if there are failed entries', async () => { 71 | const messageHeaders: DequeuedMessage[] = [ 72 | { 73 | id: 'testId', 74 | receiptHandle: 'testHandle', 75 | ...createMessageMock(), 76 | }, 77 | ]; 78 | 79 | mockedSqsClient.deleteMessageBatch.mockReturnValue( 80 | createAwsSdkRequestMock({ 81 | Successful: [], 82 | Failed: [{ Id: '', SenderFault: true, Code: '' }], 83 | }), 84 | ); 85 | expect(sqsQueueService.deleteMessages(messageHeaders)).rejects.toThrow( 86 | Error, 87 | ); 88 | mockedSqsClient.deleteMessageBatch.mockReset(); 89 | }); 90 | 91 | it('should delete no messages if input array is empty', async () => { 92 | await sqsQueueService.deleteMessages([]); 93 | expect(mockedSqsClient.deleteMessageBatch).toHaveBeenCalledTimes(0); 94 | mockedSqsClient.deleteMessageBatch.mockReset(); 95 | }); 96 | }); 97 | }); 98 | -------------------------------------------------------------------------------- /test/validators/message.validator.spec.ts: -------------------------------------------------------------------------------- 1 | import * as Express from 'express'; 2 | import * as request from 'supertest'; 3 | import { validateMessage } from '../../src/validators/message.validator'; 4 | import { checkValidation } from '../../src/middlewares/validator.middleware'; 5 | import { Response } from '../../src/interfaces/response.interface'; 6 | import { createMessageMock } from '../mock-factories/message.mock-factory'; 7 | import { json, urlencoded } from 'body-parser'; 8 | 9 | describe('message validator', () => { 10 | let app: Express.Express; 11 | beforeEach(() => { 12 | app = Express(); 13 | app.use(json()); 14 | app.use(urlencoded({ extended: true })); 15 | app.post( 16 | '/test', 17 | validateMessage, 18 | checkValidation, 19 | (req: Express.Request, res: Express.Response) => 20 | res.status(200).send('success'), 21 | ); 22 | }); 23 | 24 | it('should send status 200 if message is valid', async () => { 25 | const res = await request(app).post('/test').send(createMessageMock()); 26 | expect(res.status).toBe(200); 27 | expect((res.body as Response).errors).toBeUndefined(); 28 | }); 29 | 30 | it('should send status 400 if payload property is too short', async () => { 31 | const res = await request(app) 32 | .post('/test') 33 | .send(createMessageMock((m) => (m.payload = 'a'))); 34 | expect(res.status).toBe(400); 35 | expect((res.body as Response).errors).toHaveLength(1); 36 | }); 37 | 38 | }); 39 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "noImplicitAny": true, 9 | "target": "es2017", 10 | "allowJs": true, 11 | "sourceMap": true, 12 | "outDir": "./dist", 13 | "baseUrl": "./", 14 | "resolveJsonModule": true, 15 | "strict": true, 16 | "lib": ["es2020.string", "es2019.array", "es2020.promise"] 17 | }, 18 | "exclude": [ 19 | "bin", 20 | "coverage", 21 | "node_modules", 22 | "dist", 23 | ".build", 24 | ".serverless", 25 | "_optimize", 26 | "_warmup" 27 | ] 28 | } 29 | --------------------------------------------------------------------------------