├── .commitlintrc.yaml ├── .editorconfig ├── .eslintrc.json ├── .gitattributes ├── .github └── workflows │ ├── deploy.yaml │ └── test.yaml ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .lintstagedrc.yaml ├── .markdownlint.yaml ├── .prettierrc.yaml ├── .releaserc.yaml ├── .yarnrc.yml ├── LICENSE ├── README.md ├── package.json ├── renovate.json ├── src ├── config │ ├── index.ts │ ├── kafkaClusterConfigDto.ts │ ├── kafkaConsumerConfigDto.ts │ ├── kafkaProducerConfigDto.ts │ └── kafkaSchemaRegistryArgsConfigDto.ts ├── consts │ ├── consumersMapToken.ts │ ├── defaultConnectionName.ts │ ├── index.ts │ ├── kafkaBaseOptionsToken.ts │ ├── kafkaConsumerTransportId.ts │ ├── kafkaOptionsToken.ts │ ├── kafkaRetryConsumerTransportId.ts │ ├── producersMapToken.ts │ └── schemaRegistriesMapToken.ts ├── consumer │ ├── decorators │ │ ├── index.ts │ │ ├── kafkaConsumerEventPatternDecorator.ts │ │ ├── kafkaHeadersDecorator.ts │ │ ├── kafkaKeyDecorator.ts │ │ ├── kafkaRetryConsumerEventPatternDecorator.ts │ │ └── kafkaValueDecorator.ts │ ├── errors │ │ ├── getErrorCause.ts │ │ ├── index.ts │ │ ├── kafkaConsumerError.ts │ │ ├── kafkaConsumerNonRetriableError.ts │ │ ├── kafkaConsumerRetriableError.ts │ │ └── serializeError.ts │ ├── exceptions │ │ ├── index.ts │ │ ├── kafkaConsumerBaseExceptionFilter.ts │ │ └── kafkaConsumerErrorTopicExceptionFilter.ts │ ├── index.ts │ ├── interceptors │ │ ├── index.ts │ │ └── kafkaConsumerPayloadDecoder.ts │ ├── interfaces │ │ ├── index.ts │ │ ├── kafkaConsumerContextInterface.ts │ │ ├── kafkaConsumerErrorTopicExceptionFilterOptionsInterface.ts │ │ ├── kafkaConsumerOptionsInterface.ts │ │ ├── kafkaConsumerPayloadDecoderOptionsInterface.ts │ │ ├── kafkaConsumerPayloadHeadersInterface.ts │ │ ├── kafkaConsumerPayloadInterface.ts │ │ └── kafkaConsumerSerializedOptionsInterface.ts │ ├── kafkaConsumer.ts │ ├── kafkaConsumerMessageHandler.ts │ ├── kafkaConsumerMessageHandlerLogger.ts │ ├── kafkaRetryConsumer.ts │ └── retryStrategies │ │ ├── axiosRetryStrategy.ts │ │ ├── defaultRetryStrategy.ts │ │ ├── index.ts │ │ └── networkRetryStrategy.ts ├── decoratedProviders │ ├── decoratedProviderInterface.ts │ ├── decoratedProviders.ts │ └── index.ts ├── index.ts ├── kafkaModule.ts ├── options │ ├── index.ts │ ├── kafkaConnectionInterface.ts │ ├── kafkaOptionsInterface.ts │ └── kafkaSchemaRegistryConnectionInterface.ts ├── producer │ ├── decorators │ │ ├── index.ts │ │ ├── injectKafkaProducerDecorator.ts │ │ └── kafkaProducerDecoratedProviders.ts │ ├── index.ts │ ├── kafkaCoreProducer.ts │ └── kafkaProducer.ts └── schemaRegistry │ ├── decorators │ ├── index.ts │ ├── injectKafkaSchemaRegistryDecorator.ts │ └── kafkaSchemaRegistryDecoratedProviders.ts │ ├── index.ts │ ├── kafkaCoreSchemaRegistry.ts │ └── kafkaSchemaRegistry.ts ├── test └── expect.d.ts ├── tsconfig.build.json ├── tsconfig.json └── yarn.lock /.commitlintrc.yaml: -------------------------------------------------------------------------------- 1 | extends: 2 | - "@commitlint/config-conventional" 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | max_line_length = 80 10 | 11 | [*.{json, yaml, yml}] 12 | max_line_length = 120 13 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": ["@byndyusoft/eslint-config/header-apache-2.0", "@byndyusoft/eslint-config/backend"], 4 | "overrides": [ 5 | { 6 | "files": ["*.ts"], 7 | "rules": { 8 | // @typescript-eslint/eslint-plugin Supported Rules 9 | "@typescript-eslint/no-non-null-assertion": "off" // type hacks 10 | } 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ## GITATTRIBUTES FOR WEB PROJECTS 2 | # 3 | # These settings are for any web project. 4 | # 5 | # Details per file setting: 6 | # text These files should be normalized (i.e. convert CRLF to LF). 7 | # binary These files are binary and should be left untouched. 8 | # 9 | # Note that binary is a macro for -text -diff. 10 | ###################################################################### 11 | 12 | # Auto detect 13 | ## Handle line endings automatically for files detected as 14 | ## text and leave all files detected as binary untouched. 15 | ## This will handle all files NOT defined below. 16 | * text=auto 17 | 18 | # Source code 19 | *.bash text eol=lf 20 | *.bat text eol=crlf 21 | *.cmd text eol=crlf 22 | *.coffee text 23 | *.css text diff=css 24 | *.htm text diff=html 25 | *.html text diff=html 26 | *.inc text 27 | *.ini text 28 | *.js text 29 | *.json text 30 | *.jsx text 31 | *.less text 32 | *.ls text 33 | *.map text -diff 34 | *.od text 35 | *.onlydata text 36 | *.php text diff=php 37 | *.pl text 38 | *.ps1 text eol=crlf 39 | *.py text diff=python 40 | *.rb text diff=ruby 41 | *.sass text 42 | *.scm text 43 | *.scss text diff=css 44 | *.sh text eol=lf 45 | *.sql text 46 | *.styl text 47 | *.tag text 48 | *.ts text 49 | *.tsx text 50 | *.xml text 51 | *.xhtml text diff=html 52 | 53 | # Docker 54 | Dockerfile text 55 | 56 | # Documentation 57 | *.ipynb text 58 | *.markdown text diff=markdown 59 | *.md text diff=markdown 60 | *.mdwn text diff=markdown 61 | *.mdown text diff=markdown 62 | *.mkd text diff=markdown 63 | *.mkdn text diff=markdown 64 | *.mdtxt text 65 | *.mdtext text 66 | *.txt text 67 | AUTHORS text 68 | CHANGELOG text 69 | CHANGES text 70 | CONTRIBUTING text 71 | COPYING text 72 | copyright text 73 | *COPYRIGHT* text 74 | INSTALL text 75 | license text 76 | LICENSE text 77 | NEWS text 78 | readme text 79 | *README* text 80 | TODO text 81 | 82 | # Templates 83 | *.dot text 84 | *.ejs text 85 | *.erb text 86 | *.haml text 87 | *.handlebars text 88 | *.hbs text 89 | *.hbt text 90 | *.jade text 91 | *.latte text 92 | *.mustache text 93 | *.njk text 94 | *.phtml text 95 | *.svelte text 96 | *.tmpl text 97 | *.tpl text 98 | *.twig text 99 | *.vue text 100 | 101 | # Configs 102 | *.cnf text 103 | *.conf text 104 | *.config text 105 | .editorconfig text 106 | .env text 107 | .gitattributes text 108 | .gitconfig text 109 | .htaccess text 110 | *.lock text -diff 111 | package.json text eol=lf 112 | package-lock.json text -diff 113 | pnpm-lock.yaml text eol=lf -diff 114 | .prettierrc text 115 | yarn.lock text -diff 116 | *.toml text 117 | *.yaml text 118 | *.yml text 119 | browserslist text 120 | Makefile text 121 | makefile text 122 | 123 | # Heroku 124 | Procfile text 125 | 126 | # Graphics 127 | *.ai binary 128 | *.bmp binary 129 | *.eps binary 130 | *.gif binary 131 | *.gifv binary 132 | *.ico binary 133 | *.jng binary 134 | *.jp2 binary 135 | *.jpg binary 136 | *.jpeg binary 137 | *.jpx binary 138 | *.jxr binary 139 | *.pdf binary 140 | *.png binary 141 | *.psb binary 142 | *.psd binary 143 | # SVG treated as an asset (binary) by default. 144 | *.svg text 145 | # If you want to treat it as binary, 146 | # use the following line instead. 147 | # *.svg binary 148 | *.svgz binary 149 | *.tif binary 150 | *.tiff binary 151 | *.wbmp binary 152 | *.webp binary 153 | 154 | # Audio 155 | *.kar binary 156 | *.m4a binary 157 | *.mid binary 158 | *.midi binary 159 | *.mp3 binary 160 | *.ogg binary 161 | *.ra binary 162 | 163 | # Video 164 | *.3gpp binary 165 | *.3gp binary 166 | *.as binary 167 | *.asf binary 168 | *.asx binary 169 | *.avi binary 170 | *.fla binary 171 | *.flv binary 172 | *.m4v binary 173 | *.mng binary 174 | *.mov binary 175 | *.mp4 binary 176 | *.mpeg binary 177 | *.mpg binary 178 | *.ogv binary 179 | *.swc binary 180 | *.swf binary 181 | *.webm binary 182 | 183 | # Archives 184 | *.7z binary 185 | *.gz binary 186 | *.jar binary 187 | *.rar binary 188 | *.tar binary 189 | *.zip binary 190 | 191 | # Fonts 192 | *.ttf binary 193 | *.eot binary 194 | *.otf binary 195 | *.woff binary 196 | *.woff2 binary 197 | 198 | # Executables 199 | *.exe binary 200 | *.pyc binary 201 | 202 | # RC files (like .babelrc or .eslintrc) 203 | *.*rc text 204 | 205 | # Ignore files (like .npmignore or .gitignore) 206 | *.*ignore text 207 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: 4 | - workflow_dispatch 5 | 6 | jobs: 7 | deploy: 8 | uses: Byndyusoft/node-reusable-workflows/.github/workflows/deploy.yaml@master 9 | secrets: 10 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 11 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | test: 9 | uses: Byndyusoft/node-reusable-workflows/.github/workflows/test.yaml@master 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 2 | 3 | .idea/* 4 | 5 | ### https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore 6 | 7 | **/.vscode/* 8 | *.code-workspace 9 | 10 | # Local History for Visual Studio Code 11 | .history/ 12 | 13 | # Built Visual Studio Code Extensions 14 | *.vsix 15 | 16 | ### https://github.com/github/gitignore/blob/main/Node.gitignore 17 | 18 | # Logs 19 | logs 20 | *.log 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | lerna-debug.log* 25 | .pnpm-debug.log* 26 | 27 | # Diagnostic reports (https://nodejs.org/api/report.html) 28 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 29 | 30 | # Runtime data 31 | pids 32 | *.pid 33 | *.seed 34 | *.pid.lock 35 | 36 | # Directory for instrumented libs generated by jscoverage/JSCover 37 | lib-cov 38 | 39 | # Coverage directory used by tools like istanbul 40 | coverage 41 | *.lcov 42 | 43 | # nyc test coverage 44 | .nyc_output 45 | 46 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 47 | .grunt 48 | 49 | # Bower dependency directory (https://bower.io/) 50 | bower_components 51 | 52 | # node-waf configuration 53 | .lock-wscript 54 | 55 | # Compiled binary addons (https://nodejs.org/api/addons.html) 56 | build/Release 57 | 58 | # Dependency directories 59 | node_modules/ 60 | jspm_packages/ 61 | 62 | # Snowpack dependency directory (https://snowpack.dev/) 63 | web_modules/ 64 | 65 | # TypeScript cache 66 | *.tsbuildinfo 67 | 68 | # Optional npm cache directory 69 | .npm 70 | 71 | # Optional eslint cache 72 | .eslintcache 73 | 74 | # Optional stylelint cache 75 | .stylelintcache 76 | 77 | # Microbundle cache 78 | .rpt2_cache/ 79 | .rts2_cache_cjs/ 80 | .rts2_cache_es/ 81 | .rts2_cache_umd/ 82 | 83 | # Optional REPL history 84 | .node_repl_history 85 | 86 | # Output of 'npm pack' 87 | *.tgz 88 | 89 | # Yarn Integrity file 90 | .yarn-integrity 91 | 92 | # dotenv environment variable files 93 | .env 94 | .env.development.local 95 | .env.test.local 96 | .env.production.local 97 | .env.local 98 | 99 | # parcel-bundler cache (https://parceljs.org/) 100 | .cache 101 | .parcel-cache 102 | 103 | # Next.js build output 104 | .next 105 | out 106 | 107 | # Nuxt.js build / generate output 108 | .nuxt 109 | dist 110 | 111 | # Gatsby files 112 | .cache/ 113 | # Comment in the public line in if your project uses Gatsby and not Next.js 114 | # https://nextjs.org/blog/next-9-1#public-directory-support 115 | # public 116 | 117 | # vuepress build output 118 | .vuepress/dist 119 | 120 | # vuepress v2.x temp and cache directory 121 | .temp 122 | .cache 123 | 124 | # Docusaurus cache and generated files 125 | .docusaurus 126 | 127 | # Serverless directories 128 | .serverless/ 129 | 130 | # FuseBox cache 131 | .fusebox/ 132 | 133 | # DynamoDB Local files 134 | .dynamodb/ 135 | 136 | # TernJS port file 137 | .tern-port 138 | 139 | # Stores VSCode versions used for testing VSCode extensions 140 | .vscode-test 141 | 142 | # yarn v2 143 | .yarn/* 144 | !.yarn/patches 145 | !.yarn/plugins 146 | !.yarn/releases 147 | .pnp.* 148 | 149 | ### https://github.com/github/gitignore/blob/main/Global/Linux.gitignore 150 | 151 | *~ 152 | 153 | # temporary files which can be created if a process still has a handle open of a deleted file 154 | .fuse_hidden* 155 | 156 | # KDE directory preferences 157 | .directory 158 | 159 | # Linux trash folder which might appear on any partition or disk 160 | .Trash-* 161 | 162 | # .nfs files are created when an open file is removed but is still being accessed 163 | .nfs* 164 | 165 | ### https://github.com/github/gitignore/blob/main/Global/Windows.gitignore 166 | 167 | # Windows thumbnail cache files 168 | Thumbs.db 169 | Thumbs.db:encryptable 170 | ehthumbs.db 171 | ehthumbs_vista.db 172 | 173 | # Dump file 174 | *.stackdump 175 | 176 | # Folder config file 177 | [Dd]esktop.ini 178 | 179 | # Recycle Bin used on file shares 180 | $RECYCLE.BIN/ 181 | 182 | # Windows Installer files 183 | *.cab 184 | *.msi 185 | *.msix 186 | *.msm 187 | *.msp 188 | 189 | # Windows shortcuts 190 | *.lnk 191 | 192 | ### https://github.com/github/gitignore/blob/main/Global/macOS.gitignore 193 | 194 | # General 195 | .DS_Store 196 | .AppleDouble 197 | .LSOverride 198 | 199 | # Icon must end with two \r 200 | Icon 201 | 202 | 203 | # Thumbnails 204 | ._* 205 | 206 | # Files that might appear in the root of a volume 207 | .DocumentRevisions-V100 208 | .fseventsd 209 | .Spotlight-V100 210 | .TemporaryItems 211 | .Trashes 212 | .VolumeIcon.icns 213 | .com.apple.timemachine.donotpresent 214 | 215 | # Directories potentially created on remote AFP share 216 | .AppleDB 217 | .AppleDesktop 218 | Network Trash Folder 219 | Temporary Items 220 | .apdisk 221 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | yarn commitlint --edit "$1" 5 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | yarn lint-staged 5 | -------------------------------------------------------------------------------- /.lintstagedrc.yaml: -------------------------------------------------------------------------------- 1 | "*.{ts,js}": eslint 2 | "*.md": markdownlint 3 | "*.{ts,js,json,yaml,yml,md}": prettier 4 | -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- 1 | MD013: false 2 | MD033: false 3 | -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | trailingComma: all 2 | endOfLine: auto 3 | -------------------------------------------------------------------------------- /.releaserc.yaml: -------------------------------------------------------------------------------- 1 | branches: 2 | - name: master 3 | - name: next 4 | prerelease: rc 5 | 6 | plugins: 7 | - "@semantic-release/commit-analyzer" 8 | - "@semantic-release/release-notes-generator" 9 | - "@semantic-release/npm" 10 | - "@semantic-release/github" 11 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nest-kafka 2 | 3 | [![npm@latest](https://img.shields.io/npm/v/@byndyusoft/nest-kafka/latest.svg)](https://www.npmjs.com/package/@byndyusoft/nest-kafka) 4 | [![test](https://github.com/Byndyusoft/nest-kafka/actions/workflows/test.yaml/badge.svg?branch=master)](https://github.com/Byndyusoft/nest-kafka/actions/workflows/test.yaml) 5 | [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier) 6 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) 7 | 8 | Kafka for NestJS 9 | 10 | ## Features 11 | 12 | - Multiple connections 13 | - Consumer and Producer with Schema Registry support (using [kafkajs](https://www.npmjs.com/package/kafkajs) and [@kafkajs/confluent-schema-registry](https://www.npmjs.com/package/@kafkajs/confluent-schema-registry) under the hood) 14 | - Integration with [nest-template](https://github.com/Byndyusoft/nest-template) 15 | - Consumer 16 | - Subscribe topic is not static, you can pick it from config 17 | - Process message in async context with [Tracing](https://www.npmjs.com/package/@byndyusoft/nest-opentracing) and [Logging](https://www.npmjs.com/package/@byndyusoft/nest-pino) 18 | - String, JSON and Schema Registry decoders for key and value, headers decoder with array support 19 | - [Dead letter queue](https://www.confluent.io/blog/error-handling-patterns-in-kafka/#pattern-2) pattern support with smart retry mechanism 20 | - Support custom decoders and error handling patterns 21 | 22 | ## Requirements 23 | 24 | - Node.js v14 LTS or later 25 | - Yarn 26 | 27 | ## Install 28 | 29 | ```bash 30 | yarn add @byndyusoft/nest-kafka @byndyusoft/class-validator-extended @byndyusoft/nest-opentracing @byndyusoft/nest-pino @kafkajs/confluent-schema-registry @nestjs/common @nestjs/microservices class-transformer class-validator kafkajs rxjs 31 | ``` 32 | 33 | ## Usage 34 | 35 | ### Init 36 | 37 |
38 | 1. Create KafkaConfigDto 39 | 40 | ```typescript 41 | import { 42 | KafkaClusterConfigDto, 43 | KafkaConsumerConfigDto, 44 | KafkaProducerConfigDto, 45 | KafkaSchemaRegistryArgsConfigDto, 46 | } from "@byndyusoft/nest-kafka"; 47 | import { Type } from "class-transformer"; 48 | import { IsDefined, IsString, ValidateNested } from "class-validator"; 49 | 50 | export class KafkaConfigDto { 51 | @Type(() => KafkaClusterConfigDto) 52 | @IsDefined() 53 | @ValidateNested() 54 | public readonly cluster!: KafkaClusterConfigDto; 55 | 56 | @Type(() => KafkaConsumerConfigDto) 57 | @IsDefined() 58 | @ValidateNested() 59 | public readonly consumer!: KafkaConsumerConfigDto; 60 | 61 | @Type(() => KafkaProducerConfigDto) 62 | @IsDefined() 63 | @ValidateNested() 64 | public readonly producer!: KafkaProducerConfigDto; 65 | 66 | @Type(() => KafkaSchemaRegistryArgsConfigDto) 67 | @IsDefined() 68 | @ValidateNested() 69 | public readonly schemaRegistry!: KafkaSchemaRegistryArgsConfigDto; 70 | 71 | @IsString() 72 | public readonly topic!: string; 73 | 74 | @IsString() 75 | public readonly errorTopic!: string; 76 | } 77 | ``` 78 | 79 |
80 | 81 |
82 | 2. Add KafkaConfigDto into ConfigDto 83 | 84 | ```typescript 85 | import { Type } from "class-transformer"; 86 | import { IsDefined, ValidateNested } from "class-validator"; 87 | 88 | import { KafkaConfigDto } from "./kafkaConfigDto"; 89 | 90 | export class ConfigDto { 91 | /// ... 92 | 93 | @Type(() => KafkaConfigDto) 94 | @IsDefined() 95 | @ValidateNested() 96 | public readonly kafka!: KafkaConfigDto; 97 | 98 | /// ... 99 | } 100 | ``` 101 | 102 |
103 | 104 |
105 | 3. Add env variables mapping 106 | 107 | ```typescript 108 | import { Module } from "@nestjs/common"; 109 | 110 | import { ConfigDto } from "./dtos"; 111 | 112 | @Module({}) 113 | export class ConfigModule { 114 | // ... 115 | 116 | private static __loadConfig(): ConfigDto { 117 | const plainConfig: ConfigDto = { 118 | // ... 119 | kafka: { 120 | cluster: { 121 | brokers: process.env.KAFKA_BROKERS as string, 122 | saslMechanism: process.env.KAFKA_SASL_MECHANISM, 123 | username: process.env.KAFKA_USERNAME, 124 | password: process.env.KAFKA_PASSWORD, 125 | ssl: process.env.KAFKA_SSL, 126 | ca: process.env.KAFKA_CA, 127 | connectionTimeout: process.env.KAFKA_CONNECTION_TIMEOUT, // default is 1 s. 128 | }, 129 | consumer: { 130 | groupId: process.env.KAFKA_CONSUMER_GROUP_ID as string, 131 | allowAutoTopicCreation: 132 | process.env.KAFKA_CONSUMER_ALLOW_AUTO_TOPIC_CREATION ?? true, 133 | sessionTimeout: process.env.KAFKA_SESSION_TIMEOUT_MS ?? 30000, 134 | heartbeatInterval: process.env.KAFKA_HEARTBEAT_INTERVAL_MS ?? 3000, 135 | }, 136 | producer: { 137 | allowAutoTopicCreation: 138 | process.env.KAFKA_PRODUCER_ALLOW_AUTO_TOPIC_CREATION ?? true, 139 | }, 140 | schemaRegistry: { 141 | host: process.env.KAFKA_SCHEMA_REGISTRY_HOST as string, 142 | username: process.env.KAFKA_SCHEMA_REGISTRY_USERNAME, 143 | password: process.env.KAFKA_SCHEMA_REGISTRY_PASSWORD, 144 | }, 145 | topic: process.env.KAFKA_TOPIC as string, 146 | errorTopic: process.env.KAFKA_ERROR_TOPIC as string, 147 | }, 148 | // ... 149 | }; 150 | 151 | // ... 152 | } 153 | } 154 | ``` 155 | 156 |
157 | 158 |
159 | 4. Import KafkaModule 160 | 161 | ```typescript 162 | import { 163 | KafkaClusterConfigDto, 164 | KafkaConsumerConfigDto, 165 | KafkaModule, 166 | KafkaProducerConfigDto, 167 | KafkaSchemaRegistryArgsConfigDto, 168 | } from "@byndyusoft/nest-kafka"; 169 | 170 | import { ConfigDto } from "./config"; 171 | 172 | @Module({ 173 | imports: [ 174 | // Extra modules 175 | // ... 176 | KafkaModule.registerAsync({ 177 | inject: [ConfigDto], 178 | useFactory: (config: ConfigDto) => ({ 179 | connections: [ 180 | { 181 | cluster: KafkaClusterConfigDto.toRawConfig(config.kafka.cluster), 182 | consumer: KafkaConsumerConfigDto.toRawConfig(config.kafka.consumer), 183 | producer: KafkaProducerConfigDto.toRawConfig(config.kafka.producer), 184 | schemaRegistry: { 185 | args: KafkaSchemaRegistryArgsConfigDto.toRawConfig( 186 | config.kafka.schemaRegistry, 187 | ), 188 | }, 189 | }, 190 | ], 191 | topicPickerArgs: [config], 192 | }), 193 | }), 194 | // ... 195 | ], 196 | }) 197 | export class InfrastructureModule { 198 | // ... 199 | } 200 | ``` 201 | 202 |
203 | 204 |
205 | 4.1. You can describe multiple connections (farther use connectionName parameter in some functions to specify your connection) 206 | 207 | ```typescript 208 | import { 209 | KafkaClusterConfigDto, 210 | KafkaConsumerConfigDto, 211 | KafkaModule, 212 | KafkaProducerConfigDto, 213 | KafkaSchemaRegistryArgsConfigDto, 214 | } from "@byndyusoft/nest-kafka"; 215 | 216 | import { ConfigDto } from "./config"; 217 | 218 | @Module({ 219 | imports: [ 220 | // Extra modules 221 | // ... 222 | KafkaModule.registerAsync({ 223 | inject: [ConfigDto], 224 | useFactory: (config: ConfigDto) => ({ 225 | connections: [ 226 | { 227 | name: "connection1", 228 | cluster: KafkaClusterConfigDto.toRawConfig(config.kafka1.cluster), 229 | consumer: KafkaConsumerConfigDto.toRawConfig( 230 | config.kafka1.consumer, 231 | ), 232 | producer: KafkaProducerConfigDto.toRawConfig( 233 | config.kafka1.producer, 234 | ), 235 | schemaRegistry: { 236 | args: KafkaSchemaRegistryArgsConfigDto.toRawConfig( 237 | config.kafka1.schemaRegistry, 238 | ), 239 | }, 240 | }, 241 | { 242 | name: "connection2", 243 | cluster: KafkaClusterConfigDto.toRawConfig(config.kafka2.cluster), 244 | consumer: KafkaConsumerConfigDto.toRawConfig( 245 | config.kafka2.consumer, 246 | ), 247 | producer: KafkaProducerConfigDto.toRawConfig( 248 | config.kafka2.producer, 249 | ), 250 | schemaRegistry: { 251 | args: KafkaSchemaRegistryArgsConfigDto.toRawConfig( 252 | config.kafka2.schemaRegistry, 253 | ), 254 | }, 255 | }, 256 | ], 257 | topicPickerArgs: [config], 258 | }), 259 | }), 260 | // ... 261 | ], 262 | }) 263 | export class InfrastructureModule { 264 | // ... 265 | } 266 | ``` 267 | 268 |
269 | 270 |
271 | 4.2. If you want, you can not create consumer, producer or schemaRegistry 272 | 273 | ```typescript 274 | import { 275 | KafkaClusterConfigDto, 276 | KafkaConsumerConfigDto, 277 | KafkaModule, 278 | KafkaProducerConfigDto, 279 | KafkaSchemaRegistryArgsConfigDto, 280 | } from "@byndyusoft/nest-kafka"; 281 | 282 | import { ConfigDto } from "./config"; 283 | 284 | @Module({ 285 | imports: [ 286 | // Extra modules 287 | // ... 288 | KafkaModule.registerAsync({ 289 | inject: [ConfigDto], 290 | useFactory: (config: ConfigDto) => ({ 291 | connections: [ 292 | { 293 | cluster: KafkaClusterConfigDto.toRawConfig(config.kafka.cluster), 294 | consumer: KafkaConsumerConfigDto.toRawConfig(config.kafka.consumer), 295 | }, 296 | ], 297 | topicPickerArgs: [config], 298 | }), 299 | }), 300 | // ... 301 | ], 302 | }) 303 | export class InfrastructureModule { 304 | // ... 305 | } 306 | ``` 307 | 308 |
309 | 310 |
311 | 5. Connect microservice to start consuming messages 312 | 313 | ```typescript 314 | import { KafkaConsumer, KafkaRetryConsumer } from "@byndyusoft/nest-kafka"; 315 | import { MicroserviceOptions } from "@nestjs/microservices"; 316 | 317 | async function bootstrap(): Promise { 318 | // ... 319 | 320 | app.connectMicroservice({ 321 | strategy: app.get(KafkaConsumer), 322 | }); 323 | 324 | // you can optionally connect retry consumer 325 | app.connectMicroservice({ 326 | strategy: app.get(KafkaRetryConsumer), 327 | }); 328 | 329 | // Put `app.listen(...)` before `app.startAllMicroservice()` 330 | await app.listen(...) 331 | 332 | await app.startAllMicroservices(); 333 | 334 | // ... 335 | } 336 | 337 | // ... 338 | ``` 339 | 340 |
341 | 342 | ### Consuming Messages 343 | 344 | > [!IMPORTANT] 345 | > Put `app.startAllMicroservices()` after your `app.listen(...)` 346 | 347 | 348 | 349 |
350 | 1. Create controller and use KafkaConsumerEventPattern to describe consumer 351 | 352 | ```typescript 353 | import { 354 | IKafkaConsumerPayload, 355 | KafkaConsumerEventPattern, 356 | } from "@byndyusoft/nest-kafka"; 357 | import { Controller } from "@nestjs/common"; 358 | import { Payload } from "@nestjs/microservices"; 359 | 360 | import { ConfigDto } from "~/src"; 361 | 362 | @Controller() 363 | export class UsersConsumer { 364 | @KafkaConsumerEventPattern({ 365 | topicPicker: (config: ConfigDto) => config.kafka.topic, 366 | fromBeginning: true, 367 | }) 368 | public async onMessage( 369 | @Payload() payload: IKafkaConsumerPayload, 370 | ): Promise { 371 | // ... 372 | } 373 | } 374 | ``` 375 | 376 |
377 | 378 | 379 | 380 |
381 | 2. Decode payload 382 | 383 | ```typescript 384 | import { 385 | IKafkaConsumerPayload, 386 | KafkaConsumerEventPattern, 387 | KafkaConsumerPayloadDecoder, 388 | } from "@byndyusoft/nest-kafka"; 389 | import { Controller, UseInterceptors } from "@nestjs/common"; 390 | import { Payload } from "@nestjs/microservices"; 391 | 392 | import { ConfigDto } from "~/src"; 393 | import { UserDto } from "ᐸDtosᐳ"; 394 | 395 | @Controller() 396 | export class UsersConsumer { 397 | @KafkaConsumerEventPattern({ 398 | topicPicker: (config: ConfigDto) => config.kafka.topic, 399 | fromBeginning: true, 400 | }) 401 | @UseInterceptors( 402 | new KafkaConsumerPayloadDecoder({ 403 | key: "string", 404 | value: "json", 405 | headers: "string", 406 | }), 407 | ) 408 | public async onMessage( 409 | @Payload() payload: IKafkaConsumerPayload, 410 | ): Promise { 411 | // ... 412 | } 413 | } 414 | ``` 415 | 416 |
417 | 418 |
419 | 2.1. You can use param decorators to get key, value or headers 420 | 421 | ```typescript 422 | import { 423 | IKafkaConsumerPayloadHeaders, 424 | KafkaConsumerEventPattern, 425 | KafkaConsumerPayloadDecoder, 426 | KafkaHeaders, 427 | KafkaKey, 428 | KafkaValue, 429 | } from "@byndyusoft/nest-kafka"; 430 | import { Controller, UseInterceptors } from "@nestjs/common"; 431 | 432 | import { ConfigDto } from "~/src"; 433 | import { UserDto } from "ᐸDtosᐳ"; 434 | 435 | @Controller() 436 | export class UsersConsumer { 437 | @KafkaConsumerEventPattern({ 438 | topicPicker: (config: ConfigDto) => config.kafka.topic, 439 | fromBeginning: true, 440 | }) 441 | @UseInterceptors( 442 | new KafkaConsumerPayloadDecoder({ 443 | key: "string", 444 | value: "json", 445 | headers: "string", 446 | }), 447 | ) 448 | public async onMessage( 449 | @KafkaKey() key: string, 450 | @KafkaValue() value: UserDto, 451 | @KafkaHeaders() headers: IKafkaConsumerPayloadHeaders, 452 | ): Promise { 453 | // ... 454 | } 455 | } 456 | ``` 457 | 458 |
459 | 460 |
461 | 3. Always use some exception filter for correct error handling 462 | 463 | ```typescript 464 | import { 465 | KafkaConsumerBaseExceptionFilter, 466 | KafkaConsumerEventPattern, 467 | } from "@byndyusoft/nest-kafka"; 468 | import { Controller, UseFilters } from "@nestjs/common"; 469 | 470 | import { ConfigDto } from "~/src"; 471 | 472 | @Controller() 473 | export class UsersConsumer { 474 | @KafkaConsumerEventPattern({ 475 | topicPicker: (config: ConfigDto) => config.kafka.topic, 476 | fromBeginning: true, 477 | }) 478 | @UseFilters(/* ... */) 479 | public async onMessage(): Promise { 480 | throw new Error("some error"); 481 | } 482 | } 483 | ``` 484 | 485 |
486 | 487 |
488 | 3.1. Use KafkaConsumerBaseExceptionFilter if you prefer Stop on error pattern 489 | 490 | ```typescript 491 | import { 492 | KafkaConsumerBaseExceptionFilter, 493 | KafkaConsumerEventPattern, 494 | } from "@byndyusoft/nest-kafka"; 495 | import { Controller, UseFilters } from "@nestjs/common"; 496 | 497 | import { ConfigDto } from "~/src"; 498 | 499 | @Controller() 500 | export class UsersConsumer { 501 | @KafkaConsumerEventPattern({ 502 | topicPicker: (config: ConfigDto) => config.kafka.topic, 503 | fromBeginning: true, 504 | }) 505 | @UseFilters(new KafkaConsumerBaseExceptionFilter()) 506 | public async onMessage(): Promise { 507 | throw new Error("some error"); 508 | } 509 | } 510 | ``` 511 | 512 |
513 | 514 |
515 | 3.2. Use KafkaConsumerErrorTopicExceptionFilter if you prefer Dead letter queue pattern 516 | 517 | ```typescript 518 | import { 519 | KafkaConsumerErrorTopicExceptionFilter, 520 | KafkaConsumerEventPattern, 521 | } from "@byndyusoft/nest-kafka"; 522 | import { Controller, UseFilters } from "@nestjs/common"; 523 | 524 | import { ConfigDto } from "~/src"; 525 | 526 | @Controller() 527 | export class UsersConsumer { 528 | @KafkaConsumerEventPattern({ 529 | topicPicker: (config: ConfigDto) => config.kafka.topic, 530 | fromBeginning: true, 531 | }) 532 | @UseFilters( 533 | new KafkaConsumerErrorTopicExceptionFilter({ 534 | errorTopicPicker: (config: ConfigDto) => config.kafka.errorTopic, 535 | }), 536 | ) 537 | public async onMessage(): Promise { 538 | throw new Error("some error"); 539 | } 540 | } 541 | ``` 542 | 543 |
544 | 545 |
546 | 3.3. KafkaConsumerErrorTopicExceptionFilter also support retry topic for retriable errors 547 | 548 | ```typescript 549 | import { 550 | KafkaConsumerErrorTopicExceptionFilter, 551 | KafkaConsumerEventPattern, 552 | } from "@byndyusoft/nest-kafka"; 553 | import { Controller, UseFilters } from "@nestjs/common"; 554 | 555 | import { ConfigDto } from "~/src"; 556 | 557 | @Controller() 558 | export class UsersConsumer { 559 | @KafkaConsumerEventPattern({ 560 | topicPicker: (config: ConfigDto) => config.kafka.topic, 561 | fromBeginning: true, 562 | }) 563 | @UseFilters( 564 | new KafkaConsumerErrorTopicExceptionFilter({ 565 | retryTopicPicker: (config: ConfigDto) => config.kafka.retryTopic, 566 | errorTopicPicker: (config: ConfigDto) => config.kafka.errorTopic, 567 | }), 568 | ) 569 | public async onMessage(): Promise { 570 | throw new Error("some error"); 571 | } 572 | } 573 | ``` 574 | 575 |
576 | 577 |
578 | 3.4. Use retry consumer to consume messages from retry topic 579 | 580 | ```typescript 581 | import { 582 | KafkaConsumerErrorTopicExceptionFilter, 583 | KafkaConsumerEventPattern, 584 | } from "@byndyusoft/nest-kafka"; 585 | import { Controller, UseFilters } from "@nestjs/common"; 586 | 587 | import { ConfigDto } from "~/src"; 588 | 589 | @Controller() 590 | export class UsersRetryConsumer { 591 | @KafkaRetryConsumerEventPattern({ 592 | topicPicker: (config: ConfigDto) => config.kafka.retryTopic, 593 | fromBeginning: true, 594 | }) 595 | @UseFilters( 596 | new KafkaConsumerErrorTopicExceptionFilter({ 597 | retryTopicPicker: false, 598 | errorTopicPicker: (config: ConfigDto) => config.kafka.errorTopic, 599 | resendHeadersPrefix: "retry", 600 | }), 601 | ) 602 | public async onMessage(): Promise { 603 | throw new Error("some error"); 604 | } 605 | } 606 | ``` 607 | 608 | Run retry consumer, e.g by HTTP: 609 | 610 | ```typescript 611 | import { ApiTags } from "@byndyusoft/nest-swagger"; 612 | import { Body, Controller, HttpCode, HttpStatus, Post } from "@nestjs/common"; 613 | 614 | import { ApiCommonResponses } from "../infrastructure"; 615 | 616 | import { RunDeliveryAppointmentsRetryConsumerOnceBodyDto } from "./dtos"; 617 | import { RunDeliveryAppointmentsRetryConsumerOnceUseCase } from "./useCases"; 618 | 619 | @ApiTags("Users") 620 | @Controller({ 621 | path: "/users/retry", 622 | version: "1", 623 | }) 624 | export class UsersRetryController { 625 | public constructor( 626 | private readonly config: ConfigDto, 627 | private readonly kafkaRetryConsumer: KafkaRetryConsumer, 628 | ) {} 629 | 630 | @ApiCommonResponses(HttpStatus.BAD_REQUEST) 631 | @HttpCode(HttpStatus.NO_CONTENT) 632 | @Post("/runRetryConsumerOnce") 633 | public runDeliveryAppointmentsRetryConsumerOnce(): Promise { 634 | await this.kafkaRetryConsumer.runOnce({ 635 | topic: config.kafka.retryTopic, 636 | messagesCount: 1, 637 | }); 638 | } 639 | } 640 | ``` 641 | 642 |
643 | 644 | ### Producing Messages 645 | 646 | 647 | 648 |
649 | 1. Inject KafkaProducer 650 | 651 | ```typescript 652 | import { InjectKafkaProducer, KafkaProducer } from "@byndyusoft/nest-kafka"; 653 | import { Injectable } from "@nestjs/common"; 654 | 655 | @Injectable() 656 | export class UsersService { 657 | public constructor( 658 | @InjectKafkaProducer() 659 | private readonly __kafkaProducer: KafkaProducer, 660 | ) {} 661 | } 662 | ``` 663 | 664 |
665 | 666 | ### Schema Registry 667 | 668 | 669 | 670 |
671 | 1. Inject KafkaSchemaRegistry 672 | 673 | ```typescript 674 | import { 675 | InjectKafkaSchemaRegistry, 676 | KafkaSchemaRegistry, 677 | } from "@byndyusoft/nest-kafka"; 678 | import { Injectable } from "@nestjs/common"; 679 | 680 | @Injectable() 681 | export class UsersService { 682 | public constructor( 683 | @InjectKafkaSchemaRegistry() 684 | private readonly __kafkaSchemaRegistry: KafkaSchemaRegistry, 685 | ) {} 686 | } 687 | ``` 688 | 689 |
690 | 691 | ## Maintainers 692 | 693 | - [@Byndyusoft/owners](https://github.com/orgs/Byndyusoft/teams/owners) <> 694 | - [@Byndyusoft/team](https://github.com/orgs/Byndyusoft/teams/team) 695 | - [@KillWolfVlad](https://github.com/KillWolfVlad) 696 | 697 | ## License 698 | 699 | This repository is released under version 2.0 of the 700 | [Apache License](https://www.apache.org/licenses/LICENSE-2.0). 701 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@byndyusoft/nest-kafka", 3 | "version": "0.0.0-development", 4 | "description": "Kafka for NestJS", 5 | "homepage": "https://github.com/Byndyusoft/nest-kafka#readme", 6 | "bugs": { 7 | "url": "https://github.com/Byndyusoft/nest-kafka/issues" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/Byndyusoft/nest-kafka.git" 12 | }, 13 | "license": "Apache-2.0", 14 | "author": "Byndyusoft", 15 | "main": "./dist/index.js", 16 | "types": "./dist/index.d.ts", 17 | "files": [ 18 | "dist", 19 | "src", 20 | "!dist/*.tsbuildinfo" 21 | ], 22 | "scripts": { 23 | "_test:cov": "jest --coverage", 24 | "prebuild": "shx rm -rf ./dist", 25 | "build": "yarn run prebuild && yarn run build:src", 26 | "build:src": "tsc --project ./tsconfig.build.json", 27 | "postinstall": "husky install", 28 | "lint": "yarn run lint:eslint && yarn run lint:markdown && yarn run lint:prettier", 29 | "lint:eslint": "eslint --ignore-path ./.gitignore --max-warnings 0 --ext .ts,.js .", 30 | "lint:eslint:fix": "eslint --ignore-path ./.gitignore --fix --ext .ts,.js .", 31 | "lint:fix": "yarn run lint:eslint:fix && yarn run lint:markdown:fix && yarn run lint:prettier:fix", 32 | "lint:markdown": "markdownlint --ignore-path ./.gitignore \"./**/*.md\"", 33 | "lint:markdown:fix": "markdownlint --ignore-path ./.gitignore --fix \"./**/*.md\"", 34 | "lint:prettier": "prettier --ignore-path ./.gitignore --check \"./**/*.{ts,js,json,yaml,yml,md}\"", 35 | "lint:prettier:fix": "prettier --ignore-path ./.gitignore --write \"./**/*.{ts,js,json,yaml,yml,md}\"", 36 | "prepublishOnly": "pinst --disable", 37 | "postpublish": "pinst --enable", 38 | "test": "jest" 39 | }, 40 | "jest": { 41 | "collectCoverageFrom": [ 42 | "./src/**/*.ts", 43 | "!**/index.ts" 44 | ], 45 | "coverageDirectory": "/coverage", 46 | "coverageReporters": [ 47 | "lcov", 48 | "text" 49 | ], 50 | "moduleFileExtensions": [ 51 | "ts", 52 | "js" 53 | ], 54 | "resetMocks": true, 55 | "roots": [ 56 | "/test" 57 | ], 58 | "setupFilesAfterEnv": [ 59 | "jest-extended/all" 60 | ], 61 | "testEnvironment": "node", 62 | "testRegex": ".*\\.test\\.ts$", 63 | "transform": { 64 | "^.+\\.ts$": "ts-jest" 65 | } 66 | }, 67 | "dependencies": { 68 | "@byndyusoft/nest-dynamic-module": "^1.0.0", 69 | "@types/error-cause": "^1.0.1", 70 | "@types/retry": "^0.12.2", 71 | "error-cause": "^1.0.5", 72 | "lodash": "^4.17.21", 73 | "retry": "^0.13.1", 74 | "tslib": "^2.4.1" 75 | }, 76 | "devDependencies": { 77 | "@byndyusoft/class-validator-extended": "1.0.1", 78 | "@byndyusoft/eslint-config": "2.2.1", 79 | "@byndyusoft/nest-opentracing": "2.3.2", 80 | "@byndyusoft/nest-pino": "3.1.2-1", 81 | "@byndyusoft/tsconfig": "1.2.0", 82 | "@commitlint/cli": "17.4.1", 83 | "@commitlint/config-conventional": "17.4.0", 84 | "@kafkajs/confluent-schema-registry": "3.3.0", 85 | "@nestjs/common": "9.2.1", 86 | "@nestjs/microservices": "9.2.1", 87 | "@types/jest": "29.2.5", 88 | "@types/lodash": "4.14.191", 89 | "@types/node": "14.18.36", 90 | "axios": "1.8.4", 91 | "class-transformer": "0.5.1", 92 | "class-validator": "0.14.0", 93 | "eslint": "8.31.0", 94 | "husky": "8.0.3", 95 | "jest": "29.3.1", 96 | "jest-extended": "3.2.2", 97 | "kafkajs": "2.2.3", 98 | "lint-staged": "13.1.0", 99 | "markdownlint-cli": "0.33.0", 100 | "pino": "8.8.0", 101 | "pinst": "3.0.0", 102 | "prettier": "2.8.2", 103 | "prettier-plugin-packagejson": "2.3.0", 104 | "rxjs": "7.8.0", 105 | "semantic-release": "20.0.2", 106 | "shx": "0.3.4", 107 | "ts-jest": "29.0.3", 108 | "ts-patch": "2.1.0", 109 | "typescript": "4.9.4", 110 | "typescript-transform-paths": "3.4.6" 111 | }, 112 | "peerDependencies": { 113 | "@byndyusoft/class-validator-extended": "^1.0.1", 114 | "@byndyusoft/nest-opentracing": "^2.3.2", 115 | "@byndyusoft/nest-pino": "^3.1.2-1", 116 | "@kafkajs/confluent-schema-registry": "^3.3.0", 117 | "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", 118 | "@nestjs/microservices": "^8.0.0 || ^9.0.0 || ^10.0.0", 119 | "axios": "^0.27.2 || ^1.0.0", 120 | "class-transformer": "^0.5.1", 121 | "class-validator": "^0.14.0", 122 | "kafkajs": "^2.2.3", 123 | "rxjs": "^7.8.0" 124 | }, 125 | "packageManager": "yarn@4.0.0-rc.35", 126 | "engines": { 127 | "node": ">=14" 128 | }, 129 | "publishConfig": { 130 | "access": "public" 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["local>Byndyusoft/node-renovate-config"] 3 | } 4 | -------------------------------------------------------------------------------- /src/config/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaClusterConfigDto"; 18 | export * from "./kafkaConsumerConfigDto"; 19 | export * from "./kafkaProducerConfigDto"; 20 | export * from "./kafkaSchemaRegistryArgsConfigDto"; 21 | -------------------------------------------------------------------------------- /src/config/kafkaClusterConfigDto.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | TransformToBoolean, 19 | TransformToNumber, 20 | } from "@byndyusoft/class-validator-extended"; 21 | import { Transform } from "class-transformer"; 22 | import { 23 | ArrayMinSize, 24 | IsArray, 25 | IsBoolean, 26 | IsIn, 27 | IsInt, 28 | IsOptional, 29 | IsString, 30 | } from "class-validator"; 31 | import { KafkaConfig } from "kafkajs"; 32 | 33 | type TSupportedSaslMechanism = "plain" | "scram-sha-256" | "scram-sha-512"; 34 | 35 | interface ITransformedKafkaClusterConfig { 36 | readonly brokers: string[]; 37 | readonly saslMechanism?: TSupportedSaslMechanism; 38 | readonly username?: string; 39 | readonly password?: string; 40 | readonly ssl?: boolean; 41 | readonly ca?: Buffer; 42 | } 43 | 44 | export class KafkaClusterConfigDto { 45 | public static toRawConfig(config: KafkaClusterConfigDto): KafkaConfig { 46 | const transformedConfig = 47 | config as unknown as ITransformedKafkaClusterConfig; 48 | 49 | const { brokers } = transformedConfig; 50 | 51 | return { 52 | brokers, 53 | connectionTimeout: config.connectionTimeout, 54 | ...this.getKafkaSslConfig(transformedConfig), 55 | ...this.getKafkaSaslConfig(transformedConfig), 56 | }; 57 | } 58 | 59 | private static getKafkaSslConfig({ 60 | ssl, 61 | ca, 62 | }: ITransformedKafkaClusterConfig): Pick { 63 | if (!ssl && !ca) { 64 | return { 65 | ssl: false, 66 | }; 67 | } 68 | 69 | if (!ca) { 70 | return { 71 | ssl: true, 72 | }; 73 | } 74 | 75 | return { 76 | ssl: { 77 | rejectUnauthorized: true, 78 | requestCert: true, 79 | ca, 80 | }, 81 | }; 82 | } 83 | 84 | private static getKafkaSaslConfig({ 85 | saslMechanism, 86 | username, 87 | password, 88 | }: ITransformedKafkaClusterConfig): Pick { 89 | return !saslMechanism || !username || !password 90 | ? {} 91 | : { 92 | sasl: { 93 | mechanism: saslMechanism as "plain", 94 | username, 95 | password, 96 | }, 97 | }; 98 | } 99 | 100 | @Transform(({ value }: { value: unknown }) => 101 | typeof value === "string" ? value.split(",") : value, 102 | ) 103 | @IsArray() 104 | @ArrayMinSize(1) 105 | @IsString({ each: true }) 106 | public readonly brokers!: string | string[]; 107 | 108 | @IsIn(["plain", "scram-sha-256", "scram-sha-512"]) 109 | @IsOptional() 110 | public readonly saslMechanism?: string | TSupportedSaslMechanism; 111 | 112 | @IsString() 113 | @IsOptional() 114 | public readonly username?: string; 115 | 116 | @IsString() 117 | @IsOptional() 118 | public readonly password?: string; 119 | 120 | @TransformToBoolean() 121 | @IsBoolean() 122 | @IsOptional() 123 | public readonly ssl?: string | boolean; 124 | 125 | @Transform(({ value }: { value: unknown }) => 126 | typeof value === "string" ? Buffer.from(value, "base64") : value, 127 | ) 128 | @IsOptional() 129 | public readonly ca?: string | Buffer; 130 | 131 | @IsInt() 132 | @IsOptional() 133 | @TransformToNumber() 134 | public readonly connectionTimeout?: number; 135 | } 136 | -------------------------------------------------------------------------------- /src/config/kafkaConsumerConfigDto.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | TransformToBoolean, 19 | TransformToNumber, 20 | } from "@byndyusoft/class-validator-extended"; 21 | import { IsBoolean, IsInt, IsOptional, IsString } from "class-validator"; 22 | import { ConsumerConfig } from "kafkajs"; 23 | 24 | export class KafkaConsumerConfigDto { 25 | public static toRawConfig(config: KafkaConsumerConfigDto): ConsumerConfig { 26 | return { 27 | groupId: config.groupId, 28 | allowAutoTopicCreation: config.allowAutoTopicCreation as boolean, 29 | sessionTimeout: config.sessionTimeout as number, 30 | heartbeatInterval: config.heartbeatInterval as number, 31 | }; 32 | } 33 | 34 | @IsString() 35 | public readonly groupId!: string; 36 | 37 | @TransformToBoolean() 38 | @IsBoolean() 39 | @IsOptional() 40 | public readonly allowAutoTopicCreation?: string | boolean; 41 | 42 | @IsInt() 43 | @IsOptional() 44 | @TransformToNumber() 45 | public readonly sessionTimeout?: string | number; 46 | 47 | @IsInt() 48 | @IsOptional() 49 | @TransformToNumber() 50 | public readonly heartbeatInterval?: string | number; 51 | } 52 | -------------------------------------------------------------------------------- /src/config/kafkaProducerConfigDto.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { TransformToBoolean } from "@byndyusoft/class-validator-extended"; 18 | import { IsBoolean, IsOptional } from "class-validator"; 19 | import { ProducerConfig } from "kafkajs"; 20 | 21 | export class KafkaProducerConfigDto { 22 | public static toRawConfig(config: KafkaProducerConfigDto): ProducerConfig { 23 | return { 24 | allowAutoTopicCreation: config.allowAutoTopicCreation as boolean, 25 | }; 26 | } 27 | 28 | @TransformToBoolean() 29 | @IsBoolean() 30 | @IsOptional() 31 | public readonly allowAutoTopicCreation?: string | boolean; 32 | } 33 | -------------------------------------------------------------------------------- /src/config/kafkaSchemaRegistryArgsConfigDto.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { SchemaRegistryAPIClientArgs } from "@kafkajs/confluent-schema-registry/dist/api"; 18 | import { IsOptional, IsString } from "class-validator"; 19 | 20 | export class KafkaSchemaRegistryArgsConfigDto { 21 | public static toRawConfig( 22 | config: KafkaSchemaRegistryArgsConfigDto, 23 | ): SchemaRegistryAPIClientArgs { 24 | return { 25 | host: config.host, 26 | auth: 27 | config.username && config.password 28 | ? { 29 | username: config.username, 30 | password: config.password, 31 | } 32 | : undefined, 33 | }; 34 | } 35 | 36 | @IsString() 37 | public readonly host!: string; 38 | 39 | @IsString() 40 | @IsOptional() 41 | public readonly username?: string; 42 | 43 | @IsString() 44 | @IsOptional() 45 | public readonly password?: string; 46 | } 47 | -------------------------------------------------------------------------------- /src/consts/consumersMapToken.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const ConsumersMapToken = Symbol("ConsumersMapToken"); 18 | -------------------------------------------------------------------------------- /src/consts/defaultConnectionName.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const DefaultConnectionName = "default"; 18 | -------------------------------------------------------------------------------- /src/consts/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./consumersMapToken"; 18 | export * from "./defaultConnectionName"; 19 | export * from "./kafkaBaseOptionsToken"; 20 | export * from "./kafkaConsumerTransportId"; 21 | export * from "./kafkaOptionsToken"; 22 | export * from "./kafkaRetryConsumerTransportId"; 23 | export * from "./producersMapToken"; 24 | export * from "./schemaRegistriesMapToken"; 25 | -------------------------------------------------------------------------------- /src/consts/kafkaBaseOptionsToken.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const KafkaBaseOptionsToken = Symbol("KafkaBaseOptionsToken"); 18 | -------------------------------------------------------------------------------- /src/consts/kafkaConsumerTransportId.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const KafkaConsumerTransportId = Symbol("KafkaConsumerTransportId"); 18 | -------------------------------------------------------------------------------- /src/consts/kafkaOptionsToken.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const KafkaOptionsToken = Symbol("KafkaOptionsToken"); 18 | -------------------------------------------------------------------------------- /src/consts/kafkaRetryConsumerTransportId.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const KafkaRetryConsumerTransportId = Symbol( 18 | "KafkaRetryConsumerTransportId", 19 | ); 20 | -------------------------------------------------------------------------------- /src/consts/producersMapToken.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const ProducersMapToken = Symbol("ProducersMapToken"); 18 | -------------------------------------------------------------------------------- /src/consts/schemaRegistriesMapToken.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const SchemaRegistriesMapToken = Symbol("SchemaRegistriesMapToken"); 18 | -------------------------------------------------------------------------------- /src/consumer/decorators/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaConsumerEventPatternDecorator"; 18 | export * from "./kafkaHeadersDecorator"; 19 | export * from "./kafkaKeyDecorator"; 20 | export * from "./kafkaRetryConsumerEventPatternDecorator"; 21 | export * from "./kafkaValueDecorator"; 22 | -------------------------------------------------------------------------------- /src/consumer/decorators/kafkaConsumerEventPatternDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { EventPattern } from "@nestjs/microservices"; 18 | 19 | import { DefaultConnectionName, KafkaConsumerTransportId } from "../../consts"; 20 | import { 21 | IKafkaConsumerOptions, 22 | IKafkaConsumerSerializedOptions, 23 | } from "../interfaces"; 24 | 25 | export function KafkaConsumerEventPattern( 26 | options: IKafkaConsumerOptions, 27 | ): MethodDecorator { 28 | const serializedOptions: IKafkaConsumerSerializedOptions = { 29 | connectionName: options.connectionName ?? DefaultConnectionName, 30 | topicPicker: options.topicPicker.toString(), 31 | fromBeginning: options.fromBeginning, 32 | }; 33 | 34 | return EventPattern(serializedOptions, KafkaConsumerTransportId); 35 | } 36 | -------------------------------------------------------------------------------- /src/consumer/decorators/kafkaHeadersDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createParamDecorator, ExecutionContext } from "@nestjs/common"; 18 | 19 | import { IKafkaConsumerPayload } from "../interfaces"; 20 | 21 | export const KafkaHeaders = createParamDecorator( 22 | (property: string | undefined, context: ExecutionContext) => { 23 | const rpcHost = context.switchToRpc(); 24 | const payload: IKafkaConsumerPayload = rpcHost.getData(); 25 | 26 | return property ? payload.headers[property] : payload.headers; 27 | }, 28 | ); 29 | -------------------------------------------------------------------------------- /src/consumer/decorators/kafkaKeyDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createParamDecorator, ExecutionContext } from "@nestjs/common"; 18 | 19 | import { IKafkaConsumerPayload } from "../interfaces"; 20 | 21 | export const KafkaKey = createParamDecorator( 22 | (_data: never, context: ExecutionContext) => { 23 | const rpcHost = context.switchToRpc(); 24 | const payload: IKafkaConsumerPayload = rpcHost.getData(); 25 | 26 | return payload.key; 27 | }, 28 | ); 29 | -------------------------------------------------------------------------------- /src/consumer/decorators/kafkaRetryConsumerEventPatternDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import crypto from "crypto"; 18 | 19 | import { EventPattern } from "@nestjs/microservices"; 20 | 21 | import { KafkaRetryConsumerTransportId } from "../../consts"; 22 | import { IKafkaConsumerOptions } from "../interfaces"; 23 | 24 | export function KafkaRetryConsumerEventPattern( 25 | options: IKafkaConsumerOptions, 26 | ): MethodDecorator { 27 | return EventPattern( 28 | crypto.randomUUID(), // we don't use this arg, but we can't leave it empty 29 | KafkaRetryConsumerTransportId, 30 | options, 31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /src/consumer/decorators/kafkaValueDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createParamDecorator, ExecutionContext } from "@nestjs/common"; 18 | 19 | import { IKafkaConsumerPayload } from "../interfaces"; 20 | 21 | export const KafkaValue = createParamDecorator( 22 | (_data: never, context: ExecutionContext) => { 23 | const rpcHost = context.switchToRpc(); 24 | const payload: IKafkaConsumerPayload = rpcHost.getData(); 25 | 26 | return payload.value; 27 | }, 28 | ); 29 | -------------------------------------------------------------------------------- /src/consumer/errors/getErrorCause.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { KafkaConsumerError } from "./kafkaConsumerError"; 18 | 19 | export function getErrorCause(error: unknown): Error { 20 | return (error instanceof KafkaConsumerError ? error.cause : error) as Error; 21 | } 22 | -------------------------------------------------------------------------------- /src/consumer/errors/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./getErrorCause"; 18 | export * from "./kafkaConsumerError"; 19 | export * from "./kafkaConsumerNonRetriableError"; 20 | export * from "./kafkaConsumerRetriableError"; 21 | export * from "./serializeError"; 22 | -------------------------------------------------------------------------------- /src/consumer/errors/kafkaConsumerError.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import "error-cause/auto"; 18 | 19 | export abstract class KafkaConsumerError extends Error { 20 | public readonly retriable: boolean; 21 | 22 | protected constructor( 23 | message: string, 24 | options: { 25 | retriable: boolean; 26 | cause: unknown; 27 | }, 28 | ) { 29 | super(message, { 30 | cause: options.cause, 31 | }); 32 | 33 | this.retriable = options.retriable; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/consumer/errors/kafkaConsumerNonRetriableError.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { KafkaConsumerError } from "./kafkaConsumerError"; 18 | 19 | export class KafkaConsumerNonRetriableError extends KafkaConsumerError { 20 | public constructor(cause: unknown) { 21 | super(KafkaConsumerNonRetriableError.name, { 22 | retriable: false, 23 | cause, 24 | }); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/consumer/errors/kafkaConsumerRetriableError.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { KafkaConsumerError } from "./kafkaConsumerError"; 18 | 19 | export class KafkaConsumerRetriableError extends KafkaConsumerError { 20 | public constructor(cause: unknown) { 21 | super(KafkaConsumerRetriableError.name, { 22 | retriable: true, 23 | cause, 24 | }); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/consumer/errors/serializeError.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export function serializeError(error: Error): Record { 18 | return { 19 | message: error.message, 20 | stack: error.stack, 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /src/consumer/exceptions/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaConsumerBaseExceptionFilter"; 18 | export * from "./kafkaConsumerErrorTopicExceptionFilter"; 19 | -------------------------------------------------------------------------------- /src/consumer/exceptions/kafkaConsumerBaseExceptionFilter.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Catch, RpcExceptionFilter } from "@nestjs/common"; 18 | import { Observable, throwError } from "rxjs"; 19 | 20 | import { KafkaConsumerError, KafkaConsumerNonRetriableError } from "../errors"; 21 | 22 | @Catch() 23 | export class KafkaConsumerBaseExceptionFilter implements RpcExceptionFilter { 24 | public catch(exception: unknown): Observable { 25 | return throwError(() => 26 | exception instanceof KafkaConsumerError 27 | ? exception 28 | : new KafkaConsumerNonRetriableError(exception), 29 | ); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/consumer/exceptions/kafkaConsumerErrorTopicExceptionFilter.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable @typescript-eslint/no-explicit-any */ 18 | 19 | import { 20 | ArgumentsHost, 21 | Catch, 22 | Logger, 23 | RpcExceptionFilter, 24 | } from "@nestjs/common"; 25 | import { IHeaders } from "kafkajs"; 26 | import { from, Observable, throwError } from "rxjs"; 27 | 28 | import { 29 | getErrorCause, 30 | KafkaConsumerError, 31 | KafkaConsumerNonRetriableError, 32 | KafkaConsumerRetriableError, 33 | serializeError, 34 | } from "../errors"; 35 | import { 36 | IKafkaConsumerContext, 37 | IKafkaConsumerErrorTopicExceptionFilterOptions, 38 | IKafkaConsumerPayload, 39 | } from "../interfaces"; 40 | import { DefaultRetryStrategy } from "../retryStrategies"; 41 | 42 | @Catch() 43 | export class KafkaConsumerErrorTopicExceptionFilter 44 | implements RpcExceptionFilter 45 | { 46 | private readonly errorTopicPicker: ((...args: any[]) => string) | false; 47 | 48 | private readonly logger = new Logger( 49 | KafkaConsumerErrorTopicExceptionFilter.name, 50 | ); 51 | 52 | public constructor( 53 | private readonly options: IKafkaConsumerErrorTopicExceptionFilterOptions, 54 | ) { 55 | const errorTopicPicker = options.errorTopicPicker ?? options.topicPicker; 56 | 57 | if (errorTopicPicker === undefined) { 58 | throw new Error("errorTopicPicker must be defined"); 59 | } 60 | 61 | this.errorTopicPicker = errorTopicPicker; 62 | } 63 | 64 | private static isExceptionRetriable(exception: unknown): boolean { 65 | return DefaultRetryStrategy.isRetriable(exception); 66 | } 67 | 68 | private static makeKafkaConsumerError( 69 | exception: unknown, 70 | ): KafkaConsumerError { 71 | if (exception instanceof KafkaConsumerError) { 72 | return exception; 73 | } 74 | 75 | return KafkaConsumerErrorTopicExceptionFilter.isExceptionRetriable( 76 | exception, 77 | ) 78 | ? new KafkaConsumerRetriableError(exception) 79 | : new KafkaConsumerNonRetriableError(exception); 80 | } 81 | 82 | public catch(exception: unknown, host: ArgumentsHost): Observable { 83 | const rpcHost = host.switchToRpc(); 84 | const context: IKafkaConsumerContext = rpcHost.getContext(); 85 | 86 | const kafkaConsumerError = 87 | KafkaConsumerErrorTopicExceptionFilter.makeKafkaConsumerError(exception); 88 | 89 | if (kafkaConsumerError.retriable && !context.isFinalAttempt) { 90 | return throwError(() => kafkaConsumerError); 91 | } 92 | 93 | context.kafkaConsumerMessageHandlerLogger.error(this.logger, exception); 94 | 95 | const topic = this.getTopicForResendMessage(context, kafkaConsumerError); 96 | 97 | if (topic === false) { 98 | return throwError(() => kafkaConsumerError); 99 | } 100 | 101 | return from( 102 | this.resendMessage( 103 | kafkaConsumerError, 104 | host, 105 | this.options.connectionName ?? context.connectionName, 106 | topic, 107 | this.options.resendHeadersPrefix, 108 | ), 109 | ); 110 | } 111 | 112 | private getTopicForResendMessage( 113 | context: IKafkaConsumerContext, 114 | kafkaConsumerError: KafkaConsumerError, 115 | ): string | false { 116 | const topic = 117 | kafkaConsumerError.retriable && 118 | this.options.retryTopicPicker !== undefined 119 | ? this.options.retryTopicPicker 120 | : this.errorTopicPicker; 121 | 122 | if (!topic) { 123 | return false; 124 | } 125 | 126 | // eslint-disable-next-line @typescript-eslint/no-unsafe-argument 127 | return topic(...context.kafkaOptions.topicPickerArgs); 128 | } 129 | 130 | private async resendMessage( 131 | kafkaConsumerError: KafkaConsumerError, 132 | host: ArgumentsHost, 133 | connectionName: string, 134 | topic: string, 135 | resendHeadersPrefix = "original", 136 | ): Promise { 137 | const rpcHost = host.switchToRpc(); 138 | const payload: IKafkaConsumerPayload = rpcHost.getData(); 139 | const context: IKafkaConsumerContext = rpcHost.getContext(); 140 | 141 | const cause = getErrorCause(kafkaConsumerError); 142 | 143 | this.logger.warn(`Send message to ${topic}`); 144 | 145 | const headers: IHeaders = { 146 | ...payload.rawPayload.message.headers, 147 | [`${resendHeadersPrefix}Topic`]: payload.rawPayload.topic, 148 | [`${resendHeadersPrefix}Partition`]: String(payload.rawPayload.partition), 149 | [`${resendHeadersPrefix}Offset`]: payload.rawPayload.message.offset, 150 | [`${resendHeadersPrefix}Timestamp`]: payload.rawPayload.message.timestamp, 151 | [`${resendHeadersPrefix}TraceId`]: context.traceId, 152 | error: JSON.stringify(serializeError(cause)), 153 | }; 154 | 155 | await context.kafkaCoreProducer.send(connectionName, { 156 | topic, 157 | messages: [ 158 | { 159 | key: payload.rawPayload.message.key, 160 | value: payload.rawPayload.message.value, 161 | headers, 162 | }, 163 | ], 164 | }); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/consumer/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./decorators"; 18 | export * from "./errors"; 19 | export * from "./exceptions"; 20 | export * from "./interceptors"; 21 | export * from "./interfaces"; 22 | export * from "./kafkaConsumer"; 23 | export * from "./kafkaConsumerMessageHandler"; 24 | export * from "./kafkaConsumerMessageHandlerLogger"; 25 | export * from "./kafkaRetryConsumer"; 26 | export * from "./retryStrategies"; 27 | -------------------------------------------------------------------------------- /src/consumer/interceptors/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaConsumerPayloadDecoder"; 18 | -------------------------------------------------------------------------------- /src/consumer/interceptors/kafkaConsumerPayloadDecoder.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | CallHandler, 19 | ExecutionContext, 20 | Injectable, 21 | NestInterceptor, 22 | } from "@nestjs/common"; 23 | import { IHeaders } from "kafkajs"; 24 | import { Observable } from "rxjs"; 25 | 26 | import { 27 | IKafkaConsumerContext, 28 | IKafkaConsumerPayload, 29 | IKafkaConsumerPayloadDecoderOptions, 30 | IKafkaConsumerPayloadHeaders, 31 | } from "../interfaces"; 32 | 33 | @Injectable() 34 | export class KafkaConsumerPayloadDecoder implements NestInterceptor { 35 | public constructor( 36 | private readonly options: IKafkaConsumerPayloadDecoderOptions, 37 | ) {} 38 | 39 | private static decodeHeaders( 40 | data?: IHeaders, 41 | decoder?: "string", 42 | ): IKafkaConsumerPayloadHeaders { 43 | if (!data || !decoder) { 44 | return {}; 45 | } 46 | 47 | return Object.fromEntries( 48 | Object.entries(data).map(([key, value]) => { 49 | const isArray = Array.isArray(value); 50 | 51 | const decodedValue = [value] 52 | .flat() 53 | .map((x) => (x as Buffer).toString()); 54 | 55 | return [key, isArray ? decodedValue : decodedValue[0]]; 56 | }), 57 | ); 58 | } 59 | 60 | public async intercept( 61 | executionContext: ExecutionContext, 62 | next: CallHandler, 63 | ): Promise> { 64 | const rpcHost = executionContext.switchToRpc(); 65 | const context: IKafkaConsumerContext = rpcHost.getContext(); 66 | const payload: IKafkaConsumerPayload = rpcHost.getData(); 67 | 68 | payload.key = await this.decodeKeyOrValue( 69 | context, 70 | payload.rawPayload.message.key, 71 | this.options.key, 72 | ); 73 | 74 | payload.value = await this.decodeKeyOrValue( 75 | context, 76 | payload.rawPayload.message.value, 77 | this.options.value, 78 | ); 79 | 80 | payload.headers = KafkaConsumerPayloadDecoder.decodeHeaders( 81 | payload.rawPayload.message.headers, 82 | this.options.headers, 83 | ); 84 | 85 | return next.handle(); 86 | } 87 | 88 | private async decodeKeyOrValue( 89 | { 90 | connectionName: contextConnectionName, 91 | kafkaCoreSchemaRegistry, 92 | }: IKafkaConsumerContext, 93 | data: Buffer | null, 94 | decoder?: "string" | "json" | "schemaRegistry", 95 | ): Promise { 96 | if (!data || !decoder) { 97 | return undefined; 98 | } 99 | 100 | const connectionName = this.options.connectionName ?? contextConnectionName; 101 | 102 | switch (decoder) { 103 | case "string": 104 | return data.toString(); 105 | case "json": 106 | return JSON.parse(data.toString()) as unknown; 107 | case "schemaRegistry": 108 | return kafkaCoreSchemaRegistry.decode(connectionName, data); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/consumer/interfaces/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaConsumerContextInterface"; 18 | export * from "./kafkaConsumerErrorTopicExceptionFilterOptionsInterface"; 19 | export * from "./kafkaConsumerOptionsInterface"; 20 | export * from "./kafkaConsumerPayloadDecoderOptionsInterface"; 21 | export * from "./kafkaConsumerPayloadHeadersInterface"; 22 | export * from "./kafkaConsumerPayloadInterface"; 23 | export * from "./kafkaConsumerSerializedOptionsInterface"; 24 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerContextInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { IKafkaOptions } from "../../options"; 18 | import { KafkaCoreProducer } from "../../producer"; 19 | import { KafkaCoreSchemaRegistry } from "../../schemaRegistry"; 20 | import { KafkaConsumerMessageHandlerLogger } from "../kafkaConsumerMessageHandlerLogger"; 21 | 22 | export interface IKafkaConsumerContext { 23 | readonly connectionName: string; 24 | readonly kafkaOptions: IKafkaOptions; 25 | 26 | readonly kafkaCoreProducer: KafkaCoreProducer; 27 | readonly kafkaCoreSchemaRegistry: KafkaCoreSchemaRegistry; 28 | 29 | readonly kafkaConsumerMessageHandlerLogger: KafkaConsumerMessageHandlerLogger; 30 | 31 | readonly isFinalAttempt: boolean; 32 | readonly traceId: string; 33 | } 34 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerErrorTopicExceptionFilterOptionsInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable @typescript-eslint/no-explicit-any */ 18 | 19 | export interface IKafkaConsumerErrorTopicExceptionFilterOptions { 20 | readonly connectionName?: string; 21 | 22 | readonly retryTopicPicker?: ((...args: any[]) => string) | false; 23 | 24 | readonly errorTopicPicker?: ((...args: any[]) => string) | false; 25 | 26 | /** 27 | * @deprecated Use errorTopicPicker instead 28 | */ 29 | readonly topicPicker?: ((...args: any[]) => string) | false; 30 | 31 | /** 32 | * @default "original" 33 | */ 34 | readonly resendHeadersPrefix?: string; 35 | } 36 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerOptionsInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable @typescript-eslint/no-explicit-any */ 18 | 19 | export interface IKafkaConsumerOptions { 20 | readonly connectionName?: string; 21 | 22 | readonly topicPicker: (...args: any[]) => string | string[]; 23 | 24 | readonly fromBeginning?: boolean; 25 | } 26 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerPayloadDecoderOptionsInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export interface IKafkaConsumerPayloadDecoderOptions { 18 | readonly connectionName?: string; 19 | 20 | readonly key?: "string" | "json" | "schemaRegistry"; 21 | readonly value?: "string" | "json" | "schemaRegistry"; 22 | readonly headers?: "string"; 23 | } 24 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerPayloadHeadersInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export interface IKafkaConsumerPayloadHeaders { 18 | readonly [key: string]: string | string[]; 19 | } 20 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerPayloadInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { EachMessagePayload } from "kafkajs"; 18 | 19 | import { IKafkaConsumerPayloadHeaders } from "./kafkaConsumerPayloadHeadersInterface"; 20 | 21 | export interface IKafkaConsumerPayload< 22 | TKey = unknown, 23 | TValue = unknown, 24 | THeaders = IKafkaConsumerPayloadHeaders, 25 | > { 26 | key: TKey; 27 | value: TValue; 28 | headers: THeaders; 29 | 30 | readonly rawPayload: EachMessagePayload; 31 | } 32 | -------------------------------------------------------------------------------- /src/consumer/interfaces/kafkaConsumerSerializedOptionsInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export interface IKafkaConsumerSerializedOptions { 18 | readonly connectionName: string; 19 | 20 | readonly topicPicker: string; 21 | 22 | readonly fromBeginning?: boolean; 23 | } 24 | -------------------------------------------------------------------------------- /src/consumer/kafkaConsumer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable no-await-in-loop */ 18 | 19 | import vm from "vm"; 20 | 21 | import { Inject, Injectable, OnApplicationBootstrap } from "@nestjs/common"; 22 | import { 23 | CustomTransportStrategy, 24 | MessageHandler, 25 | Server, 26 | } from "@nestjs/microservices"; 27 | import { Consumer } from "kafkajs"; 28 | 29 | import { 30 | ConsumersMapToken, 31 | KafkaConsumerTransportId, 32 | KafkaOptionsToken, 33 | } from "../consts"; 34 | import { IKafkaOptions } from "../options"; 35 | 36 | import { IKafkaConsumerSerializedOptions } from "./interfaces"; 37 | import { KafkaConsumerMessageHandler } from "./kafkaConsumerMessageHandler"; 38 | 39 | @Injectable() 40 | export class KafkaConsumer 41 | extends Server 42 | implements CustomTransportStrategy, OnApplicationBootstrap 43 | { 44 | public readonly transportId = KafkaConsumerTransportId; 45 | 46 | private applicationBootstrapped = false; 47 | 48 | private readonly messageHandlersMap: Map< 49 | string, 50 | Map 51 | > = new Map(); 52 | 53 | public constructor( 54 | @Inject(ConsumersMapToken) 55 | private readonly consumersMap: Map, 56 | private readonly kafkaConsumerMessageHandler: KafkaConsumerMessageHandler, 57 | @Inject(KafkaOptionsToken) 58 | private readonly kafkaOptions: IKafkaOptions, 59 | ) { 60 | super(); 61 | } 62 | 63 | public async close(): Promise { 64 | await Promise.all( 65 | [...this.consumersMap.values()].map((x) => x.disconnect()), 66 | ); 67 | } 68 | 69 | public listen(callback: (error?: unknown) => void): void { 70 | if (!this.applicationBootstrapped) { 71 | const errorMessage = 72 | "Application is not bootstrapped. " + 73 | "Ensure that you have called app.listen() " + 74 | "before calling app.startAllMicroservices(). " + 75 | "The listen method was invoked before the application finished bootstrapping."; 76 | 77 | this.logger.error(errorMessage); 78 | return callback(new Error(errorMessage)); 79 | } 80 | 81 | this.start().then(callback).catch(callback); 82 | } 83 | 84 | public onApplicationBootstrap(): void { 85 | this.applicationBootstrapped = true; 86 | } 87 | 88 | private addMessageHandlerToMap( 89 | connectionName: string, 90 | topics: string[], 91 | messageHandler: MessageHandler, 92 | ): void { 93 | if (!this.messageHandlersMap.has(connectionName)) { 94 | this.messageHandlersMap.set(connectionName, new Map()); 95 | } 96 | 97 | const connectionsMap = this.messageHandlersMap.get(connectionName)!; 98 | 99 | for (const topic of topics) { 100 | connectionsMap.set(topic, messageHandler); 101 | } 102 | } 103 | 104 | private getConsumer(connectionName: string): Consumer { 105 | const consumer = this.consumersMap.get(connectionName); 106 | 107 | if (!consumer) { 108 | throw new Error( 109 | `Consumer for connection name "${connectionName}" doesn't exists!`, 110 | ); 111 | } 112 | 113 | return consumer; 114 | } 115 | 116 | private async run(): Promise { 117 | for (const [connectionName, consumer] of this.consumersMap) { 118 | await consumer.run({ 119 | eachMessage: async (rawPayload) => { 120 | const messageHandler = this.messageHandlersMap 121 | .get(connectionName)! 122 | .get(rawPayload.topic)!; 123 | 124 | await this.kafkaConsumerMessageHandler.handleMessage( 125 | connectionName, 126 | messageHandler, 127 | rawPayload, 128 | ); 129 | }, 130 | }); 131 | } 132 | } 133 | 134 | private async start(): Promise { 135 | await Promise.all([...this.consumersMap.values()].map((x) => x.connect())); 136 | 137 | await this.subscribe(); 138 | await this.run(); 139 | } 140 | 141 | private async subscribe(): Promise { 142 | const context = vm.createContext({ 143 | topicPickerArgs: this.kafkaOptions.topicPickerArgs, 144 | }); 145 | 146 | for (const [ 147 | messagePattern, 148 | messageHandler, 149 | ] of this.getHandlers().entries()) { 150 | const { connectionName, topicPicker, fromBeginning } = JSON.parse( 151 | messagePattern, 152 | ) as IKafkaConsumerSerializedOptions; 153 | 154 | const topics = [ 155 | vm.runInContext(`(${topicPicker})(...topicPickerArgs)`, context) as 156 | | string 157 | | string[], 158 | ].flat(); 159 | 160 | await this.getConsumer(connectionName).subscribe({ 161 | topics, 162 | fromBeginning, 163 | }); 164 | 165 | this.addMessageHandlerToMap(connectionName, topics, messageHandler); 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/consumer/kafkaConsumerMessageHandler.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { TracingService } from "@byndyusoft/nest-opentracing"; 18 | import { AsyncContext as TracingAsyncContext } from "@byndyusoft/nest-opentracing/dist/async-context"; 19 | import { PinoLogger } from "@byndyusoft/nest-pino"; 20 | import { storage as loggerStorage, Store } from "@byndyusoft/nest-pino/storage"; 21 | import { Inject, Injectable } from "@nestjs/common"; 22 | import { MessageHandler } from "@nestjs/microservices"; 23 | import { EachMessagePayload } from "kafkajs"; 24 | import retry from "retry"; 25 | import { isObservable, lastValueFrom } from "rxjs"; 26 | 27 | import { KafkaOptionsToken } from "../consts"; 28 | import { IKafkaOptions } from "../options"; 29 | import { KafkaCoreProducer } from "../producer"; 30 | import { KafkaCoreSchemaRegistry } from "../schemaRegistry"; 31 | 32 | import { getErrorCause, KafkaConsumerError } from "./errors"; 33 | import { IKafkaConsumerContext, IKafkaConsumerPayload } from "./interfaces"; 34 | import { KafkaConsumerMessageHandlerLogger } from "./kafkaConsumerMessageHandlerLogger"; 35 | 36 | interface IProcessMessageOptions { 37 | readonly connectionName: string; 38 | readonly messageHandler: MessageHandler< 39 | IKafkaConsumerPayload, 40 | IKafkaConsumerContext, 41 | unknown 42 | >; 43 | readonly rawPayload: EachMessagePayload; 44 | readonly currentAttempt: number; 45 | readonly isFinalAttempt: boolean; 46 | } 47 | 48 | @Injectable() 49 | export class KafkaConsumerMessageHandler { 50 | private readonly totalAttempts: number; 51 | 52 | public constructor( 53 | private readonly kafkaConsumerMessageHandlerLogger: KafkaConsumerMessageHandlerLogger, 54 | private readonly kafkaCoreProducer: KafkaCoreProducer, 55 | private readonly kafkaCoreSchemaRegistry: KafkaCoreSchemaRegistry, 56 | @Inject(KafkaOptionsToken) 57 | private readonly kafkaOptions: IKafkaOptions, 58 | private readonly logger: PinoLogger, 59 | private readonly tracingAsyncContext: TracingAsyncContext, 60 | private readonly tracingService: TracingService, 61 | ) { 62 | this.logger.setContext(KafkaConsumerMessageHandler.name); 63 | 64 | this.totalAttempts = 65 | (this.kafkaOptions.consumerRetryOptions?.retries ?? 10) + 1; 66 | } 67 | 68 | public handleMessage( 69 | connectionName: string, 70 | messageHandler: MessageHandler, 71 | rawPayload: EachMessagePayload, 72 | ): Promise { 73 | return new Promise((resolve, reject) => { 74 | const operation = retry.operation(this.kafkaOptions.consumerRetryOptions); 75 | 76 | operation.attempt((currentAttempt) => { 77 | this.processMessage({ 78 | connectionName, 79 | messageHandler, 80 | rawPayload, 81 | currentAttempt, 82 | isFinalAttempt: currentAttempt === this.totalAttempts, 83 | }) 84 | .then(resolve) 85 | .catch((error) => { 86 | if (error instanceof KafkaConsumerError && error.retriable) { 87 | if (!operation.retry(error.cause as Error)) { 88 | reject(operation.mainError()); 89 | } 90 | 91 | return; 92 | } 93 | 94 | operation.stop(); 95 | reject(getErrorCause(error)); 96 | }); 97 | }); 98 | }); 99 | } 100 | 101 | private prepareAsyncContext( 102 | { 103 | connectionName, 104 | rawPayload, 105 | currentAttempt, 106 | isFinalAttempt, 107 | }: IProcessMessageOptions, 108 | callback: () => Promise, 109 | ): Promise { 110 | return new Promise((resolve, reject) => { 111 | this.tracingAsyncContext.run(() => { 112 | const rootSpan = this.tracingService.initRootSpan( 113 | `${rawPayload.topic} [${connectionName}]`, 114 | ); 115 | 116 | const tags = { 117 | connectionName, 118 | topic: rawPayload.topic, 119 | partition: rawPayload.partition, 120 | offset: rawPayload.message.offset, 121 | currentAttempt, 122 | isFinalAttempt, 123 | }; 124 | 125 | rootSpan.addTags(tags); 126 | 127 | const rawLogger = this.logger.logger.child({ 128 | ...tags, 129 | traceId: rootSpan.context().toTraceId(), 130 | }); 131 | 132 | loggerStorage.run(new Store(rawLogger), () => { 133 | // eslint-disable-next-line n/callback-return 134 | callback().then(resolve).catch(reject); 135 | }); 136 | }); 137 | }); 138 | } 139 | 140 | private processMessage(options: IProcessMessageOptions): Promise { 141 | return this.prepareAsyncContext(options, () => 142 | this.processMessageInAsyncContext(options), 143 | ); 144 | } 145 | 146 | private async processMessageInAsyncContext({ 147 | connectionName, 148 | messageHandler, 149 | rawPayload, 150 | isFinalAttempt, 151 | }: IProcessMessageOptions): Promise { 152 | const rootSpan = this.tracingService.getRootSpan(); 153 | 154 | try { 155 | const payload: IKafkaConsumerPayload = { 156 | key: undefined, 157 | value: undefined, 158 | headers: {}, 159 | rawPayload, 160 | }; 161 | 162 | const context: IKafkaConsumerContext = { 163 | connectionName, 164 | kafkaOptions: this.kafkaOptions, 165 | kafkaCoreProducer: this.kafkaCoreProducer, 166 | kafkaCoreSchemaRegistry: this.kafkaCoreSchemaRegistry, 167 | kafkaConsumerMessageHandlerLogger: 168 | this.kafkaConsumerMessageHandlerLogger, 169 | isFinalAttempt, 170 | traceId: rootSpan.context().toTraceId(), 171 | }; 172 | 173 | const resultOrStream = await messageHandler(payload, context); 174 | 175 | if (isObservable(resultOrStream)) { 176 | await lastValueFrom(resultOrStream); 177 | } 178 | } catch (error) { 179 | this.kafkaConsumerMessageHandlerLogger.error(this.logger, error); 180 | 181 | throw error; 182 | } finally { 183 | rootSpan.finish(); 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/consumer/kafkaConsumerMessageHandlerLogger.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Tags, TracingService } from "@byndyusoft/nest-opentracing"; 18 | import { PinoLogger } from "@byndyusoft/nest-pino"; 19 | import { Injectable, LoggerService } from "@nestjs/common"; 20 | 21 | import { getErrorCause, serializeError } from "./errors"; 22 | 23 | @Injectable() 24 | export class KafkaConsumerMessageHandlerLogger { 25 | public constructor(private readonly tracingService: TracingService) {} 26 | 27 | public error(logger: LoggerService | PinoLogger, error: unknown): void { 28 | const rootSpan = this.tracingService.getRootSpan(); 29 | 30 | const cause = getErrorCause(error); 31 | 32 | logger.error(cause); 33 | 34 | rootSpan.setTag(Tags.ERROR, true); 35 | rootSpan.setTag(Tags.SAMPLING_PRIORITY, 1); 36 | 37 | rootSpan.log({ 38 | event: "error", 39 | ...serializeError(cause), 40 | }); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/consumer/kafkaRetryConsumer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import vm from "vm"; 18 | 19 | import { Inject, Injectable } from "@nestjs/common"; 20 | import { 21 | CustomTransportStrategy, 22 | MessageHandler, 23 | Server, 24 | } from "@nestjs/microservices"; 25 | import { Kafka } from "kafkajs"; 26 | 27 | import { 28 | DefaultConnectionName, 29 | KafkaOptionsToken, 30 | KafkaRetryConsumerTransportId, 31 | } from "../consts"; 32 | import { IKafkaConnection, IKafkaOptions } from "../options"; 33 | 34 | import { IKafkaConsumerOptions } from "./interfaces"; 35 | import { KafkaConsumerMessageHandler } from "./kafkaConsumerMessageHandler"; 36 | 37 | export interface IKafkaRetryConsumerRunOnceOptions { 38 | readonly connectionName?: string; 39 | readonly topic: string; 40 | readonly messagesCount: number; 41 | } 42 | 43 | @Injectable() 44 | export class KafkaRetryConsumer 45 | extends Server 46 | implements CustomTransportStrategy 47 | { 48 | public readonly transportId = KafkaRetryConsumerTransportId; 49 | 50 | private readonly connectionOptionsMap: Map; 51 | 52 | private readonly messageHandlersMap: Map< 53 | string, 54 | Map 55 | > = new Map(); 56 | 57 | public constructor( 58 | private readonly kafkaConsumerMessageHandler: KafkaConsumerMessageHandler, 59 | @Inject(KafkaOptionsToken) 60 | private readonly kafkaOptions: IKafkaOptions, 61 | ) { 62 | super(); 63 | 64 | this.connectionOptionsMap = new Map( 65 | kafkaOptions.connections 66 | .filter((x) => x.retryConsumer) 67 | .map((x) => [x.name ?? DefaultConnectionName, x]), 68 | ); 69 | } 70 | 71 | public close(): Promise { 72 | return Promise.resolve(); 73 | } 74 | 75 | public listen(callback: (error?: unknown) => void): void { 76 | try { 77 | this.init(); 78 | return callback(); 79 | } catch (error) { 80 | return callback(error); 81 | } 82 | } 83 | 84 | public async runOnce({ 85 | connectionName = DefaultConnectionName, 86 | topic, 87 | messagesCount: initialMessagesCount, 88 | }: IKafkaRetryConsumerRunOnceOptions): Promise { 89 | if (initialMessagesCount <= 0) { 90 | throw new Error("messagesCount must be greater than zero!"); 91 | } 92 | 93 | const connectionOptions = this.getConnectionOptions(connectionName); 94 | 95 | const { 96 | messageHandler, 97 | extras: { fromBeginning }, 98 | } = this.getMessageHandlerAndExtras(connectionName, topic); 99 | 100 | const messagesCount = await this.calculateMessagesCount( 101 | topic, 102 | initialMessagesCount, 103 | connectionOptions, 104 | ); 105 | 106 | if (messagesCount === 0) { 107 | return; 108 | } 109 | 110 | const retryConsumer = new Kafka(connectionOptions.cluster).consumer( 111 | connectionOptions.retryConsumer!, 112 | ); 113 | 114 | await retryConsumer.connect(); 115 | 116 | let processedMessagesCount = 0; 117 | 118 | try { 119 | await retryConsumer.subscribe({ 120 | topic, 121 | fromBeginning, 122 | }); 123 | 124 | await new Promise((resolve, reject) => { 125 | retryConsumer 126 | .run({ 127 | autoCommit: false, 128 | eachMessage: async (rawPayload) => { 129 | try { 130 | await this.kafkaConsumerMessageHandler.handleMessage( 131 | connectionName, 132 | messageHandler, 133 | rawPayload, 134 | ); 135 | 136 | await retryConsumer.commitOffsets([ 137 | { 138 | topic, 139 | partition: rawPayload.partition, 140 | offset: (BigInt(rawPayload.message.offset) + 1n).toString(), 141 | }, 142 | ]); 143 | 144 | processedMessagesCount++; 145 | 146 | if (processedMessagesCount === messagesCount) { 147 | retryConsumer.pause([{ topic }]); 148 | resolve(); 149 | } 150 | } catch (error) { 151 | retryConsumer.pause([{ topic }]); 152 | reject(error); 153 | } 154 | }, 155 | }) 156 | .catch(reject); 157 | }); 158 | } finally { 159 | await retryConsumer.disconnect(); 160 | } 161 | } 162 | 163 | private addMessageHandlerToMap( 164 | connectionName: string, 165 | topics: string[], 166 | messageHandler: MessageHandler, 167 | ): void { 168 | if (!this.messageHandlersMap.has(connectionName)) { 169 | this.messageHandlersMap.set(connectionName, new Map()); 170 | } 171 | 172 | const connectionsMap = this.messageHandlersMap.get(connectionName)!; 173 | 174 | for (const topic of topics) { 175 | connectionsMap.set(topic, messageHandler); 176 | } 177 | } 178 | 179 | private async calculateMessagesCount( 180 | topic: string, 181 | initialMessagesCount: number, 182 | connectionOptions: IKafkaConnection, 183 | ): Promise { 184 | let overallLag = 0n; 185 | 186 | const admin = new Kafka(connectionOptions.cluster).admin(); 187 | 188 | await admin.connect(); 189 | 190 | try { 191 | const [topicOffsets, [retryConsumerOffsets]] = await Promise.all([ 192 | admin.fetchTopicOffsets(topic), 193 | admin.fetchOffsets({ 194 | groupId: connectionOptions.retryConsumer!.groupId, 195 | topics: [topic], 196 | }), 197 | ]); 198 | 199 | for (const topicPartitionOffset of topicOffsets) { 200 | const retryConsumerPartitionOffset = BigInt( 201 | retryConsumerOffsets.partitions.find( 202 | (x) => x.partition === topicPartitionOffset.partition, 203 | )?.offset ?? 0n, 204 | ); 205 | 206 | overallLag += 207 | BigInt(topicPartitionOffset.high) - 208 | (retryConsumerPartitionOffset < 0n 209 | ? 0n 210 | : retryConsumerPartitionOffset); 211 | } 212 | } finally { 213 | await admin.disconnect(); 214 | } 215 | 216 | return Math.min(initialMessagesCount, Number(overallLag)); 217 | } 218 | 219 | private getConnectionOptions(connectionName: string): IKafkaConnection { 220 | const connectionOptions = this.connectionOptionsMap.get(connectionName); 221 | 222 | if (!connectionOptions) { 223 | throw new Error( 224 | `Connection options with name ${connectionName} doesn't exists!`, 225 | ); 226 | } 227 | 228 | return connectionOptions; 229 | } 230 | 231 | private getMessageHandlerAndExtras( 232 | connectionName: string, 233 | topic: string, 234 | ): { 235 | readonly messageHandler: MessageHandler; 236 | readonly extras: IKafkaConsumerOptions; 237 | } { 238 | const messageHandler = this.messageHandlersMap 239 | .get(connectionName) 240 | ?.get(topic); 241 | 242 | if (!messageHandler) { 243 | throw new Error( 244 | `messageHandler for connection name ${connectionName} and topic ${topic} doesn't exists!`, 245 | ); 246 | } 247 | 248 | return { 249 | messageHandler, 250 | extras: messageHandler.extras as IKafkaConsumerOptions, 251 | }; 252 | } 253 | 254 | private init(): void { 255 | const vmContext = vm.createContext({ 256 | topicPickerArgs: this.kafkaOptions.topicPickerArgs, 257 | }); 258 | 259 | for (const [, messageHandler] of this.getHandlers().entries()) { 260 | const { connectionName = DefaultConnectionName, topicPicker } = 261 | messageHandler.extras as IKafkaConsumerOptions; 262 | 263 | const topics = [ 264 | vm.runInContext( 265 | `(${topicPicker.toString()})(...topicPickerArgs)`, 266 | vmContext, 267 | ) as string | string[], 268 | ].flat(); 269 | 270 | this.addMessageHandlerToMap(connectionName, topics, messageHandler); 271 | } 272 | } 273 | } 274 | -------------------------------------------------------------------------------- /src/consumer/retryStrategies/axiosRetryStrategy.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { HttpStatus } from "@nestjs/common"; 18 | import axios from "axios"; 19 | 20 | export class AxiosRetryStrategy { 21 | // See https://stackoverflow.com/questions/51770071/what-are-the-http-codes-to-automatically-retry-the-request 22 | private static readonly retriableStatuses = new Set([ 23 | // 4xx 24 | HttpStatus.REQUEST_TIMEOUT, 25 | HttpStatus.TOO_MANY_REQUESTS, 26 | // 5xx 27 | HttpStatus.INTERNAL_SERVER_ERROR, // e.g. when we can't connect to DB 28 | HttpStatus.BAD_GATEWAY, 29 | HttpStatus.SERVICE_UNAVAILABLE, 30 | HttpStatus.GATEWAY_TIMEOUT, 31 | ]); 32 | 33 | public static isRetriable(error: unknown): boolean { 34 | if (!axios.isAxiosError(error)) { 35 | return false; 36 | } 37 | 38 | const status = error.response?.status; 39 | 40 | if (!status) { 41 | return false; 42 | } 43 | 44 | return AxiosRetryStrategy.retriableStatuses.has(status); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/consumer/retryStrategies/defaultRetryStrategy.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { AxiosRetryStrategy } from "./axiosRetryStrategy"; 18 | import { NetworkRetryStrategy } from "./networkRetryStrategy"; 19 | 20 | export class DefaultRetryStrategy { 21 | private static readonly retryStrategies = [ 22 | NetworkRetryStrategy, 23 | AxiosRetryStrategy, 24 | ]; 25 | 26 | public static isRetriable(error: unknown): boolean { 27 | return DefaultRetryStrategy.retryStrategies.some((x) => 28 | x.isRetriable(error), 29 | ); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/consumer/retryStrategies/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./axiosRetryStrategy"; 18 | export * from "./defaultRetryStrategy"; 19 | export * from "./networkRetryStrategy"; 20 | -------------------------------------------------------------------------------- /src/consumer/retryStrategies/networkRetryStrategy.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export class NetworkRetryStrategy { 18 | // See https://github.com/FGRibreau/node-request-retry/blob/master/strategies/NetworkError.js 19 | // and ECONNABORTED for axios request timeout, see https://github.com/axios/axios/issues/1543 20 | private static readonly retriableCodes = new Set([ 21 | "ECONNRESET", 22 | "ENOTFOUND", 23 | "ESOCKETTIMEDOUT", 24 | "ETIMEDOUT", 25 | "ECONNABORTED", 26 | "ECONNREFUSED", 27 | "EHOSTUNREACH", 28 | "EPIPE", 29 | "EAI_AGAIN", 30 | "EBUSY", 31 | ]); 32 | 33 | public static isRetriable(error: unknown): boolean { 34 | if (!error) { 35 | return false; 36 | } 37 | 38 | return NetworkRetryStrategy.retriableCodes.has( 39 | (error as { code: string }).code, 40 | ); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/decoratedProviders/decoratedProviderInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export interface IDecoratedProvider { 18 | connectionName: string; 19 | } 20 | -------------------------------------------------------------------------------- /src/decoratedProviders/decoratedProviders.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Provider, Type } from "@nestjs/common"; 18 | 19 | import { IDecoratedProvider } from "./decoratedProviderInterface"; 20 | 21 | export class DecoratedProviders { 22 | private readonly providersMap: Map = new Map(); 23 | 24 | public constructor( 25 | private readonly decoratedProviderClass: Type, 26 | ) {} 27 | 28 | public createProviders(): Provider[] { 29 | return [...this.providersMap.entries()].map(([connectionName, token]) => ({ 30 | provide: token, 31 | inject: [this.decoratedProviderClass], 32 | useFactory(decoratedProvider: IDecoratedProvider) { 33 | decoratedProvider.connectionName = connectionName; 34 | 35 | return decoratedProvider; 36 | }, 37 | })); 38 | } 39 | 40 | public getToken(connectionName: string): symbol { 41 | if (!this.providersMap.has(connectionName)) { 42 | this.providersMap.set( 43 | connectionName, 44 | Symbol(`${this.decoratedProviderClass.name}-${connectionName}`), 45 | ); 46 | } 47 | 48 | return this.providersMap.get(connectionName)!; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/decoratedProviders/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./decoratedProviderInterface"; 18 | export * from "./decoratedProviders"; 19 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./config"; 18 | export * from "./consts"; 19 | export * from "./consumer"; 20 | export * from "./decoratedProviders"; 21 | export * from "./kafkaModule"; 22 | export * from "./options"; 23 | export * from "./producer"; 24 | export * from "./schemaRegistry"; 25 | -------------------------------------------------------------------------------- /src/kafkaModule.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | DynamicModuleHelper, 19 | TRegisterAsyncOptions, 20 | } from "@byndyusoft/nest-dynamic-module"; 21 | import { SchemaRegistry } from "@kafkajs/confluent-schema-registry"; 22 | import { DynamicModule, Logger, Module } from "@nestjs/common"; 23 | import { Consumer, Kafka, KafkaConfig, logLevel, Producer } from "kafkajs"; 24 | import _ from "lodash"; 25 | 26 | import { 27 | ConsumersMapToken, 28 | DefaultConnectionName, 29 | KafkaBaseOptionsToken, 30 | KafkaOptionsToken, 31 | ProducersMapToken, 32 | SchemaRegistriesMapToken, 33 | } from "./consts"; 34 | import { 35 | KafkaConsumer, 36 | KafkaConsumerMessageHandler, 37 | KafkaConsumerMessageHandlerLogger, 38 | KafkaRetryConsumer, 39 | } from "./consumer"; 40 | import { IKafkaOptions } from "./options"; 41 | import { 42 | KafkaCoreProducer, 43 | KafkaProducer, 44 | kafkaProducerDecoratedProviders, 45 | } from "./producer"; 46 | import { 47 | KafkaCoreSchemaRegistry, 48 | KafkaSchemaRegistry, 49 | kafkaSchemaRegistryDecoratedProviders, 50 | } from "./schemaRegistry"; 51 | 52 | @Module({}) 53 | export class KafkaModule { 54 | public static registerAsync( 55 | options?: TRegisterAsyncOptions, 56 | ): DynamicModule { 57 | const providers = [ 58 | KafkaConsumer, 59 | KafkaConsumerMessageHandler, 60 | KafkaConsumerMessageHandlerLogger, 61 | KafkaCoreProducer, 62 | KafkaCoreSchemaRegistry, 63 | KafkaProducer, 64 | KafkaRetryConsumer, 65 | KafkaSchemaRegistry, 66 | ...kafkaProducerDecoratedProviders.createProviders(), 67 | ...kafkaSchemaRegistryDecoratedProviders.createProviders(), 68 | ]; 69 | 70 | return DynamicModuleHelper.registerAsync( 71 | { 72 | module: KafkaModule, 73 | global: true, 74 | providers: [ 75 | { 76 | provide: KafkaOptionsToken, 77 | inject: [KafkaBaseOptionsToken], 78 | useFactory: (kafkaOptions: IKafkaOptions) => 79 | KafkaModule.kafkaOptionsFactory(kafkaOptions), 80 | }, 81 | { 82 | provide: ConsumersMapToken, 83 | inject: [KafkaOptionsToken], 84 | useFactory: (kafkaOptions: IKafkaOptions) => 85 | KafkaModule.consumersMapTokenFactory(kafkaOptions), 86 | }, 87 | { 88 | provide: ProducersMapToken, 89 | inject: [KafkaOptionsToken], 90 | useFactory: (kafkaOptions: IKafkaOptions) => 91 | KafkaModule.producersMapTokenFactory(kafkaOptions), 92 | }, 93 | { 94 | provide: SchemaRegistriesMapToken, 95 | inject: [KafkaOptionsToken], 96 | useFactory: (kafkaOptions: IKafkaOptions) => 97 | KafkaModule.schemaRegistriesMapTokenFactory(kafkaOptions), 98 | }, 99 | ...providers, 100 | ], 101 | exports: providers, 102 | }, 103 | KafkaBaseOptionsToken, 104 | options, 105 | ); 106 | } 107 | 108 | private static consumersMapTokenFactory( 109 | kafkaOptions: IKafkaOptions, 110 | ): Map { 111 | return new Map( 112 | kafkaOptions.connections 113 | .filter((x) => x.consumer) 114 | .map((x) => [ 115 | x.name ?? DefaultConnectionName, 116 | new Kafka(x.cluster).consumer(x.consumer!), 117 | ]), 118 | ); 119 | } 120 | 121 | private static getKafkaLogger( 122 | connectionName: string, 123 | ): Pick { 124 | const logger = new Logger(`kafkajs-${connectionName}`); 125 | 126 | return { 127 | logLevel: logLevel.DEBUG, 128 | logCreator() { 129 | return (entry) => { 130 | if (entry.level === logLevel.NOTHING) { 131 | return; 132 | } 133 | 134 | let methodName: "debug" | "log" | "warn" | "error"; 135 | switch (entry.level) { 136 | case logLevel.DEBUG: 137 | methodName = "debug"; 138 | break; 139 | case logLevel.INFO: 140 | methodName = "log"; 141 | break; 142 | case logLevel.WARN: 143 | methodName = "warn"; 144 | break; 145 | case logLevel.ERROR: 146 | methodName = "error"; 147 | break; 148 | } 149 | 150 | logger[methodName]( 151 | _.omit( 152 | { 153 | namespace: entry.namespace, 154 | ...entry.log, 155 | }, 156 | "logger", 157 | "message", 158 | "timestamp", 159 | ), 160 | entry.log.message, 161 | ); 162 | }; 163 | }, 164 | }; 165 | } 166 | 167 | private static kafkaOptionsFactory( 168 | kafkaOptions: IKafkaOptions, 169 | ): IKafkaOptions { 170 | return { 171 | ...kafkaOptions, 172 | connections: kafkaOptions.connections.map((connection) => ({ 173 | ...connection, 174 | cluster: { 175 | ...connection.cluster, 176 | ...(!connection.cluster.logLevel && !connection.cluster.logCreator 177 | ? KafkaModule.getKafkaLogger( 178 | connection.name ?? DefaultConnectionName, 179 | ) 180 | : {}), 181 | }, 182 | })), 183 | consumerRetryOptions: kafkaOptions.consumerRetryOptions ?? { 184 | retries: 3, 185 | }, 186 | }; 187 | } 188 | 189 | private static producersMapTokenFactory( 190 | kafkaOptions: IKafkaOptions, 191 | ): Map { 192 | return new Map( 193 | kafkaOptions.connections 194 | .filter((x) => x.producer) 195 | .map((x) => [ 196 | x.name ?? DefaultConnectionName, 197 | new Kafka(x.cluster).producer(x.producer), 198 | ]), 199 | ); 200 | } 201 | 202 | private static schemaRegistriesMapTokenFactory( 203 | kafkaOptions: IKafkaOptions, 204 | ): Map { 205 | return new Map( 206 | kafkaOptions.connections 207 | .filter((x) => x.schemaRegistry) 208 | .map((x) => [ 209 | x.name ?? DefaultConnectionName, 210 | new SchemaRegistry(x.schemaRegistry!.args, x.schemaRegistry!.options), 211 | ]), 212 | ); 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/options/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./kafkaConnectionInterface"; 18 | export * from "./kafkaOptionsInterface"; 19 | export * from "./kafkaSchemaRegistryConnectionInterface"; 20 | -------------------------------------------------------------------------------- /src/options/kafkaConnectionInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { ConsumerConfig, KafkaConfig, ProducerConfig } from "kafkajs"; 18 | 19 | import { IKafkaSchemaRegistryConnection } from "./kafkaSchemaRegistryConnectionInterface"; 20 | 21 | export interface IKafkaConnection { 22 | readonly name?: string; 23 | 24 | readonly cluster: KafkaConfig; 25 | 26 | readonly consumer?: ConsumerConfig; 27 | 28 | readonly retryConsumer?: ConsumerConfig; 29 | 30 | readonly producer?: ProducerConfig; 31 | 32 | readonly schemaRegistry?: IKafkaSchemaRegistryConnection; 33 | } 34 | -------------------------------------------------------------------------------- /src/options/kafkaOptionsInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* eslint-disable @typescript-eslint/no-explicit-any */ 18 | 19 | import { WrapOptions } from "retry"; 20 | 21 | import { IKafkaConnection } from "./kafkaConnectionInterface"; 22 | 23 | export interface IKafkaOptions { 24 | readonly connections: IKafkaConnection[]; 25 | 26 | readonly topicPickerArgs: any[]; 27 | 28 | readonly consumerRetryOptions?: WrapOptions; 29 | } 30 | -------------------------------------------------------------------------------- /src/options/kafkaSchemaRegistryConnectionInterface.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { SchemaRegistryAPIClientOptions } from "@kafkajs/confluent-schema-registry/dist/@types"; 18 | import { SchemaRegistryAPIClientArgs } from "@kafkajs/confluent-schema-registry/dist/api"; 19 | 20 | export interface IKafkaSchemaRegistryConnection { 21 | readonly args: SchemaRegistryAPIClientArgs; 22 | 23 | readonly options?: SchemaRegistryAPIClientOptions; 24 | } 25 | -------------------------------------------------------------------------------- /src/producer/decorators/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./injectKafkaProducerDecorator"; 18 | export * from "./kafkaProducerDecoratedProviders"; 19 | -------------------------------------------------------------------------------- /src/producer/decorators/injectKafkaProducerDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Inject } from "@nestjs/common"; 18 | 19 | import { DefaultConnectionName } from "../../consts"; 20 | 21 | import { kafkaProducerDecoratedProviders } from "./kafkaProducerDecoratedProviders"; 22 | 23 | export function InjectKafkaProducer( 24 | connectionName: string = DefaultConnectionName, 25 | ): ParameterDecorator { 26 | return Inject(kafkaProducerDecoratedProviders.getToken(connectionName)); 27 | } 28 | -------------------------------------------------------------------------------- /src/producer/decorators/kafkaProducerDecoratedProviders.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { DecoratedProviders } from "../../decoratedProviders"; 18 | import { KafkaProducer } from "../kafkaProducer"; 19 | 20 | export const kafkaProducerDecoratedProviders = new DecoratedProviders( 21 | KafkaProducer, 22 | ); 23 | -------------------------------------------------------------------------------- /src/producer/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./decorators"; 18 | export * from "./kafkaCoreProducer"; 19 | export * from "./kafkaProducer"; 20 | -------------------------------------------------------------------------------- /src/producer/kafkaCoreProducer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | Inject, 19 | Injectable, 20 | OnApplicationShutdown, 21 | OnModuleInit, 22 | } from "@nestjs/common"; 23 | import { Producer } from "kafkajs"; 24 | 25 | import { ProducersMapToken } from "../consts"; 26 | 27 | @Injectable() 28 | export class KafkaCoreProducer implements OnModuleInit, OnApplicationShutdown { 29 | public constructor( 30 | @Inject(ProducersMapToken) 31 | private readonly producersMap: Map, 32 | ) {} 33 | 34 | public async onApplicationShutdown(): Promise { 35 | await Promise.all( 36 | [...this.producersMap.values()].map((x) => x.disconnect()), 37 | ); 38 | } 39 | 40 | public async onModuleInit(): Promise { 41 | await Promise.all([...this.producersMap.values()].map((x) => x.connect())); 42 | } 43 | 44 | public send( 45 | connectionName: string, 46 | ...args: Parameters 47 | ): ReturnType { 48 | return this.getProducer(connectionName).send(...args); 49 | } 50 | 51 | public sendBatch( 52 | connectionName: string, 53 | ...args: Parameters 54 | ): ReturnType { 55 | return this.getProducer(connectionName).sendBatch(...args); 56 | } 57 | 58 | private getProducer(connectionName: string): Producer { 59 | const producer = this.producersMap.get(connectionName); 60 | 61 | if (!producer) { 62 | throw new Error( 63 | `Producer for connection name "${connectionName}" doesn't exists!`, 64 | ); 65 | } 66 | 67 | return producer; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/producer/kafkaProducer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Injectable, Scope } from "@nestjs/common"; 18 | import { Producer } from "kafkajs"; 19 | 20 | import { IDecoratedProvider } from "../decoratedProviders"; 21 | 22 | import { KafkaCoreProducer } from "./kafkaCoreProducer"; 23 | 24 | @Injectable({ scope: Scope.TRANSIENT }) 25 | export class KafkaProducer implements IDecoratedProvider { 26 | private connectionNameValue?: string; 27 | 28 | public constructor(private readonly kafkaCoreProducer: KafkaCoreProducer) {} 29 | 30 | public get connectionName(): string { 31 | if (!this.connectionNameValue) { 32 | throw new Error(`"connectionName" in KafkaProducer must be initialized!`); 33 | } 34 | 35 | return this.connectionNameValue; 36 | } 37 | 38 | public set connectionName(value: string) { 39 | this.connectionNameValue = value; 40 | } 41 | 42 | public send( 43 | ...args: Parameters 44 | ): ReturnType { 45 | return this.kafkaCoreProducer.send(this.connectionName, ...args); 46 | } 47 | 48 | public sendBatch( 49 | ...args: Parameters 50 | ): ReturnType { 51 | return this.kafkaCoreProducer.sendBatch(this.connectionName, ...args); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/schemaRegistry/decorators/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./injectKafkaSchemaRegistryDecorator"; 18 | export * from "./kafkaSchemaRegistryDecoratedProviders"; 19 | -------------------------------------------------------------------------------- /src/schemaRegistry/decorators/injectKafkaSchemaRegistryDecorator.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { Inject } from "@nestjs/common"; 18 | 19 | import { DefaultConnectionName } from "../../consts"; 20 | 21 | import { kafkaSchemaRegistryDecoratedProviders } from "./kafkaSchemaRegistryDecoratedProviders"; 22 | 23 | export function InjectKafkaSchemaRegistry( 24 | connectionName: string = DefaultConnectionName, 25 | ): ParameterDecorator { 26 | return Inject(kafkaSchemaRegistryDecoratedProviders.getToken(connectionName)); 27 | } 28 | -------------------------------------------------------------------------------- /src/schemaRegistry/decorators/kafkaSchemaRegistryDecoratedProviders.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { DecoratedProviders } from "../../decoratedProviders"; 18 | import { KafkaSchemaRegistry } from "../kafkaSchemaRegistry"; 19 | 20 | export const kafkaSchemaRegistryDecoratedProviders = new DecoratedProviders( 21 | KafkaSchemaRegistry, 22 | ); 23 | -------------------------------------------------------------------------------- /src/schemaRegistry/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./decorators"; 18 | export * from "./kafkaCoreSchemaRegistry"; 19 | export * from "./kafkaSchemaRegistry"; 20 | -------------------------------------------------------------------------------- /src/schemaRegistry/kafkaCoreSchemaRegistry.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { SchemaRegistry } from "@kafkajs/confluent-schema-registry"; 18 | import { Inject, Injectable } from "@nestjs/common"; 19 | 20 | import { SchemaRegistriesMapToken } from "../consts"; 21 | 22 | @Injectable() 23 | export class KafkaCoreSchemaRegistry { 24 | public constructor( 25 | @Inject(SchemaRegistriesMapToken) 26 | private readonly schemaRegistriesMap: Map, 27 | ) {} 28 | 29 | public decode( 30 | connectionName: string, 31 | ...args: Parameters 32 | ): ReturnType { 33 | return this.getSchemaRegistry(connectionName).decode(...args); 34 | } 35 | 36 | public encode( 37 | connectionName: string, 38 | ...args: Parameters 39 | ): ReturnType { 40 | return this.getSchemaRegistry(connectionName).encode(...args); 41 | } 42 | 43 | public getLatestSchemaId( 44 | connectionName: string, 45 | ...args: Parameters 46 | ): ReturnType { 47 | return this.getSchemaRegistry(connectionName).getLatestSchemaId(...args); 48 | } 49 | 50 | public getSchema( 51 | connectionName: string, 52 | ...args: Parameters 53 | ): ReturnType { 54 | return this.getSchemaRegistry(connectionName).getSchema(...args); 55 | } 56 | 57 | private getSchemaRegistry(connectionName: string): SchemaRegistry { 58 | const schemaRegistry = this.schemaRegistriesMap.get(connectionName); 59 | 60 | if (!schemaRegistry) { 61 | throw new Error( 62 | `SchemaRegistry for connection name "${connectionName}" doesn't exists!`, 63 | ); 64 | } 65 | 66 | return schemaRegistry; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/schemaRegistry/kafkaSchemaRegistry.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { SchemaRegistry } from "@kafkajs/confluent-schema-registry"; 18 | import { Injectable, Scope } from "@nestjs/common"; 19 | 20 | import { IDecoratedProvider } from "../decoratedProviders"; 21 | 22 | import { KafkaCoreSchemaRegistry } from "./kafkaCoreSchemaRegistry"; 23 | 24 | @Injectable({ scope: Scope.TRANSIENT }) 25 | export class KafkaSchemaRegistry implements IDecoratedProvider { 26 | private connectionNameValue?: string; 27 | 28 | public constructor( 29 | private readonly kafkaCoreSchemaRegistry: KafkaCoreSchemaRegistry, 30 | ) {} 31 | 32 | public get connectionName(): string { 33 | if (!this.connectionNameValue) { 34 | throw new Error( 35 | `"connectionName" in KafkaSchemaRegistry must be initialized!`, 36 | ); 37 | } 38 | 39 | return this.connectionNameValue; 40 | } 41 | 42 | public set connectionName(value: string) { 43 | this.connectionNameValue = value; 44 | } 45 | 46 | public decode( 47 | ...args: Parameters 48 | ): ReturnType { 49 | return this.kafkaCoreSchemaRegistry.decode(this.connectionName, ...args); 50 | } 51 | 52 | public encode( 53 | ...args: Parameters 54 | ): ReturnType { 55 | return this.kafkaCoreSchemaRegistry.encode(this.connectionName, ...args); 56 | } 57 | 58 | public getLatestSchemaId( 59 | ...args: Parameters 60 | ): ReturnType { 61 | return this.kafkaCoreSchemaRegistry.getLatestSchemaId( 62 | this.connectionName, 63 | ...args, 64 | ); 65 | } 66 | 67 | public getSchema( 68 | ...args: Parameters 69 | ): ReturnType { 70 | return this.kafkaCoreSchemaRegistry.getSchema(this.connectionName, ...args); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/expect.d.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Byndyusoft 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import "jest-extended"; 18 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src"] 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@byndyusoft/tsconfig/tsconfig.node14.json", 3 | "compilerOptions": { 4 | // Emit 5 | "outDir": "./dist" 6 | }, 7 | "include": ["src", "test"] 8 | } 9 | --------------------------------------------------------------------------------