├── .env.sample ├── .github └── workflows │ ├── ci.yml │ ├── cleanup-neon-branch.yml │ ├── platformatic-deploy.yml │ └── platformatic-preview.yml ├── .gitignore ├── .npmrc ├── .taprc.yml ├── LICENSE ├── NOTICE ├── README.md ├── architecture.png ├── docker-compose-apple-silicon.yml ├── global.d.ts ├── lib ├── backoff.js ├── cron.js └── executor.js ├── migrations ├── 001.do.sql ├── 001.undo.sql ├── 002.do.sql ├── 002.undo.sql ├── 003.do.sql ├── 003.undo.sql ├── 004.do.sql ├── 004.undo.sql ├── 005.do.sql ├── 005.undo.sql ├── 006.do.sql ├── 006.undo.sql ├── 007.do.sql └── 007.undo.sql ├── package-lock.json ├── package.json ├── platformatic.db.json ├── plugin.js ├── renovate.json ├── target.mjs ├── test ├── backoff.test.js ├── cron.test.js ├── election.test.js ├── helper.js ├── parallelism.test.js ├── queue.test.js └── retries.test.js └── types ├── Cron.d.ts ├── Message.d.ts ├── Page.d.ts ├── Queue.d.ts └── index.d.ts /.env.sample: -------------------------------------------------------------------------------- 1 | PLT_SERVER_HOSTNAME=127.0.0.1 2 | PORT=3042 3 | PLT_SERVER_LOGGER_LEVEL=info 4 | DATABASE_URL=sqlite://./db.sqlite 5 | PLT_ADMIN_SECRET=THISISASECRET 6 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '*.md' 8 | pull_request: 9 | paths-ignore: 10 | - 'docs/**' 11 | - '*.md' 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [18.x] 20 | steps: 21 | - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 22 | 23 | - name: Use Node.js 24 | uses: actions/setup-node@v4 25 | with: 26 | node-version: ${{ matrix.node-version }} 27 | 28 | - uses: ikalnytskyi/action-setup-postgres@v4 29 | 30 | - name: Install 31 | run: | 32 | npm ci 33 | 34 | - name: Run tests 35 | run: | 36 | npm run test 37 | -------------------------------------------------------------------------------- /.github/workflows/cleanup-neon-branch.yml: -------------------------------------------------------------------------------- 1 | name: Cleanup Neon Database Branch 2 | on: 3 | push: 4 | branches: 5 | - 'main' 6 | jobs: 7 | delete-branch: 8 | environment: 9 | name: development 10 | permissions: write-all 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Get PR info 14 | id: get-pr-info 15 | uses: actions-ecosystem/action-get-merged-pull-request@v1.0.1 16 | with: 17 | github_token: ${{secrets.GITHUB_TOKEN}} 18 | - run: | 19 | echo ${{ steps.get-pr-info.outputs.number}} 20 | - name: Delete Neon Branch 21 | if: ${{ steps.get-pr-info.outputs.number }} 22 | uses: neondatabase/delete-branch-action@v3 23 | with: 24 | project_id: ${{ secrets.NEON_PROJECT_ID }} 25 | branch: pr-${{ steps.get-pr-info.outputs.number }} 26 | api_key: ${{ secrets.NEON_API_KEY }} 27 | -------------------------------------------------------------------------------- /.github/workflows/platformatic-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Platformatic application to the cloud 2 | on: 3 | push: 4 | branches: 5 | - 'main' 6 | 7 | # This allows a subsequently queued workflow run to interrupt previous runs 8 | concurrency: 9 | group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | build_and_deploy: 14 | runs-on: ubuntu-latest 15 | environment: 16 | name: main 17 | steps: 18 | - name: Checkout application project repository 19 | uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 20 | - name: npm install --omit=dev 21 | run: npm install --omit=dev 22 | - name: Deploy project 23 | uses: platformatic/onestep@latest 24 | with: 25 | github_token: ${{ secrets.GITHUB_TOKEN }} 26 | platformatic_workspace_id: ${{ secrets.PLATFORMATIC_STATIC_WORKSPACE_ID }} 27 | platformatic_workspace_key: ${{ secrets.PLATFORMATIC_STATIC_WORKSPACE_KEY }} 28 | platformatic_config_path: ./platformatic.db.json 29 | env: 30 | PLT_SERVER_LOGGER_LEVEL: info 31 | PORT: 3042 32 | PLT_SERVER_HOSTNAME: 127.0.0.1 33 | PLT_ADMIN_SECRET: ${{ secrets.PLT_ADMIN_SECRET }} 34 | PLT_LOCK: ${{ vars.PLT_LOCK }} 35 | PLT_LEADER_POLL: ${{ vars.PLT_LEADER_POLL }} 36 | DATABASE_URL: ${{ secrets.NEON_DB_URL }} 37 | -------------------------------------------------------------------------------- /.github/workflows/platformatic-preview.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Platformatic cloud 2 | on: 3 | pull_request: 4 | paths-ignore: 5 | - 'docs/**' 6 | - '**.md' 7 | 8 | # This allows a subsequently queued workflow run to interrupt previous runs 9 | concurrency: 10 | group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | build_and_deploy: 15 | runs-on: ubuntu-latest 16 | environment: 17 | name: development 18 | steps: 19 | - name: Checkout application project repository 20 | uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 21 | - name: npm install --omit=dev 22 | run: npm install --omit=dev 23 | - name: Get PR number 24 | id: get_pull_number 25 | run: | 26 | pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") 27 | echo "pull_number=${pull_number}" >> $GITHUB_OUTPUT 28 | echo $pull_number 29 | - uses: neondatabase/create-branch-action@v4 30 | with: 31 | project_id: ${{ secrets.NEON_PROJECT_ID }} 32 | branch_name: pr-${{ steps.get_pull_number.outputs.pull_number }} 33 | api_key: ${{ secrets.NEON_API_KEY }} 34 | id: create-branch 35 | - name: Deploy project 36 | uses: platformatic/onestep@latest 37 | with: 38 | github_token: ${{ secrets.GITHUB_TOKEN }} 39 | platformatic_workspace_id: ${{ secrets.PLATFORMATIC_DYNAMIC_WORKSPACE_ID }} 40 | platformatic_workspace_key: ${{ secrets.PLATFORMATIC_DYNAMIC_WORKSPACE_KEY }} 41 | platformatic_config_path: ./platformatic.db.json 42 | env: 43 | PLT_SERVER_LOGGER_LEVEL: info 44 | PORT: 3042 45 | PLT_SERVER_HOSTNAME: 127.0.0.1 46 | PLT_ADMIN_SECRET: ${{ secrets.PLT_ADMIN_SECRET }} 47 | PLT_LOCK: ${{ vars.PLT_LOCK }} 48 | PLT_LEADER_POLL: ${{ vars.PLT_LEADER_POLL }} 49 | DATABASE_URL: ${{ steps.create-branch.outputs.db_url }} 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | db.sqlite 107 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=true 2 | -------------------------------------------------------------------------------- /.taprc.yml: -------------------------------------------------------------------------------- 1 | jobs: 1 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Platformatic 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unscalable-queue-system 2 | 3 | Tiny Queue System that will call your service back 4 | after a given time. 5 | 6 | ![Architecture](./architecture.png) 7 | 8 | ## License 9 | 10 | Apache 2.0 11 | -------------------------------------------------------------------------------- /architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/platformatic/unscalable-queue-system/44bc81d5a89400ebfdc9c1c7d2cb64fbdb9d1e20/architecture.png -------------------------------------------------------------------------------- /docker-compose-apple-silicon.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | services: 3 | postgresql: 4 | ports: 5 | - "5432:5432" 6 | image: "arm64v8/postgres:16-alpine" 7 | environment: 8 | - POSTGRES_PASSWORD=postgres 9 | -------------------------------------------------------------------------------- /global.d.ts: -------------------------------------------------------------------------------- 1 | import type { PlatformaticApp, PlatformaticDBMixin, PlatformaticDBConfig, Entity, Entities, EntityHooks } from '@platformatic/db' 2 | import { EntityTypes, Cron,Message,Page,Queue } from './types' 3 | 4 | declare module 'fastify' { 5 | interface FastifyInstance { 6 | getSchema(schemaId: T): { 7 | '$id': string, 8 | title: string, 9 | description: string, 10 | type: string, 11 | properties: { 12 | [x in keyof EntityTypes[T]]: { type: string, nullable?: boolean } 13 | }, 14 | required: string[] 15 | }; 16 | } 17 | } 18 | 19 | interface AppEntities extends Entities { 20 | cron: Entity, 21 | message: Entity, 22 | page: Entity, 23 | queue: Entity, 24 | } 25 | 26 | interface AppEntityHooks { 27 | addEntityHooks(entityName: 'cron', hooks: EntityHooks): any 28 | addEntityHooks(entityName: 'message', hooks: EntityHooks): any 29 | addEntityHooks(entityName: 'page', hooks: EntityHooks): any 30 | addEntityHooks(entityName: 'queue', hooks: EntityHooks): any 31 | } 32 | 33 | declare module 'fastify' { 34 | interface FastifyInstance { 35 | platformatic: PlatformaticApp & 36 | PlatformaticDBMixin & 37 | AppEntityHooks 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /lib/backoff.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | function computeBackoff (opts) { 4 | opts ||= {} 5 | opts.maxRetries ||= Number.MAX_SAFE_INTEGER 6 | if (!(opts.maxRetries >= 1)) { 7 | throw new Error('backoff requires maxRetries greater or equal to one') 8 | } 9 | const maxRetries = opts.maxRetries 10 | let retries = opts.retries || 0 11 | const waitFor = 100 * Math.pow(2, retries) 12 | if (retries++ >= maxRetries) { 13 | return false 14 | } 15 | return { retries, maxRetries, waitFor } 16 | } 17 | 18 | module.exports.computeBackoff = computeBackoff 19 | -------------------------------------------------------------------------------- /lib/cron.js: -------------------------------------------------------------------------------- 1 | /// 2 | 'use strict' 3 | 4 | const cronParser = require('cron-parser') 5 | const fp = require('fastify-plugin') 6 | 7 | /** @param {import('fastify').FastifyInstance} app */ 8 | module.exports = fp(async function (app) { 9 | app.platformatic.addEntityHooks('cron', { 10 | save (original, args) { 11 | const input = args.input 12 | 13 | // This might fail if save is called from a transaction 14 | // this is not the case right now. 15 | return app.platformatic.db.tx(runInTransaction) 16 | 17 | async function runInTransaction (tx) { 18 | const schedule = args.input.schedule 19 | let interval 20 | try { 21 | interval = cronParser.parseExpression(schedule) 22 | } catch (err) { 23 | const _err = new Error('Invalid cron expression') 24 | _err.cause = err 25 | throw _err 26 | } 27 | 28 | const next = interval.next() 29 | const cron = await original({ 30 | ...args, 31 | tx 32 | }) 33 | await app.platformatic.entities.message.save({ 34 | ...args, 35 | input: { 36 | cronId: cron.id, 37 | queueId: input.queueId, 38 | when: next, 39 | headers: input.headers, 40 | body: input.body 41 | }, 42 | tx 43 | }) 44 | return cron 45 | } 46 | } 47 | }) 48 | }) 49 | -------------------------------------------------------------------------------- /lib/executor.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const cronParser = require('cron-parser') 4 | const { computeBackoff } = require('./backoff') 5 | const { request } = require('undici') 6 | 7 | class Executor { 8 | constructor (app) { 9 | this.app = app 10 | this.timer = null 11 | this.execute = () => { 12 | /* c8 ignore next 3 */ 13 | this._execute().catch((err) => { 14 | app.log.error({ err }) 15 | }) 16 | } 17 | } 18 | 19 | async _execute () { 20 | const app = this.app 21 | const now = new Date() 22 | const { db, sql } = app.platformatic 23 | 24 | /* Write a join sql query between messages and queue on queue_id column */ 25 | const messages = await db.query(sql` 26 | SELECT queues.callback_url AS "callbackUrl", 27 | queues.method as "method", 28 | messages.body AS "body", 29 | messages.headers AS "headers", 30 | messages.retries AS "retries", 31 | queues.headers AS "queueHeaders", 32 | queues.max_retries AS "maxRetries", 33 | queues.dead_letter_queue_id AS "deadLetterQueueId", 34 | messages."when" AS "when", 35 | messages.cron_id AS "cronId", 36 | messages.id AS "id" 37 | 38 | FROM messages 39 | 40 | INNER JOIN queues ON messages.queue_id = queues.id 41 | 42 | WHERE messages.sent_at IS NULL 43 | AND messages.failed = false 44 | AND messages."when" <= ${now} 45 | 46 | LIMIT 10 47 | `) 48 | 49 | const res = await Promise.allSettled(messages.map(async (message) => { 50 | const { callbackUrl, method, body, maxRetries, deadLetterQueueId } = message 51 | // We must JSON.parse(message.headers) because SQLite store JSON 52 | // as strings. 53 | const headers = { 54 | ...message.queueHeaders, 55 | ...message.headers 56 | } 57 | headers['content-type'] ||= 'application/json' 58 | 59 | // We do not want this call to be inside a transaction 60 | const succesful = await this.makeCallback(callbackUrl, method, headers, body) 61 | 62 | // We are not handling this error right now, but we should 63 | await db.tx(async (tx) => { 64 | if (succesful) { 65 | await app.platformatic.entities.message.save({ 66 | input: { 67 | id: message.id, 68 | sentAt: new Date() 69 | }, 70 | tx 71 | }) 72 | app.log.info({ callbackUrl, method }, 'callback succesful!') 73 | } else { 74 | const backoff = computeBackoff({ retries: message.retries, maxRetries }) 75 | if (!backoff) { 76 | app.log.warn({ message, body }, 'callback failed') 77 | await app.platformatic.entities.message.save({ 78 | input: { 79 | id: message.id, 80 | sentAt: new Date(), 81 | failed: true 82 | }, 83 | tx 84 | }) 85 | if (deadLetterQueueId) { 86 | await app.platformatic.entities.message.save({ 87 | input: { 88 | queueId: deadLetterQueueId, 89 | body: message.body, 90 | headers: message.headers 91 | }, 92 | tx 93 | }) 94 | } 95 | } else { 96 | app.log.info({ callbackUrl, method }, 'callback failed, scheduling retry!') 97 | const newItem = { 98 | id: message.id, 99 | retries: backoff.retries, 100 | when: new Date(Date.now() + backoff.waitFor) 101 | } 102 | await app.platformatic.entities.message.save({ input: newItem, tx }) 103 | } 104 | } 105 | 106 | // let's schedule the next call if it's a cron 107 | if (message.cronId) { 108 | const [cron] = await app.platformatic.entities.cron.find({ where: { id: { eq: message.cronId } }, tx }) 109 | const interval = cronParser.parseExpression(cron.schedule) 110 | const next = interval.next() 111 | await app.platformatic.entities.message.save({ 112 | input: { 113 | queueId: cron.queueId, 114 | when: next, 115 | headers: cron.headers, 116 | body: cron.body, 117 | cronId: cron.id 118 | }, 119 | tx 120 | }) 121 | } 122 | }) 123 | })) 124 | 125 | /* c8 ignore next 4 */ 126 | for (const r of res) { 127 | if (r.status === 'rejected') { 128 | app.log.error({ err: r.reason }, 'error while executing message') 129 | } 130 | } 131 | 132 | const [next] = await app.platformatic.entities.message.find({ 133 | where: { 134 | sentAt: { 135 | eq: null 136 | } 137 | }, 138 | orderBy: [{ 139 | field: 'when', direction: 'asc' 140 | }], 141 | limit: 1 142 | }) 143 | 144 | if (next) { 145 | const whenTime = new Date(next.when).getTime() 146 | const nowTime = new Date(now).getTime() 147 | const delay = whenTime - nowTime 148 | clearTimeout(this.timer) 149 | this.timer = setTimeout(this.execute, delay) 150 | this.nextTime = now + delay 151 | } 152 | } 153 | 154 | updateTimer (date) { 155 | if ((this.nextTime > 0 && date.getTime() < this.nextTime) || !this.timer) { 156 | clearTimeout(this.timer) 157 | this.nextTime = date.getTime() 158 | const delay = this.nextTime - Date.now() 159 | this.timer = setTimeout(this.execute, delay) 160 | } 161 | } 162 | 163 | stop () { 164 | clearTimeout(this.timer) 165 | } 166 | 167 | async makeCallback (callbackUrl, method, headers, body, message) { 168 | try { 169 | const res = await request(callbackUrl, { 170 | method, 171 | headers, 172 | body 173 | }) 174 | if (res.statusCode >= 200 && res.statusCode < 300) { 175 | return true 176 | } else { 177 | let body 178 | if (res.headers['content-type'].indexOf('application/json') === 0) { 179 | body = await res.body.json() 180 | } else if (res.headers['content-type'] === 'text/plain') { 181 | body = await res.body.text() 182 | } else { 183 | res.body.resume() 184 | // not interested in the errors 185 | res.body.on('error', () => {}) 186 | } 187 | 188 | this.app.log.warn({ message, statusCode: res.statusCode, body }, 'callback unsuccessful, maybe retry') 189 | return false 190 | } 191 | /* c8 ignore next 4 */ 192 | } catch (err) { 193 | this.app.log.warn({ err }, 'error processing callback') 194 | return false 195 | } 196 | } 197 | } 198 | 199 | module.exports = Executor 200 | -------------------------------------------------------------------------------- /migrations/001.do.sql: -------------------------------------------------------------------------------- 1 | 2 | /* create a queues table */ 3 | CREATE TABLE queues ( 4 | id SERIAL PRIMARY KEY, 5 | name VARCHAR(255) NOT NULL, 6 | created_at TIMESTAMP NOT NULL, 7 | updated_at TIMESTAMP NOT NULL 8 | ); 9 | -------------------------------------------------------------------------------- /migrations/001.undo.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Add SQL in this file to drop the database tables 3 | DROP TABLE queues; 4 | -------------------------------------------------------------------------------- /migrations/002.do.sql: -------------------------------------------------------------------------------- 1 | /* creates an items table */ 2 | CREATE TABLE items ( 3 | id SERIAL PRIMARY KEY, 4 | queue_id INTEGER NOT NULL REFERENCES queues(id), 5 | "when" TIMESTAMP NOT NULL, 6 | /* add some validations, this should be an URL */ 7 | callback_url VARCHAR(2048) NOT NULL, 8 | /* add some validations, this should be an enum */ 9 | method VARCHAR(10) NOT NULL, 10 | body TEXT, 11 | headers JSON, 12 | sent_at TIMESTAMP, 13 | created_at TIMESTAMP NOT NULL, 14 | updated_at TIMESTAMP NOT NULL 15 | ); 16 | -------------------------------------------------------------------------------- /migrations/002.undo.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE items; 2 | -------------------------------------------------------------------------------- /migrations/003.do.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE items ADD COLUMN failed BOOLEAN NOT NULL DEFAULT FALSE; 2 | -------------------------------------------------------------------------------- /migrations/003.undo.sql: -------------------------------------------------------------------------------- 1 | /* remove column failed from table items */ 2 | ALTER TABLE items DROP COLUMN failed; 3 | -------------------------------------------------------------------------------- /migrations/004.do.sql: -------------------------------------------------------------------------------- 1 | /* add retries column to table items */ 2 | ALTER TABLE items ADD COLUMN retries INTEGER NOT NULL DEFAULT 0; 3 | -------------------------------------------------------------------------------- /migrations/004.undo.sql: -------------------------------------------------------------------------------- 1 | /* remove column retries from table items */ 2 | ALTER TABLE items DROP COLUMN retries; 3 | -------------------------------------------------------------------------------- /migrations/005.do.sql: -------------------------------------------------------------------------------- 1 | /* create crons table */ 2 | CREATE TABLE crons ( 3 | id SERIAL PRIMARY KEY, 4 | queue_id INTEGER NOT NULL REFERENCES queues(id), 5 | 6 | schedule VARCHAR(255) NOT NULL, 7 | created_at TIMESTAMP NOT NULL, 8 | updated_at TIMESTAMP NOT NULL, 9 | 10 | /* these will be needed for creating the item */ 11 | 12 | /* add some validations, this should be an URL */ 13 | callback_url VARCHAR(2048) NOT NULL, 14 | /* add some validations, this should be an enum */ 15 | method VARCHAR(10) NOT NULL, 16 | body TEXT, 17 | headers JSON 18 | ); 19 | 20 | /* ADD cron_id field to items */ 21 | ALTER TABLE items ADD COLUMN cron_id INTEGER REFERENCES crons(id); 22 | -------------------------------------------------------------------------------- /migrations/005.undo.sql: -------------------------------------------------------------------------------- 1 | 2 | DROP TABLE crons; 3 | ALTER TABLE items DROP COLUMN cron_id; 4 | -------------------------------------------------------------------------------- /migrations/006.do.sql: -------------------------------------------------------------------------------- 1 | /* rename table items to messages */ 2 | ALTER TABLE items RENAME TO messages; 3 | 4 | /* add callback_url column to queue */ 5 | ALTER TABLE queues ADD COLUMN callback_url TEXT NOT NULL; 6 | ALTER TABLE queues ADD COLUMN method TEXT NOT NULL; 7 | ALTER TABLE queues ADD COLUMN headers JSON; 8 | 9 | /* drop column callback_url from message */ 10 | ALTER TABLE messages DROP COLUMN callback_url; 11 | ALTER TABLE messages DROP COLUMN method; 12 | 13 | /* drop column callback_url from message */ 14 | ALTER TABLE crons DROP COLUMN callback_url; 15 | ALTER TABLE crons DROP COLUMN method; 16 | -------------------------------------------------------------------------------- /migrations/006.undo.sql: -------------------------------------------------------------------------------- 1 | /* rename table messages to items */ 2 | ALTER TABLE messages RENAME TO items; 3 | 4 | /* drop column callback_url from queue */ 5 | ALTER TABLE queues DROP COLUMN callback_url; 6 | ALTER TABLE queues DROP COLUMN method; 7 | ALTER TABLE queues DROP COLUMN headers; 8 | 9 | ALTER TABLE items ADD COLUMN callback_url TEXT; 10 | ALTER TABLE items ADD COLUMN method TEXT; 11 | 12 | ALTER TABLE crons ADD COLUMN callback_url TEXT; 13 | ALTER TABLE crons ADD COLUMN method TEXT; 14 | -------------------------------------------------------------------------------- /migrations/007.do.sql: -------------------------------------------------------------------------------- 1 | /* add column dead_letter_queue_id to table queues */ 2 | ALTER TABLE queues ADD COLUMN dead_letter_queue_id INTEGER REFERENCES queues(id) ON DELETE SET NULL; 3 | 4 | /* add column max_retries to table queues */ 5 | ALTER TABLE queues ADD COLUMN max_retries INTEGER NOT NULL DEFAULT 5; 6 | -------------------------------------------------------------------------------- /migrations/007.undo.sql: -------------------------------------------------------------------------------- 1 | /* remove column max_retries from table queues */ 2 | ALTER TABLE queues DROP COLUMN max_retries; 3 | 4 | /* remove column dead_letter_queue_id from table queues */ 5 | ALTER TABLE queues DROP COLUMN dead_letter_queue_id; 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "start": "platformatic db start", 4 | "lint": "standard | snazzy", 5 | "test:unit": "c8 --100 tap --no-coverage test/*.test.js", 6 | "test": "npm run lint && npm run test:unit" 7 | }, 8 | "devDependencies": { 9 | "@databases/pg": "^5.4.1", 10 | "@fastify/pre-commit": "^2.0.2", 11 | "fastify": "^4.10.2", 12 | "pino-pretty": "^10.0.0", 13 | "snazzy": "^9.0.0", 14 | "standard": "^17.0.0", 15 | "tap": "^16.3.2" 16 | }, 17 | "dependencies": { 18 | "@platformatic/db": "^1.0.0", 19 | "c8": "^8.0.0", 20 | "cron-parser": "^4.7.1", 21 | "fastify-plugin": "^4.5.0", 22 | "platformatic": "^1.0.0" 23 | }, 24 | "engines": { 25 | "node": "^18.8.0 || >=19" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /platformatic.db.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://platformatic.dev/schemas/v0.45.0/db", 3 | "server": { 4 | "hostname": "{PLT_SERVER_HOSTNAME}", 5 | "port": "{PORT}", 6 | "logger": { 7 | "level": "{PLT_SERVER_LOGGER_LEVEL}" 8 | }, 9 | "cors": { 10 | "origin": "*" 11 | } 12 | }, 13 | "authorization": { 14 | "adminSecret": "{PLT_ADMIN_SECRET}", 15 | "rules": [ 16 | { 17 | "role": "anonymous", 18 | "entity": "queue", 19 | "find": false, 20 | "save": false, 21 | "delete": false 22 | }, 23 | { 24 | "role": "anonymous", 25 | "entity": "message", 26 | "find": false, 27 | "save": false, 28 | "delete": false 29 | } 30 | ] 31 | }, 32 | "migrations": { 33 | "dir": "migrations" 34 | }, 35 | "types": { 36 | "autogenerate": true 37 | }, 38 | "plugins": { 39 | "paths": [ 40 | { 41 | "path": "plugin.js", 42 | "options": { 43 | "lock": "{PLT_LOCK}", 44 | "leaderPoll": "{PLT_LEADER_POLL}" 45 | } 46 | } 47 | ] 48 | }, 49 | "db": { 50 | "connectionString": "{DATABASE_URL}", 51 | "graphql": true, 52 | "openapi": true, 53 | "events": false 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /plugin.js: -------------------------------------------------------------------------------- 1 | /// 2 | 'use strict' 3 | 4 | // Needed to work with dates & postgresql 5 | // See https://node-postgres.com/features/types/ 6 | process.env.TZ = 'UTC' 7 | 8 | const cronPlugin = require('./lib/cron') 9 | const Executor = require('./lib/executor') 10 | const { scheduler } = require('timers/promises') 11 | const { on } = require('events') 12 | 13 | /** @param {import('fastify').FastifyInstance} app */ 14 | module.exports = async function (app, options) { 15 | const lock = Number(options.lock) || 42 16 | /* c8 ignore next 1 */ 17 | const leaderPoll = Number(options.leaderPoll) || 10000 18 | 19 | app.log.info('Locking cron plugin to advisory lock %d', lock) 20 | 21 | const dummyExecutor = { 22 | execute () {}, 23 | updateTimer () { 24 | const { db, sql } = app.platformatic 25 | db.query(sql` 26 | NOTIFY "update_timer"; 27 | `) 28 | /* c8 ignore next 3 */ 29 | .catch((err) => { 30 | app.log.error({ err }, 'Error in dummy updateTimer') 31 | }) 32 | }, 33 | stop () {} 34 | } 35 | 36 | let executor = dummyExecutor 37 | let elected = false 38 | 39 | const abortController = new AbortController() 40 | 41 | async function amITheLeader () { 42 | const { db, sql } = app.platformatic 43 | await db.task(async (t) => { 44 | while (!abortController.signal.aborted) { 45 | const [{ leader }] = await t.query(sql` 46 | SELECT pg_try_advisory_lock(${lock}) as leader; 47 | `) 48 | if (leader && !elected) { 49 | app.log.info('This instance is the leader') 50 | executor = new Executor(app) 51 | executor.execute() 52 | elected = true 53 | ;(async () => { 54 | await t.query(sql` 55 | LISTEN "update_timer"; 56 | `) 57 | for await (const notification of on(t._driver.client, 'notification', { signal: abortController.signal })) { 58 | app.log.debug({ notification }, 'Received notification') 59 | try { 60 | await executor.execute() 61 | /* c8 ignore next 3 */ 62 | } catch (err) { 63 | app.log.warn({ err }, 'error while processing notification') 64 | } 65 | // TODO: write automated tests for this 66 | } 67 | /* c8 ignore next 19 */ 68 | })() 69 | .catch((err) => { 70 | if (err.name !== 'AbortError') { 71 | // an error occurred, and it's expected 72 | app.log.error({ err }, 'Error in notification loop') 73 | } else { 74 | abortController.abort() 75 | } 76 | }) 77 | } else if (leader && elected) { 78 | app.log.debug('This instance is still the leader') 79 | } else if (!leader && elected) { 80 | // this should never happen 81 | app.log.warn('This instance was the leader but is not anymore') 82 | await executor.stop() 83 | executor = dummyExecutor 84 | elected = false 85 | } else { 86 | app.log.debug('This instance is not the leader') 87 | executor = dummyExecutor 88 | } 89 | try { 90 | await scheduler.wait(leaderPoll, { signal: abortController.signal }) 91 | } catch { 92 | break 93 | } 94 | } 95 | }) 96 | app.log.debug('leader loop stopped') 97 | } 98 | 99 | let leaderLoop = amITheLeader() 100 | 101 | retryLeaderLoop(leaderLoop) 102 | 103 | /* c8 ignore next 10 */ 104 | function retryLeaderLoop () { 105 | leaderLoop.catch((err) => { 106 | app.log.error({ err }, 'Error in leader loop') 107 | return executor.stop() 108 | }).then(() => { 109 | if (!abortController.signal.aborted) { 110 | leaderLoop = amITheLeader() 111 | retryLeaderLoop(leaderLoop) 112 | } 113 | }) 114 | } 115 | 116 | app.platformatic.addEntityHooks('message', { 117 | async insert (original, { inputs, ...rest }) { 118 | const now = new Date() // now 119 | for (const input of inputs) { 120 | input.when = now 121 | } 122 | 123 | const res = await original({ inputs, ...rest }) 124 | 125 | for (const input of inputs) { 126 | const date = new Date(input.when) 127 | executor.updateTimer(date) 128 | } 129 | 130 | return res 131 | }, 132 | 133 | async save (original, { input, ...rest }) { 134 | if (!input.when) { 135 | input.when = new Date() // now 136 | } 137 | 138 | const res = await original({ input, ...rest }) 139 | 140 | const date = new Date(input.when) 141 | executor.updateTimer(date) 142 | 143 | return res 144 | } 145 | }) 146 | 147 | await app.register(cronPlugin) 148 | 149 | await executor.execute() 150 | 151 | app.addHook('onClose', async () => { 152 | abortController.abort() 153 | await leaderLoop 154 | executor.stop() 155 | }) 156 | } 157 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "rangeStrategy": "update-lockfile", 7 | "prHourlyLimit": 10, 8 | "packageRules": [ 9 | { 10 | "matchUpdateTypes": ["minor", "patch", "pin", "digest"], 11 | "automerge": true 12 | }, 13 | { 14 | "groupName": "platformatic", 15 | "matchPackagePrefixes": [ 16 | "@platformatic/", 17 | "platformatic" 18 | ] 19 | } 20 | ], 21 | "lockFileMaintenance": { "enabled": true } 22 | } 23 | -------------------------------------------------------------------------------- /target.mjs: -------------------------------------------------------------------------------- 1 | import fastify from 'fastify' 2 | 3 | const app = fastify({ 4 | logger: { 5 | transport: { 6 | target: 'pino-pretty' 7 | } 8 | } 9 | }) 10 | 11 | app.post('/', async (request, reply) => { 12 | request.log.info({ body: request.body }, 'request body') 13 | return { status: 'ok' } 14 | }) 15 | 16 | await app.listen({ port: 3000 }) 17 | -------------------------------------------------------------------------------- /test/backoff.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { test } = require('tap') 4 | const { computeBackoff } = require('../lib/backoff') 5 | 6 | test('negative maxRetries', async ({ teardown, equal, plan }) => { 7 | plan(1) 8 | try { 9 | computeBackoff({ maxRetries: -1 }) 10 | } catch (e) { 11 | equal(e.message, 'backoff requires maxRetries greater or equal to one') 12 | } 13 | }) 14 | 15 | test('computeBackoff', async ({ teardown, same, plan }) => { 16 | same(computeBackoff({ retries: 0, maxRetries: 5 }), { retries: 1, maxRetries: 5, waitFor: 100 }) 17 | same(computeBackoff({ retries: 1, maxRetries: 5 }), { retries: 2, maxRetries: 5, waitFor: 200 }) 18 | same(computeBackoff({ retries: 2, maxRetries: 5 }), { retries: 3, maxRetries: 5, waitFor: 400 }) 19 | same(computeBackoff({ retries: 3, maxRetries: 5 }), { retries: 4, maxRetries: 5, waitFor: 800 }) 20 | same(computeBackoff({ retries: 4, maxRetries: 5 }), { retries: 5, maxRetries: 5, waitFor: 1600 }) 21 | same(computeBackoff({ retries: 5, maxRetries: 5 }), false) 22 | }) 23 | -------------------------------------------------------------------------------- /test/cron.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { buildServer, adminSecret, cleandb } = require('./helper') 4 | const { test, beforeEach } = require('tap') 5 | const { once, EventEmitter } = require('events') 6 | const Fastify = require('fastify') 7 | 8 | beforeEach(cleandb) 9 | 10 | test('happy path', async ({ teardown, equal, plan, same }) => { 11 | plan(6) 12 | const ee = new EventEmitter() 13 | const server = await buildServer(teardown) 14 | 15 | const target = Fastify() 16 | target.post('/', async (req, reply) => { 17 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 18 | ee.emit('called') 19 | return { ok: true } 20 | }) 21 | 22 | teardown(() => target.close()) 23 | await target.listen({ port: 0 }) 24 | const targetUrl = `http://localhost:${target.server.address().port}` 25 | 26 | let queueId 27 | { 28 | const res = await server.inject({ 29 | method: 'POST', 30 | url: '/graphql', 31 | headers: { 32 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 33 | }, 34 | payload: { 35 | query: ` 36 | mutation($callbackUrl: String!) { 37 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 38 | id 39 | } 40 | } 41 | `, 42 | variables: { 43 | callbackUrl: targetUrl 44 | } 45 | } 46 | }) 47 | equal(res.statusCode, 200) 48 | const body = res.json() 49 | const { data } = body 50 | queueId = data.saveQueue.id 51 | equal(queueId, '1') 52 | } 53 | 54 | const p1 = once(ee, 'called') 55 | const schedule = '*/1 * * * * *' 56 | 57 | { 58 | const msg = JSON.stringify({ 59 | message: 'HELLO FOLKS!' 60 | }) 61 | const query = ` 62 | mutation($body: String!, $queueId: ID, $schedule: String!) { 63 | saveCron(input: { queueId: $queueId, headers: "{ \\"content-type\\": \\"application/json\\" }", body: $body, schedule: $schedule }) { 64 | id 65 | schedule 66 | } 67 | } 68 | ` 69 | 70 | const res = await server.inject({ 71 | method: 'POST', 72 | url: '/graphql', 73 | headers: { 74 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 75 | }, 76 | payload: { 77 | query, 78 | variables: { 79 | body: msg, 80 | queueId, 81 | schedule 82 | } 83 | } 84 | }) 85 | const body = res.json() 86 | equal(res.statusCode, 200) 87 | 88 | const { data } = body 89 | equal(data.saveCron.schedule, schedule) 90 | 91 | /* 92 | * Add items 93 | * 94 | * items { 95 | * id 96 | * when 97 | * } 98 | * 99 | * equal(data.saveCron.items.length, 1) 100 | * const item = data.saveCron.items[0] 101 | * const when = new Date(item.when) 102 | * equal(when.getTime() - now <= 1000, true) 103 | */ 104 | } 105 | 106 | await p1 107 | 108 | const p2 = once(ee, 'called') 109 | await p2 110 | }) 111 | 112 | test('invalid cron expression', async ({ teardown, equal, plan, same }) => { 113 | plan(4) 114 | const server = await buildServer(teardown) 115 | 116 | const targetUrl = 'http://localhost:4242' 117 | 118 | let queueId 119 | { 120 | const res = await server.inject({ 121 | method: 'POST', 122 | url: '/graphql', 123 | headers: { 124 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 125 | }, 126 | payload: { 127 | query: ` 128 | mutation($callbackUrl: String!) { 129 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 130 | id 131 | } 132 | } 133 | `, 134 | variables: { 135 | callbackUrl: targetUrl 136 | } 137 | } 138 | }) 139 | equal(res.statusCode, 200) 140 | const body = res.json() 141 | const { data } = body 142 | queueId = data.saveQueue.id 143 | equal(queueId, '1') 144 | } 145 | 146 | const schedule = 'hello world' 147 | 148 | { 149 | const msg = JSON.stringify({ 150 | message: 'HELLO FOLKS!' 151 | }) 152 | const query = ` 153 | mutation($body: String!, $queueId: ID, $schedule: String!) { 154 | saveCron(input: { queueId: $queueId, headers: "{ \\"content-type\\": \\"application/json\\" }", body: $body, schedule: $schedule }) { 155 | id 156 | schedule 157 | } 158 | } 159 | ` 160 | 161 | const res = await server.inject({ 162 | method: 'POST', 163 | url: '/graphql', 164 | headers: { 165 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 166 | }, 167 | payload: { 168 | query, 169 | variables: { 170 | body: msg, 171 | queueId, 172 | schedule 173 | } 174 | } 175 | }) 176 | const body = res.json() 177 | equal(res.statusCode, 200) 178 | same(body.errors[0].message, 'Invalid cron expression') 179 | } 180 | }) 181 | -------------------------------------------------------------------------------- /test/election.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { buildServer, adminSecret, cleandb } = require('./helper') 4 | const { test, beforeEach } = require('tap') 5 | const { once, EventEmitter } = require('events') 6 | const Fastify = require('fastify') 7 | 8 | beforeEach(cleandb) 9 | 10 | test('happy path', async ({ teardown, equal, plan, same }) => { 11 | const ee = new EventEmitter() 12 | const server1 = await buildServer(teardown) 13 | const server2 = await buildServer(teardown) 14 | 15 | const target = Fastify() 16 | target.post('/', async (req, reply) => { 17 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 18 | ee.emit('called') 19 | return { ok: true } 20 | }) 21 | 22 | teardown(() => target.close()) 23 | await target.listen({ port: 0 }) 24 | const targetUrl = `http://localhost:${target.server.address().port}` 25 | 26 | let queueId 27 | { 28 | const res = await server1.inject({ 29 | method: 'POST', 30 | url: '/graphql', 31 | headers: { 32 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 33 | }, 34 | payload: { 35 | query: ` 36 | mutation($callbackUrl: String!) { 37 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 38 | id 39 | } 40 | } 41 | `, 42 | variables: { 43 | callbackUrl: targetUrl 44 | } 45 | } 46 | }) 47 | equal(res.statusCode, 200) 48 | const body = res.json() 49 | const { data } = body 50 | queueId = data.saveQueue.id 51 | equal(queueId, '1') 52 | } 53 | 54 | const p = once(ee, 'called') 55 | { 56 | const msg = JSON.stringify({ 57 | message: 'HELLO FOLKS!' 58 | }) 59 | const now = Date.now() 60 | const query = ` 61 | mutation($body: String!, $queueId: ID) { 62 | saveMessage(input: { queueId: $queueId, headers: "{ \\"content-type\\": \\"application/json\\" }", body: $body }) { 63 | id 64 | when 65 | } 66 | } 67 | ` 68 | 69 | const res = await server2.inject({ 70 | method: 'POST', 71 | url: '/graphql', 72 | headers: { 73 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 74 | }, 75 | payload: { 76 | query, 77 | variables: { 78 | body: msg, 79 | queueId 80 | } 81 | } 82 | }) 83 | const body = res.json() 84 | equal(res.statusCode, 200) 85 | 86 | const { data } = body 87 | const when = new Date(data.saveMessage.when) 88 | equal(when.getTime() - now >= 0, true) 89 | } 90 | 91 | await p 92 | }) 93 | 94 | test('re-election', async ({ teardown, equal, plan, same }) => { 95 | const ee = new EventEmitter() 96 | const server1 = await buildServer(teardown) 97 | const server2 = await buildServer(teardown) 98 | 99 | const target = Fastify() 100 | target.post('/', async (req, reply) => { 101 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 102 | ee.emit('called') 103 | return { ok: true } 104 | }) 105 | 106 | teardown(() => target.close()) 107 | await target.listen({ port: 0 }) 108 | const targetUrl = `http://localhost:${target.server.address().port}` 109 | 110 | let queueId 111 | { 112 | const res = await server1.inject({ 113 | method: 'POST', 114 | url: '/graphql', 115 | headers: { 116 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 117 | }, 118 | payload: { 119 | query: ` 120 | mutation($callbackUrl: String!) { 121 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 122 | id 123 | } 124 | } 125 | `, 126 | variables: { 127 | callbackUrl: targetUrl 128 | } 129 | } 130 | }) 131 | equal(res.statusCode, 200) 132 | const body = res.json() 133 | const { data } = body 134 | queueId = data.saveQueue.id 135 | equal(queueId, '1') 136 | } 137 | 138 | await server1.close() 139 | 140 | const p = once(ee, 'called') 141 | { 142 | const msg = JSON.stringify({ 143 | message: 'HELLO FOLKS!' 144 | }) 145 | const now = Date.now() 146 | const query = ` 147 | mutation($body: String!, $queueId: ID) { 148 | saveMessage(input: { queueId: $queueId, headers: "{ \\"content-type\\": \\"application/json\\" }", body: $body }) { 149 | id 150 | when 151 | } 152 | } 153 | ` 154 | 155 | const res = await server2.inject({ 156 | method: 'POST', 157 | url: '/graphql', 158 | headers: { 159 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 160 | }, 161 | payload: { 162 | query, 163 | variables: { 164 | body: msg, 165 | queueId 166 | } 167 | } 168 | }) 169 | const body = res.json() 170 | equal(res.statusCode, 200) 171 | 172 | const { data } = body 173 | const when = new Date(data.saveMessage.when) 174 | equal(when.getTime() - now >= 0, true) 175 | } 176 | 177 | await p 178 | }) 179 | -------------------------------------------------------------------------------- /test/helper.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { teardown } = require('tap') 4 | const { join } = require('path') 5 | const { readFile } = require('fs/promises') 6 | const { setGlobalDispatcher, Agent } = require('undici') 7 | const db = require('@platformatic/db') 8 | const createConnectionPool = require('@databases/pg') 9 | 10 | setGlobalDispatcher(new Agent({ 11 | keepAliveTimeout: 10, 12 | keepAliveMaxTimeout: 10 13 | })) 14 | 15 | const adminSecret = 'admin-secret' 16 | 17 | async function getConfig () { 18 | const config = JSON.parse(await readFile(join(__dirname, '../platformatic.db.json'), 'utf8')) 19 | config.server.port = 0 20 | config.server.logger = { level: 'error' } 21 | // config.server.logger = false 22 | config.db.connectionString = 'postgres://postgres:postgres@127.0.0.1:5432/postgres' 23 | config.migrations.autoApply = true 24 | config.types.autogenerate = false 25 | config.authorization.adminSecret = adminSecret 26 | config.plugins.paths[0].options.leaderPoll = 1000 27 | return { config } 28 | } 29 | 30 | async function buildServer (teardown) { 31 | const { config } = await getConfig() 32 | const server = await db.buildServer(config) 33 | teardown(() => server.close()) 34 | return server 35 | } 36 | 37 | let pool = null 38 | 39 | async function cleandb () { 40 | if (!pool) { 41 | pool = createConnectionPool({ 42 | connectionString: 'postgres://postgres:postgres@127.0.0.1:5432/postgres', 43 | bigIntMode: 'bigint' 44 | }) 45 | teardown(() => pool.dispose()) 46 | } 47 | 48 | const sql = createConnectionPool.sql 49 | 50 | // TODO use schemas 51 | try { 52 | await pool.query(sql`DROP TABLE MESSAGES;`) 53 | } catch {} 54 | try { 55 | await pool.query(sql`DROP TABLE CRONS;`) 56 | } catch {} 57 | try { 58 | await pool.query(sql`DROP TABLE QUEUES;`) 59 | } catch {} 60 | try { 61 | await pool.query(sql`DROP TABLE VERSIONS;`) 62 | } catch {} 63 | } 64 | 65 | module.exports.getConfig = getConfig 66 | module.exports.adminSecret = adminSecret 67 | module.exports.buildServer = buildServer 68 | module.exports.cleandb = cleandb 69 | -------------------------------------------------------------------------------- /test/parallelism.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { buildServer, adminSecret, cleandb } = require('./helper') 4 | const { test, beforeEach } = require('tap') 5 | const { once, EventEmitter } = require('events') 6 | const Fastify = require('fastify') 7 | 8 | beforeEach(cleandb) 9 | 10 | test('happy path', async ({ teardown, equal, plan, same, pass }) => { 11 | plan(5) 12 | const ee = new EventEmitter() 13 | const server = await buildServer(teardown) 14 | 15 | const target = Fastify() 16 | const p = once(ee, 'called') 17 | let called = false 18 | target.post('/', async (req, reply) => { 19 | // This if block is to make sure that the first request waits 20 | // for the second to complete 21 | if (!called) { 22 | called = true 23 | await p 24 | } else { 25 | ee.emit('called') 26 | } 27 | 28 | pass('request completed') 29 | return { ok: true } 30 | }) 31 | 32 | teardown(() => target.close()) 33 | await target.listen({ port: 0 }) 34 | const targetUrl = `http://localhost:${target.server.address().port}` 35 | 36 | let queueId 37 | { 38 | const res = await server.inject({ 39 | method: 'POST', 40 | url: '/graphql', 41 | headers: { 42 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 43 | }, 44 | payload: { 45 | query: ` 46 | mutation($callbackUrl: String!) { 47 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 48 | id 49 | } 50 | } 51 | `, 52 | variables: { 53 | callbackUrl: targetUrl 54 | } 55 | } 56 | }) 57 | equal(res.statusCode, 200) 58 | const body = res.json() 59 | const { data } = body 60 | queueId = data.saveQueue.id 61 | equal(queueId, '1') 62 | } 63 | 64 | { 65 | const query = ` 66 | mutation($messages: [MessageInput]!) { 67 | insertMessages(inputs: $messages) { 68 | id 69 | when 70 | } 71 | } 72 | ` 73 | 74 | const res = await server.inject({ 75 | method: 'POST', 76 | url: '/graphql', 77 | headers: { 78 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 79 | }, 80 | payload: { 81 | query, 82 | variables: { 83 | messages: [{ 84 | body: JSON.stringify({ message: 'HELLO FOLKS!' }), 85 | queueId 86 | }, { 87 | body: JSON.stringify({ message: 'HELLO FOLKS2!' }), 88 | queueId 89 | }] 90 | } 91 | } 92 | }) 93 | equal(res.statusCode, 200) 94 | } 95 | 96 | await p 97 | await new Promise(resolve => setImmediate(resolve)) 98 | }) 99 | -------------------------------------------------------------------------------- /test/queue.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { buildServer, adminSecret, cleandb } = require('./helper') 4 | const { test, beforeEach } = require('tap') 5 | const { once, EventEmitter } = require('events') 6 | const Fastify = require('fastify') 7 | 8 | beforeEach(cleandb) 9 | 10 | test('happy path', async ({ teardown, equal, plan, same }) => { 11 | plan(5) 12 | const ee = new EventEmitter() 13 | const server = await buildServer(teardown) 14 | 15 | const target = Fastify() 16 | target.post('/', async (req, reply) => { 17 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 18 | ee.emit('called') 19 | return { ok: true } 20 | }) 21 | 22 | teardown(() => target.close()) 23 | await target.listen({ port: 0 }) 24 | const targetUrl = `http://localhost:${target.server.address().port}` 25 | 26 | let queueId 27 | { 28 | const res = await server.inject({ 29 | method: 'POST', 30 | url: '/graphql', 31 | headers: { 32 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 33 | }, 34 | payload: { 35 | query: ` 36 | mutation($callbackUrl: String!) { 37 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 38 | id 39 | } 40 | } 41 | `, 42 | variables: { 43 | callbackUrl: targetUrl 44 | } 45 | } 46 | }) 47 | equal(res.statusCode, 200) 48 | const body = res.json() 49 | const { data } = body 50 | queueId = data.saveQueue.id 51 | equal(queueId, '1') 52 | } 53 | 54 | const p = once(ee, 'called') 55 | { 56 | const msg = JSON.stringify({ 57 | message: 'HELLO FOLKS!' 58 | }) 59 | const now = Date.now() 60 | const query = ` 61 | mutation($body: String!, $queueId: ID) { 62 | saveMessage(input: { queueId: $queueId, headers: "{ \\"content-type\\": \\"application/json\\" }", body: $body }) { 63 | id 64 | when 65 | } 66 | } 67 | ` 68 | 69 | const res = await server.inject({ 70 | method: 'POST', 71 | url: '/graphql', 72 | headers: { 73 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 74 | }, 75 | payload: { 76 | query, 77 | variables: { 78 | body: msg, 79 | queueId 80 | } 81 | } 82 | }) 83 | const body = res.json() 84 | equal(res.statusCode, 200) 85 | 86 | const { data } = body 87 | const when = new Date(data.saveMessage.when) 88 | equal(when.getTime() - now >= 0, true) 89 | } 90 | 91 | await p 92 | }) 93 | 94 | test('`text plain` content type', async ({ teardown, equal, plan, same }) => { 95 | plan(5) 96 | const ee = new EventEmitter() 97 | const server = await buildServer(teardown) 98 | 99 | const target = Fastify() 100 | target.post('/', async (req, reply) => { 101 | same(req.body, 'HELLO FOLKS!', 'message is equal') 102 | ee.emit('called') 103 | return { ok: true } 104 | }) 105 | 106 | teardown(() => target.close()) 107 | await target.listen({ port: 0 }) 108 | const targetUrl = `http://localhost:${target.server.address().port}` 109 | 110 | let queueId 111 | { 112 | const res = await server.inject({ 113 | method: 'POST', 114 | url: '/graphql', 115 | headers: { 116 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 117 | }, 118 | payload: { 119 | query: ` 120 | mutation($callbackUrl: String!) { 121 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 122 | id 123 | } 124 | } 125 | `, 126 | variables: { 127 | callbackUrl: targetUrl 128 | } 129 | } 130 | }) 131 | equal(res.statusCode, 200) 132 | const body = res.json() 133 | const { data } = body 134 | queueId = data.saveQueue.id 135 | equal(queueId, '1') 136 | } 137 | 138 | const p = once(ee, 'called') 139 | { 140 | const msg = 'HELLO FOLKS!' 141 | const now = Date.now() 142 | const res = await server.inject({ 143 | method: 'POST', 144 | url: '/graphql', 145 | headers: { 146 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 147 | }, 148 | payload: { 149 | query: ` 150 | mutation($body: String!, $queueId: ID) { 151 | saveMessage(input: { queueId: $queueId, body: $body, headers: "{ \\"content-type\\": \\"text/plain\\" }" } ) { 152 | id 153 | when 154 | } 155 | } 156 | `, 157 | variables: { 158 | body: msg, 159 | queueId 160 | } 161 | } 162 | }) 163 | const body = res.json() 164 | equal(res.statusCode, 200) 165 | const { data } = body 166 | const when = new Date(data.saveMessage.when) 167 | equal(when.getTime() - now >= 0, true) 168 | } 169 | 170 | await p 171 | }) 172 | 173 | test('future when', async ({ teardown, equal, plan, same }) => { 174 | plan(6) 175 | 176 | const ee = new EventEmitter() 177 | const server = await buildServer(teardown) 178 | 179 | const target = Fastify() 180 | target.post('/', async (req, reply) => { 181 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 182 | ee.emit('called', Date.now()) 183 | return { ok: true } 184 | }) 185 | 186 | teardown(() => target.close()) 187 | await target.listen({ port: 0 }) 188 | const targetUrl = `http://localhost:${target.server.address().port}` 189 | 190 | let queueId 191 | { 192 | const res = await server.inject({ 193 | method: 'POST', 194 | url: '/graphql', 195 | headers: { 196 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 197 | }, 198 | payload: { 199 | query: ` 200 | mutation($callbackUrl: String!) { 201 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 202 | id 203 | } 204 | } 205 | `, 206 | variables: { 207 | callbackUrl: targetUrl 208 | } 209 | } 210 | }) 211 | equal(res.statusCode, 200) 212 | const body = res.json() 213 | const { data } = body 214 | queueId = data.saveQueue.id 215 | equal(queueId, '1') 216 | } 217 | 218 | const p = once(ee, 'called') 219 | const now = Date.now() 220 | 221 | { 222 | const msg = JSON.stringify({ 223 | message: 'HELLO FOLKS!' 224 | }) 225 | const afterOneSecond = new Date(now + 1000).toISOString() 226 | const query = ` 227 | mutation($body: String!, $queueId: ID, $when: DateTime!) { 228 | saveMessage(input: { queueId: $queueId, body: $body, when: $when }) { 229 | id 230 | when 231 | } 232 | } 233 | ` 234 | 235 | const res = await server.inject({ 236 | method: 'POST', 237 | url: '/graphql', 238 | headers: { 239 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 240 | }, 241 | payload: { 242 | query, 243 | variables: { 244 | body: msg, 245 | queueId, 246 | callbackUrl: targetUrl, 247 | when: afterOneSecond 248 | } 249 | } 250 | }) 251 | const body = res.json() 252 | equal(res.statusCode, 200) 253 | 254 | const { data } = body 255 | equal(data.saveMessage.when, afterOneSecond) 256 | } 257 | 258 | const [calledAt] = await p 259 | equal(calledAt - now >= 1000, true) 260 | }) 261 | 262 | test('only admins can write', async ({ teardown, equal, plan, same }) => { 263 | plan(4) 264 | const server = await buildServer(teardown) 265 | 266 | const targetUrl = 'http://localhost:4242' 267 | 268 | { 269 | const res = await server.inject({ 270 | method: 'POST', 271 | url: '/graphql', 272 | payload: { 273 | query: ` 274 | mutation($callbackUrl: String!) { 275 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 276 | id 277 | } 278 | } 279 | `, 280 | variables: { 281 | callbackUrl: targetUrl 282 | } 283 | } 284 | }) 285 | equal(res.statusCode, 200) 286 | const body = res.json() 287 | equal(body.errors[0].message, 'operation not allowed') 288 | } 289 | 290 | { 291 | const msg = JSON.stringify({ 292 | message: 'HELLO FOLKS!' 293 | }) 294 | const query = ` 295 | mutation($body: String!, $queueId: ID) { 296 | saveMessage(input: { queueId: $queueId, body: $body }) { 297 | id 298 | when 299 | } 300 | } 301 | ` 302 | 303 | const res = await server.inject({ 304 | method: 'POST', 305 | url: '/graphql', 306 | payload: { 307 | query, 308 | variables: { 309 | body: msg, 310 | queueId: 1 311 | } 312 | } 313 | }) 314 | const body = res.json() 315 | equal(res.statusCode, 200) 316 | equal(body.errors[0].message, 'operation not allowed') 317 | } 318 | }) 319 | 320 | test('`text plain` content type header in the Queue', async ({ teardown, equal, plan, same }) => { 321 | plan(5) 322 | const ee = new EventEmitter() 323 | const server = await buildServer(teardown) 324 | 325 | const target = Fastify() 326 | target.post('/', async (req, reply) => { 327 | same(req.body, 'HELLO FOLKS!', 'message is equal') 328 | ee.emit('called') 329 | return { ok: true } 330 | }) 331 | 332 | teardown(() => target.close()) 333 | await target.listen({ port: 0 }) 334 | const targetUrl = `http://localhost:${target.server.address().port}` 335 | 336 | let queueId 337 | { 338 | const res = await server.inject({ 339 | method: 'POST', 340 | url: '/graphql', 341 | headers: { 342 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 343 | }, 344 | payload: { 345 | query: ` 346 | mutation($callbackUrl: String!) { 347 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST", headers: "{ \\"content-type\\": \\"text/plain\\" }" }) { 348 | id 349 | } 350 | } 351 | `, 352 | variables: { 353 | callbackUrl: targetUrl 354 | } 355 | } 356 | }) 357 | equal(res.statusCode, 200) 358 | const body = res.json() 359 | const { data } = body 360 | queueId = data.saveQueue.id 361 | equal(queueId, '1') 362 | } 363 | 364 | const p = once(ee, 'called') 365 | { 366 | const msg = 'HELLO FOLKS!' 367 | const now = Date.now() 368 | const res = await server.inject({ 369 | method: 'POST', 370 | url: '/graphql', 371 | headers: { 372 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 373 | }, 374 | payload: { 375 | query: ` 376 | mutation($body: String!, $queueId: ID) { 377 | saveMessage(input: { queueId: $queueId, body: $body } ) { 378 | id 379 | when 380 | } 381 | } 382 | `, 383 | variables: { 384 | body: msg, 385 | queueId 386 | } 387 | } 388 | }) 389 | const body = res.json() 390 | equal(res.statusCode, 200) 391 | const { data } = body 392 | const when = new Date(data.saveMessage.when) 393 | equal(when.getTime() - now >= 0, true) 394 | } 395 | 396 | await p 397 | }) 398 | -------------------------------------------------------------------------------- /test/retries.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { buildServer, adminSecret, cleandb } = require('./helper') 4 | const { test, beforeEach } = require('tap') 5 | const { once, EventEmitter } = require('events') 6 | const Fastify = require('fastify') 7 | 8 | beforeEach(cleandb) 9 | 10 | test('retries on failure', async ({ teardown, equal, plan, same }) => { 11 | plan(6) 12 | const ee = new EventEmitter() 13 | const server = await buildServer(teardown) 14 | 15 | const target = Fastify() 16 | let called = 0 17 | target.post('/', async (req, reply) => { 18 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 19 | ee.emit('called') 20 | if (called++ === 0) { 21 | throw new Error('first call') 22 | } 23 | return { ok: true } 24 | }) 25 | 26 | teardown(() => target.close()) 27 | await target.listen({ port: 0 }) 28 | const targetUrl = `http://localhost:${target.server.address().port}` 29 | 30 | let queueId 31 | { 32 | const res = await server.inject({ 33 | method: 'POST', 34 | url: '/graphql', 35 | headers: { 36 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 37 | }, 38 | payload: { 39 | query: ` 40 | mutation($callbackUrl: String!) { 41 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 42 | id 43 | } 44 | } 45 | `, 46 | variables: { 47 | callbackUrl: targetUrl 48 | } 49 | } 50 | }) 51 | equal(res.statusCode, 200) 52 | const body = res.json() 53 | const { data } = body 54 | queueId = data.saveQueue.id 55 | equal(queueId, '1') 56 | } 57 | 58 | let p = once(ee, 'called') 59 | { 60 | const msg = JSON.stringify({ 61 | message: 'HELLO FOLKS!' 62 | }) 63 | const now = Date.now() 64 | const query = ` 65 | mutation($body: String!, $queueId: ID) { 66 | saveMessage(input: { queueId: $queueId, body: $body }) { 67 | id 68 | when 69 | } 70 | } 71 | ` 72 | 73 | const res = await server.inject({ 74 | method: 'POST', 75 | url: '/graphql', 76 | headers: { 77 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 78 | }, 79 | payload: { 80 | query, 81 | variables: { 82 | body: msg, 83 | queueId 84 | } 85 | } 86 | }) 87 | const body = res.json() 88 | equal(res.statusCode, 200) 89 | 90 | const { data } = body 91 | const when = new Date(data.saveMessage.when) 92 | equal(when.getTime() - now >= 0, true) 93 | } 94 | 95 | await p 96 | p = once(ee, 'called') 97 | await p 98 | }) 99 | 100 | test('send a message to the dead letter queue after retries are completed', async ({ teardown, equal, plan, same }) => { 101 | plan(9) 102 | const ee = new EventEmitter() 103 | const server = await buildServer(teardown) 104 | 105 | const target = Fastify() 106 | target.post('/', async (req, reply) => { 107 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 108 | throw new Error('This is down') 109 | }) 110 | 111 | teardown(() => target.close()) 112 | await target.listen({ port: 0 }) 113 | const targetUrl = `http://localhost:${target.server.address().port}` 114 | 115 | const deadLetterTarget = Fastify() 116 | deadLetterTarget.post('/', async (req, reply) => { 117 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 118 | ee.emit('called') 119 | return { ok: true } 120 | }) 121 | 122 | teardown(() => deadLetterTarget.close()) 123 | await deadLetterTarget.listen({ port: 0 }) 124 | const deadLetterTargetURL = `http://localhost:${deadLetterTarget.server.address().port}` 125 | 126 | let deadLetterQueue 127 | { 128 | const res = await server.inject({ 129 | method: 'POST', 130 | url: '/graphql', 131 | headers: { 132 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 133 | }, 134 | payload: { 135 | query: ` 136 | mutation($callbackUrl: String!) { 137 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 138 | id 139 | } 140 | } 141 | `, 142 | variables: { 143 | callbackUrl: deadLetterTargetURL 144 | } 145 | } 146 | }) 147 | equal(res.statusCode, 200) 148 | const body = res.json() 149 | const { data } = body 150 | deadLetterQueue = data.saveQueue.id 151 | equal(deadLetterQueue, '1') 152 | } 153 | 154 | let queueId 155 | { 156 | const res = await server.inject({ 157 | method: 'POST', 158 | url: '/graphql', 159 | headers: { 160 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 161 | }, 162 | payload: { 163 | query: ` 164 | mutation($callbackUrl: String!, $deadLetterQueueId: ID) { 165 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST", deadLetterQueueId: $deadLetterQueueId, maxRetries: 1 }) { 166 | id 167 | } 168 | } 169 | `, 170 | variables: { 171 | callbackUrl: targetUrl, 172 | deadLetterQueueId: deadLetterQueue 173 | } 174 | } 175 | }) 176 | equal(res.statusCode, 200) 177 | const body = res.json() 178 | const { data } = body 179 | queueId = data.saveQueue.id 180 | equal(queueId, '2') 181 | } 182 | 183 | const p = once(ee, 'called') 184 | { 185 | const msg = JSON.stringify({ 186 | message: 'HELLO FOLKS!' 187 | }) 188 | const now = Date.now() 189 | const query = ` 190 | mutation($body: String!, $queueId: ID) { 191 | saveMessage(input: { queueId: $queueId, body: $body }) { 192 | id 193 | when 194 | } 195 | } 196 | ` 197 | 198 | const res = await server.inject({ 199 | method: 'POST', 200 | url: '/graphql', 201 | headers: { 202 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 203 | }, 204 | payload: { 205 | query, 206 | variables: { 207 | body: msg, 208 | queueId 209 | } 210 | } 211 | }) 212 | const body = res.json() 213 | equal(res.statusCode, 200) 214 | 215 | const { data } = body 216 | const when = new Date(data.saveMessage.when) 217 | equal(when.getTime() - now >= 0, true) 218 | } 219 | 220 | await p 221 | }) 222 | 223 | test('send a message to the dead letter queue after retries are completed without content-type', async ({ teardown, equal, plan, same }) => { 224 | plan(9) 225 | const ee = new EventEmitter() 226 | const server = await buildServer(teardown) 227 | 228 | const target = Fastify() 229 | target.post('/', (req, reply) => { 230 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 231 | reply.status(500).send('This is down') 232 | }) 233 | 234 | teardown(() => target.close()) 235 | await target.listen({ port: 0 }) 236 | const targetUrl = `http://localhost:${target.server.address().port}` 237 | 238 | const deadLetterTarget = Fastify() 239 | deadLetterTarget.post('/', async (req, reply) => { 240 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 241 | ee.emit('called') 242 | return { ok: true } 243 | }) 244 | 245 | teardown(() => deadLetterTarget.close()) 246 | await deadLetterTarget.listen({ port: 0 }) 247 | const deadLetterTargetURL = `http://localhost:${deadLetterTarget.server.address().port}` 248 | 249 | let deadLetterQueue 250 | { 251 | const res = await server.inject({ 252 | method: 'POST', 253 | url: '/graphql', 254 | headers: { 255 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 256 | }, 257 | payload: { 258 | query: ` 259 | mutation($callbackUrl: String!) { 260 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 261 | id 262 | } 263 | } 264 | `, 265 | variables: { 266 | callbackUrl: deadLetterTargetURL 267 | } 268 | } 269 | }) 270 | equal(res.statusCode, 200) 271 | const body = res.json() 272 | const { data } = body 273 | deadLetterQueue = data.saveQueue.id 274 | equal(deadLetterQueue, '1') 275 | } 276 | 277 | let queueId 278 | { 279 | const res = await server.inject({ 280 | method: 'POST', 281 | url: '/graphql', 282 | headers: { 283 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 284 | }, 285 | payload: { 286 | query: ` 287 | mutation($callbackUrl: String!, $deadLetterQueueId: ID) { 288 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST", deadLetterQueueId: $deadLetterQueueId, maxRetries: 1 }) { 289 | id 290 | } 291 | } 292 | `, 293 | variables: { 294 | callbackUrl: targetUrl, 295 | deadLetterQueueId: deadLetterQueue 296 | } 297 | } 298 | }) 299 | equal(res.statusCode, 200) 300 | const body = res.json() 301 | const { data } = body 302 | queueId = data.saveQueue.id 303 | equal(queueId, '2') 304 | } 305 | 306 | const p = once(ee, 'called') 307 | { 308 | const msg = JSON.stringify({ 309 | message: 'HELLO FOLKS!' 310 | }) 311 | const now = Date.now() 312 | const query = ` 313 | mutation($body: String!, $queueId: ID) { 314 | saveMessage(input: { queueId: $queueId, body: $body }) { 315 | id 316 | when 317 | } 318 | } 319 | ` 320 | 321 | const res = await server.inject({ 322 | method: 'POST', 323 | url: '/graphql', 324 | headers: { 325 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 326 | }, 327 | payload: { 328 | query, 329 | variables: { 330 | body: msg, 331 | queueId 332 | } 333 | } 334 | }) 335 | const body = res.json() 336 | equal(res.statusCode, 200) 337 | 338 | const { data } = body 339 | const when = new Date(data.saveMessage.when) 340 | equal(when.getTime() - now >= 0, true) 341 | } 342 | 343 | await p 344 | }) 345 | 346 | test('send a message to the dead letter queue after retries are completed with text/plain', async ({ teardown, equal, plan, same }) => { 347 | plan(9) 348 | const ee = new EventEmitter() 349 | const server = await buildServer(teardown) 350 | 351 | const target = Fastify() 352 | target.post('/', (req, reply) => { 353 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 354 | reply.status(500).headers({ 'content-type': 'text/plain' }).send('This is down') 355 | }) 356 | 357 | teardown(() => target.close()) 358 | await target.listen({ port: 0 }) 359 | const targetUrl = `http://localhost:${target.server.address().port}` 360 | 361 | const deadLetterTarget = Fastify() 362 | deadLetterTarget.post('/', async (req, reply) => { 363 | same(req.body, { message: 'HELLO FOLKS!' }, 'message is equal') 364 | ee.emit('called') 365 | return { ok: true } 366 | }) 367 | 368 | teardown(() => deadLetterTarget.close()) 369 | await deadLetterTarget.listen({ port: 0 }) 370 | const deadLetterTargetURL = `http://localhost:${deadLetterTarget.server.address().port}` 371 | 372 | let deadLetterQueue 373 | { 374 | const res = await server.inject({ 375 | method: 'POST', 376 | url: '/graphql', 377 | headers: { 378 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 379 | }, 380 | payload: { 381 | query: ` 382 | mutation($callbackUrl: String!) { 383 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST" }) { 384 | id 385 | } 386 | } 387 | `, 388 | variables: { 389 | callbackUrl: deadLetterTargetURL 390 | } 391 | } 392 | }) 393 | equal(res.statusCode, 200) 394 | const body = res.json() 395 | const { data } = body 396 | deadLetterQueue = data.saveQueue.id 397 | equal(deadLetterQueue, '1') 398 | } 399 | 400 | let queueId 401 | { 402 | const res = await server.inject({ 403 | method: 'POST', 404 | url: '/graphql', 405 | headers: { 406 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 407 | }, 408 | payload: { 409 | query: ` 410 | mutation($callbackUrl: String!, $deadLetterQueueId: ID) { 411 | saveQueue(input: { name: "test", callbackUrl: $callbackUrl, method: "POST", deadLetterQueueId: $deadLetterQueueId, maxRetries: 1 }) { 412 | id 413 | } 414 | } 415 | `, 416 | variables: { 417 | callbackUrl: targetUrl, 418 | deadLetterQueueId: deadLetterQueue 419 | } 420 | } 421 | }) 422 | equal(res.statusCode, 200) 423 | const body = res.json() 424 | const { data } = body 425 | queueId = data.saveQueue.id 426 | equal(queueId, '2') 427 | } 428 | 429 | const p = once(ee, 'called') 430 | { 431 | const msg = JSON.stringify({ 432 | message: 'HELLO FOLKS!' 433 | }) 434 | const now = Date.now() 435 | const query = ` 436 | mutation($body: String!, $queueId: ID) { 437 | saveMessage(input: { queueId: $queueId, body: $body }) { 438 | id 439 | when 440 | } 441 | } 442 | ` 443 | 444 | const res = await server.inject({ 445 | method: 'POST', 446 | url: '/graphql', 447 | headers: { 448 | 'X-PLATFORMATIC-ADMIN-SECRET': adminSecret 449 | }, 450 | payload: { 451 | query, 452 | variables: { 453 | body: msg, 454 | queueId 455 | } 456 | } 457 | }) 458 | const body = res.json() 459 | equal(res.statusCode, 200) 460 | 461 | const { data } = body 462 | const when = new Date(data.saveMessage.when) 463 | equal(when.getTime() - now >= 0, true) 464 | } 465 | 466 | await p 467 | }) 468 | -------------------------------------------------------------------------------- /types/Cron.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Cron 3 | * A Cron 4 | */ 5 | declare interface Cron { 6 | id?: number; 7 | body?: string | null; 8 | createdAt?: string | null; 9 | headers?: { 10 | [name: string]: any; 11 | } | null; 12 | queueId: number; 13 | schedule: string; 14 | updatedAt?: string | null; 15 | } 16 | export { Cron }; 17 | -------------------------------------------------------------------------------- /types/Message.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Message 3 | * A Message 4 | */ 5 | declare interface Message { 6 | id?: number; 7 | body?: string | null; 8 | createdAt?: string | null; 9 | cronId?: number | null; 10 | failed: boolean; 11 | headers?: { 12 | [name: string]: any; 13 | } | null; 14 | queueId: number; 15 | retries: number; 16 | sentAt?: string | null; 17 | updatedAt?: string | null; 18 | when: string; 19 | } 20 | export { Message }; 21 | -------------------------------------------------------------------------------- /types/Page.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Page 3 | * A Page 4 | */ 5 | declare interface Page { 6 | id?: number; 7 | title?: string | null; 8 | } 9 | export { Page }; 10 | -------------------------------------------------------------------------------- /types/Queue.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Queue 3 | * A Queue 4 | */ 5 | declare interface Queue { 6 | id?: number; 7 | callbackUrl: string; 8 | createdAt?: string | null; 9 | deadLetterQueueId?: number | null; 10 | headers?: { 11 | [name: string]: any; 12 | } | null; 13 | maxRetries: number; 14 | method: string; 15 | name: string; 16 | updatedAt?: string | null; 17 | } 18 | export { Queue }; 19 | -------------------------------------------------------------------------------- /types/index.d.ts: -------------------------------------------------------------------------------- 1 | import { Cron } from './Cron' 2 | import { Message } from './Message' 3 | import { Page } from './Page' 4 | import { Queue } from './Queue' 5 | 6 | interface EntityTypes { 7 | Cron: Cron 8 | Message: Message 9 | Page: Page 10 | Queue: Queue 11 | } 12 | 13 | export { EntityTypes, Cron, Message, Page, Queue } --------------------------------------------------------------------------------